Skip to content

Commit 8b20d63

Browse files
robgruenrobgruen
committed
Merge remote-tracking branch 'origin' into dev/robgruen/android
2 parents ac116eb + 1cc966a commit 8b20d63

File tree

20 files changed

+304
-60
lines changed

20 files changed

+304
-60
lines changed

‎dotnet/autoShell/AutoShell.cs

+15
Original file line numberDiff line numberDiff line change
@@ -361,6 +361,18 @@ static void CloseApplication(string friendlyName)
361361
}
362362
}
363363

364+
[DllImport("user32.dll", CharSet = CharSet.Auto)]
365+
private static extern int SystemParametersInfo(int uAction, int uParam, string lpvParam, int fuWinIni);
366+
367+
private const int SPI_SETDESKWALLPAPER = 20;
368+
private const int SPIF_UPDATEINIFILE = 0x01;
369+
private const int SPIF_SENDCHANGE = 0x02;
370+
371+
private static void SetDesktopWallpaper(string imagePath)
372+
{
373+
SystemParametersInfo(SPI_SETDESKWALLPAPER, 0, imagePath, SPIF_UPDATEINIFILE | SPIF_SENDCHANGE);
374+
}
375+
364376
static bool execLine(string line)
365377
{
366378
var quit = false;
@@ -418,6 +430,9 @@ static bool execLine(string line)
418430
var installedApps = GetAllInstalledAppsIds();
419431
Console.WriteLine(JsonConvert.SerializeObject(installedApps.Keys));
420432
break;
433+
case "setWallpaper":
434+
SetDesktopWallpaper(value);
435+
break;
421436
default:
422437
Debug.WriteLine("Unknown command: " + key);
423438
break;

‎dotnet/autoShell/autoShell.csproj

+2-1
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,11 @@
88
<OutputType>Exe</OutputType>
99
<RootNamespace>autoShell</RootNamespace>
1010
<AssemblyName>autoShell</AssemblyName>
11-
<TargetFrameworkVersion>v4.8.1</TargetFrameworkVersion>
11+
<TargetFrameworkVersion>v4.8</TargetFrameworkVersion>
1212
<FileAlignment>512</FileAlignment>
1313
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
1414
<Deterministic>true</Deterministic>
15+
<TargetFrameworkProfile />
1516
</PropertyGroup>
1617
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
1718
<PlatformTarget>AnyCPU</PlatformTarget>

‎ts/packages/agentSdk/src/action.ts

+2
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ export type ActionResultSuccessNoDisplay = {
2020
entities: Entity[];
2121
dynamicDisplayId?: string | undefined;
2222
dynamicDisplayNextRefreshMs?: number | undefined;
23+
additionalInstructions?: string[] | undefined;
2324
error?: undefined;
2425
};
2526

@@ -29,6 +30,7 @@ export type ActionResultSuccess = {
2930
entities: Entity[];
3031
dynamicDisplayId?: string | undefined;
3132
dynamicDisplayNextRefreshMs?: number | undefined;
33+
additionalInstructions?: string[] | undefined;
3234
error?: undefined;
3335
};
3436

‎ts/packages/agents/desktop/src/actionHandler.ts

+1
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ async function executeDesktopAction(
4646
const message = await runDesktopActions(
4747
action as DesktopActions,
4848
context.sessionContext.agentContext,
49+
context.sessionContext.sessionStorage!,
4950
);
5051
return createActionResult(message);
5152
}

‎ts/packages/agents/desktop/src/actionsSchema.ts

+9-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,8 @@ export type DesktopActions =
1010
| SwitchToWindowAction
1111
| SetVolumeAction
1212
| RestoreVolumeAction
13-
| MuteVolumeAction;
13+
| MuteVolumeAction
14+
| SetWallpaperAction;
1415

1516
// Launches a new program window on a Windows Desktop
1617
// Example:
@@ -88,6 +89,13 @@ export type MuteVolumeAction = {
8889
};
8990
};
9091

92+
export type SetWallpaperAction = {
93+
actionName: "setWallpaper";
94+
parameters: {
95+
filePath?: string; // The path to the file
96+
url?: string; // The url to the image
97+
};
98+
};
9199
export type KnownPrograms =
92100
| "chrome"
93101
| "word"

‎ts/packages/agents/desktop/src/connector.ts

+65
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,10 @@ import { ProgramNameIndex, loadProgramNameIndex } from "./programNameIndex.js";
77
import { Storage } from "@typeagent/agent-sdk";
88
import registerDebug from "debug";
99
import { DesktopActions } from "./actionsSchema.js";
10+
import fs from "node:fs";
11+
import path from "node:path";
12+
import os from "node:os";
13+
import { downloadImage } from "common-utils";
1014

1115
const debug = registerDebug("typeagent:desktop");
1216
const debugData = registerDebug("typeagent:desktop:data");
@@ -63,11 +67,72 @@ async function ensureAutomationProcess(agentContext: DesktopActionContext) {
6367
export async function runDesktopActions(
6468
action: DesktopActions,
6569
agentContext: DesktopActionContext,
70+
sessionStorage: Storage,
6671
) {
6772
let confirmationMessage = "OK";
6873
let actionData = "";
6974
const actionName = action.actionName;
7075
switch (actionName) {
76+
case "setWallpaper": {
77+
let file = action.parameters.filePath;
78+
const rootTypeAgentDir = path.join(os.homedir(), ".typeagent");
79+
80+
// if the requested image a URL, then download it
81+
if (action.parameters.url !== undefined) {
82+
file = `../downloaded_images/${path.basename(action.parameters.url)}`;
83+
if (path.extname(file).length == 0) {
84+
file += ".png";
85+
}
86+
if (
87+
await downloadImage(
88+
action.parameters.url,
89+
file,
90+
sessionStorage!,
91+
)
92+
) {
93+
file = file.substring(3);
94+
} else {
95+
confirmationMessage =
96+
"Failed to dowload the requested image.";
97+
break;
98+
}
99+
}
100+
101+
if (file !== undefined) {
102+
if (
103+
file.startsWith("/") ||
104+
file.indexOf(":") == 2 ||
105+
fs.existsSync(file)
106+
) {
107+
actionData = file;
108+
} else {
109+
// if the file path is relative we'll have to search for the image since we don't have root storage dir
110+
// TODO: add shared agent storage or known storage location (requires permissions, trusted agents, etc.)
111+
const files = fs
112+
.readdirSync(rootTypeAgentDir, { recursive: true })
113+
.filter((allFilesPaths) =>
114+
(allFilesPaths as string).endsWith(
115+
path.basename(file),
116+
),
117+
);
118+
119+
if (files.length > 0) {
120+
actionData = path.join(
121+
rootTypeAgentDir,
122+
files[0] as string,
123+
);
124+
} else {
125+
actionData = file;
126+
}
127+
}
128+
} else {
129+
confirmationMessage = "Unknown wallpaper location.";
130+
break;
131+
}
132+
133+
confirmationMessage = "Set wallpaper to " + actionData;
134+
break;
135+
}
71136
case "launchProgram": {
72137
actionData = await mapInputToAppName(
73138
action.parameters.name,

‎ts/packages/agents/image/src/imageActionHandler.ts

+33-8
Original file line numberDiff line numberDiff line change
@@ -8,19 +8,20 @@ import {
88
ActionResult,
99
ActionResultSuccess,
1010
} from "@typeagent/agent-sdk";
11-
import { StopWatch } from "common-utils";
11+
import { downloadImage, StopWatch } from "common-utils";
1212
import {
1313
createActionResult,
1414
createActionResultFromHtmlDisplayWithScript,
1515
} from "@typeagent/agent-sdk/helpers/action";
1616
import { bing, GeneratedImage, openai } from "aiclient";
1717
import { Image } from "../../../aiclient/dist/bing.js";
18-
import { randomBytes } from "crypto";
18+
import { randomBytes, randomUUID } from "crypto";
1919
import {
2020
CreateImageAction,
2121
FindImageAction,
2222
ImageAction,
2323
} from "./imageActionSchema.js";
24+
import path from "path";
2425

2526
export function instantiate(): AppAgent {
2627
return {
@@ -75,11 +76,6 @@ async function handlePhotoAction(
7576
result = createActionResult(
7677
`Unable to find any images for ${findImageAction.parameters.searchTerm}`,
7778
);
78-
// } else if (searchResults.length == 1) {
79-
// result = createActionResultFromHtmlDisplay(
80-
// `<img class="chat-input-image" src="${searchResults[0].contentUrl}" />`,
81-
// "Found 1 image.",
82-
// );
8379
} else {
8480
const urls: string[] = [];
8581
const captions: string[] = [];
@@ -88,6 +84,15 @@ async function handlePhotoAction(
8884
captions.push(findImageAction.parameters.searchTerm);
8985
});
9086
result = createCarouselForImages(urls, captions);
87+
88+
// add the found images to the entities
89+
for (let i = 0; i < searchResults.length; i++) {
90+
result.entities.push({
91+
name: path.basename(searchResults[i].contentUrl),
92+
type: ["image", "url", "search"],
93+
additionalEntityText: searchResults[i].contentUrl,
94+
});
95+
}
9196
}
9297
break;
9398
}
@@ -141,6 +146,23 @@ async function handlePhotoAction(
141146
captions.push(i.revised_prompt);
142147
});
143148
result = createCarouselForImages(urls, captions);
149+
150+
// save the generated image in the session store and add the image to the knoweledge store
151+
const id = randomUUID();
152+
const fileName = `../generated_images/${id.toString()}.png`;
153+
if (
154+
await downloadImage(
155+
urls[0],
156+
fileName,
157+
photoContext.sessionContext.sessionStorage!,
158+
)
159+
) {
160+
// add the generated image to the entities
161+
result.entities.push({
162+
name: fileName.substring(3),
163+
type: ["file", "image", "ai_generated"],
164+
});
165+
}
144166
}
145167
break;
146168
default:
@@ -153,6 +175,7 @@ function createCarouselForImages(
153175
images: string[],
154176
captions: string[],
155177
): ActionResultSuccess {
178+
let literal: string = `There are ${images.length} shown. `;
156179
const hash: string = randomBytes(4).readUInt32LE(0).toString();
157180
const jScript: string = `
158181
<script>
@@ -216,6 +239,8 @@ function createCarouselForImages(
216239
</div>`;
217240

218241
carouselDots += `<span class="dot ${hash}" onclick="slideShow_${hash}.currentSlide(${index + 1})"></span>`;
242+
243+
literal += `Image ${index + 1}: ${url}, Caption: ${captions[index]} `;
219244
});
220245

221246
const carousel_end: string = `
@@ -233,6 +258,6 @@ function createCarouselForImages(
233258

234259
return createActionResultFromHtmlDisplayWithScript(
235260
carousel_start + carousel + carousel_end + jScript,
236-
`There are ${images.length} shown.`,
261+
literal,
237262
);
238263
}

‎ts/packages/cache/src/explanation/requestAction.ts

+1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { AppAction, Entity } from "@typeagent/agent-sdk";
66
export type HistoryContext = {
77
promptSections: PromptSection[];
88
entities: Entity[];
9+
additionalInstructions?: string[] | undefined;
910
};
1011

1112
export function normalizeParamValue(value: ParamValueType) {

‎ts/packages/commonUtils/src/image.ts

+27
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
// Copyright (c) Microsoft Corporation.
22
// Licensed under the MIT License.
33

4+
import { Storage } from "@typeagent/agent-sdk";
5+
import { getBlob } from "aiclient";
46
import ExifReader from "exifreader";
57

68
export class CachedImageWithDetails {
@@ -30,3 +32,28 @@ export function extractRelevantExifTags(exifTags: ExifReader.Tags) {
3032
console.log(tags.replace("\n\n", "\n"));
3133
return tags;
3234
}
35+
36+
/**
37+
* Dowloads the supplied uri and saves it to local session storage
38+
* @param uri The uri of the image to download
39+
* @param fileName The name of the file to save the image locally as (including relative path)
40+
*/
41+
export async function downloadImage(
42+
uri: string,
43+
fileName: string,
44+
storage: Storage,
45+
): Promise<boolean> {
46+
return new Promise<boolean>(async (resolve) => {
47+
// save the generated image in the session store
48+
const blobResponse = await getBlob(uri);
49+
if (blobResponse.success) {
50+
const ab = Buffer.from(await blobResponse.data.arrayBuffer());
51+
52+
storage.write(fileName, ab.toString("base64"));
53+
54+
resolve(true);
55+
}
56+
57+
resolve(false);
58+
});
59+
}

‎ts/packages/commonUtils/src/indexNode.ts

+3-2
Original file line numberDiff line numberDiff line change
@@ -44,12 +44,13 @@ export * from "./profiler/profileReader.js";
4444

4545
export { createRpc } from "./rpc.js";
4646

47-
export { CachedImageWithDetails, getImageElement } from "./image.js";
47+
export * from "./image.js";
4848

4949
export {
5050
getFileExtensionForMimeType,
5151
getMimeTypeFromFileExtension as getMimeType,
52-
isMimeTypeSupported,
52+
isImageMimeTypeSupported,
53+
isImageFileType,
5354
} from "./mimeTypes.js";
5455

5556
export { getObjectProperty, setObjectProperty } from "./objectProperty.js";

‎ts/packages/commonUtils/src/mimeTypes.ts

+11-1
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ export function getMimeTypeFromFileExtension(fileExtension: string): string {
3838
throw "Unsupported file extension.";
3939
}
4040

41-
export function isMimeTypeSupported(mime: string): boolean {
41+
export function isImageMimeTypeSupported(mime: string): boolean {
4242
switch (mime) {
4343
case "image/png":
4444
case "image/jpg":
@@ -48,3 +48,13 @@ export function isMimeTypeSupported(mime: string): boolean {
4848
return false;
4949
}
5050
}
51+
52+
export function isImageFileType(fileExtension: string): boolean {
53+
if (fileExtension.startsWith(".")) {
54+
fileExtension = fileExtension.substring(1);
55+
}
56+
57+
const imageFileTypes: Set<string> = new Set<string>(["png", "jpg", "jpeg"]);
58+
59+
return imageFileTypes.has(fileExtension);
60+
}

‎ts/packages/dispatcher/src/action/actionHandlers.ts

+2
Original file line numberDiff line numberDiff line change
@@ -270,6 +270,8 @@ async function executeAction(
270270
result.entities,
271271
"assistant",
272272
systemContext.requestId,
273+
undefined,
274+
result.additionalInstructions,
273275
);
274276
}
275277

‎ts/packages/dispatcher/src/action/storageImpl.ts

+11-1
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import {
1313
DataProtectionScope,
1414
PersistenceCreator,
1515
} from "@azure/msal-node-extensions";
16+
import { isImageFileType } from "common-utils";
1617

1718
export function getStorage(name: string, baseDir: string): Storage {
1819
const getFullPath = (storagePath: string) => {
@@ -45,7 +46,16 @@ export function getStorage(name: string, baseDir: string): Storage {
4546
if (!fs.existsSync(dirName)) {
4647
await fs.promises.mkdir(dirName, { recursive: true });
4748
}
48-
return fs.promises.writeFile(fullPath, data);
49+
50+
// images are passed in as base64 strings so we need to encode them properly on disk
51+
if (isImageFileType(path.extname(storagePath))) {
52+
return fs.promises.writeFile(
53+
fullPath,
54+
Buffer.from(data, "base64"),
55+
);
56+
} else {
57+
return fs.promises.writeFile(fullPath, data);
58+
}
4959
},
5060
delete: async (storagePath: string) => {
5161
const fullPath = getFullPath(storagePath);

0 commit comments

Comments
 (0)