diff --git a/ts/packages/shell/src/main/index.ts b/ts/packages/shell/src/main/index.ts index 61e15630..faa34319 100644 --- a/ts/packages/shell/src/main/index.ts +++ b/ts/packages/shell/src/main/index.ts @@ -698,18 +698,22 @@ async function initialize() { }); ipcMain.on("open-image-file", async () => { - // TODO: imeplement - const result = await dialog.showOpenDialog(mainWindow); + const result = await dialog.showOpenDialog(mainWindow, { + filters: [ + { + name: "Image files", + extensions: ["png", "jpg", "jpeg", "gif"], + }, + ], + }); + if (result && !result.canceled) { let paths = result.filePaths; if (paths && paths.length > 0) { - const content = readFileSync(paths[0], "utf-8").toString(); - console.log(content); - return content; + const content = readFileSync(paths[0], "base64"); + chatView.webContents.send("file-selected", paths[0], content); } } - - return null; }); ipcMain.on( diff --git a/ts/packages/shell/src/preload/electronTypes.ts b/ts/packages/shell/src/preload/electronTypes.ts index a3c73171..d8c86dc2 100644 --- a/ts/packages/shell/src/preload/electronTypes.ts +++ b/ts/packages/shell/src/preload/electronTypes.ts @@ -75,6 +75,13 @@ export interface ClientAPI { onChatHistory( callback: (e: Electron.IpcRendererEvent, chatHistory: string) => void, ): void; + onFileSelected( + callback: ( + e: Electron.IpcRendererEvent, + fileName: string, + fileContent: string, + ) => void, + ): void; registerClientIO(clientIO: ClientIO); } diff --git a/ts/packages/shell/src/preload/index.ts b/ts/packages/shell/src/preload/index.ts index 9cfb25e2..71744cce 100644 --- a/ts/packages/shell/src/preload/index.ts +++ b/ts/packages/shell/src/preload/index.ts @@ -46,6 +46,9 @@ const api: ClientAPI = { onChatHistory(callback) { ipcRenderer.on("chat-history", callback); }, + onFileSelected(callback) { + ipcRenderer.on("file-selected", callback); + }, registerClientIO: (clientIO: ClientIO) => { if (clientIORegistered) { throw new Error("ClientIO already registered"); diff --git a/ts/packages/shell/src/renderer/assets/styles.less b/ts/packages/shell/src/renderer/assets/styles.less index bbd6ef39..fe293ad3 100644 --- a/ts/packages/shell/src/renderer/assets/styles.less +++ b/ts/packages/shell/src/renderer/assets/styles.less @@ -476,6 +476,12 @@ table.table-message td { margin-right: 8px; } +.agent-name:hover .clickable { + cursor: pointer; + text-decoration: underline; + font-weight: bolder; +} + .agent-name[action-data]:hover:after { content: attr(action-data); position: absolute; @@ -510,17 +516,8 @@ table.table-message td { right: 0; } -.chat-message-explained[data-expl]:hover:after { - content: attr(data-expl); - position: absolute; - top: 0; - right: 0; +.chat-message-action-data { white-space: nowrap; - background-color: lavender; - color: gray; - height: 100%; - border-radius: 5px; - margin-right: 5px; font-size: 80%; } @@ -555,8 +552,8 @@ table.table-message td { } .chat-input-image { - max-width: 320px; - max-height: 320px; + max-width: 100%; + max-height: 100%; width: auto; height: auto; overflow: auto; diff --git a/ts/packages/shell/src/renderer/src/chatInput.ts b/ts/packages/shell/src/renderer/src/chatInput.ts index 23d4dac2..1469eb57 100644 --- a/ts/packages/shell/src/renderer/src/chatInput.ts +++ b/ts/packages/shell/src/renderer/src/chatInput.ts @@ -263,18 +263,6 @@ export class ChatInput { e.preventDefault(); }; - // this.fileInput = document.createElement("input"); - // this.fileInput.type = "file"; - // this.fileInput.classList.add("chat-message-hidden"); - // this.fileInput.id = "image_upload"; - // this.inputContainer.append(this.fileInput); - // this.fileInput.accept = "image/*,.jpg,.png,.gif"; - // this.fileInput.onchange = () => { - // if (this.fileInput.files && this.fileInput.files?.length > 0) { - // this.loadImageFile(this.fileInput.files[0]); - // } - // }; - this.micButton = document.createElement("button"); this.micButton.appendChild(iconMicrophone()); this.micButton.id = buttonId; @@ -348,15 +336,27 @@ export class ChatInput { this.inputContainer.appendChild(this.sendButton); } + /** + * Loads the contents of the supplied image into the input text box. + * @param file The file whose contents to load + */ async loadImageFile(file: File) { let buffer: ArrayBuffer = await file.arrayBuffer(); - let dropImg: HTMLImageElement = document.createElement("img"); - let mimeType = file.name + this.loadImageContent(file.name, _arrayBufferToBase64(buffer)); + } + + /** + * Creates and sets an image in the input text area. + * @param mimeType The mime type of the supplied image content + * @param content The base64 encoded image content + */ + public async loadImageContent(fileName: string, content: string) { + let mimeType = fileName .toLowerCase() - .substring(file.name.lastIndexOf(".") + 1, file.name.length); + .substring(fileName.lastIndexOf(".") + 1, fileName.length); - if (file.name.toLowerCase().endsWith(".jpg")) { + if (fileName.toLowerCase().endsWith(".jpg")) { mimeType = "jpeg"; } @@ -365,13 +365,15 @@ export class ChatInput { "jpeg", "png", ]); + if (!supportedMimeTypes.has(mimeType)) { - console.log(`Unsupported MIME type for '${file.name}'`); + console.log(`Unsupported MIME type for '${fileName}'`); this.textarea.getTextEntry().innerText = `Unsupported file type '${mimeType}'. Supported types: ${Array.from(supportedMimeTypes).toString()}`; return; } - dropImg.src = - `data:image/${mimeType};base64,` + _arrayBufferToBase64(buffer); + + let dropImg: HTMLImageElement = document.createElement("img"); + dropImg.src = `data:image/${mimeType};base64,` + content; dropImg.className = "chat-input-dropImage"; diff --git a/ts/packages/shell/src/renderer/src/main.ts b/ts/packages/shell/src/renderer/src/main.ts index 26978d0c..09412143 100644 --- a/ts/packages/shell/src/renderer/src/main.ts +++ b/ts/packages/shell/src/renderer/src/main.ts @@ -22,6 +22,7 @@ import { createWebSocket, webapi, webdispatcher } from "./webSocketAPI"; import * as jose from "jose"; import { AppAgentEvent } from "@typeagent/agent-sdk"; import { ClientIO, Dispatcher } from "agent-dispatcher"; +import { swapContent } from "./setContent"; export function getClientAPI(): ClientAPI { if (globalThis.api !== undefined) { @@ -321,10 +322,32 @@ function addEvents( lastSeparatorText!.innerText = getDateDifferenceDescription(new Date(), timeStamp); } + + // rewire up action-data click handler + const nameDiv = div.querySelector(".agent-name.clickable"); + if (nameDiv != null) { + const messageDiv = div.querySelector( + ".chat-message-content", + ); + + if (messageDiv) { + nameDiv.addEventListener("click", () => { + swapContent( + nameDiv as HTMLSpanElement, + messageDiv as HTMLDivElement, + ); + }); + } + } + + // TODO: wire up any other functionality (player agent?) } } } }); + api.onFileSelected((_, fileName: string, fileContent: string) => { + chatView.chatInput.loadImageContent(fileName, fileContent); + }); } function showNotifications( diff --git a/ts/packages/shell/src/renderer/src/messageContainer.ts b/ts/packages/shell/src/renderer/src/messageContainer.ts index 857c15bf..6fbb835e 100644 --- a/ts/packages/shell/src/renderer/src/messageContainer.ts +++ b/ts/packages/shell/src/renderer/src/messageContainer.ts @@ -14,31 +14,12 @@ import { } from "agent-dispatcher"; import { ChoicePanel, InputChoice } from "./choicePanel"; -import { setContent } from "./setContent"; +import { setContent, swapContent } from "./setContent"; import { ChatView } from "./chatView"; import { iconCheckMarkCircle, iconRoadrunner, iconX } from "./icon"; import { TemplateEditor } from "./templateEditor"; import { SettingsView } from "./settingsView"; -function createTimestampDiv(timestamp: Date, className: string) { - const timeStampDiv = document.createElement("div"); - timeStampDiv.classList.add(className); - - const nameSpan = document.createElement("span"); - nameSpan.className = "agent-name"; - timeStampDiv.appendChild(nameSpan); // name placeholder - - const dateSpan = document.createElement("span"); - dateSpan.className = "timestring"; - dateSpan.setAttribute("data", timestamp.toString()); - - timeStampDiv.appendChild(dateSpan); // time string - - dateSpan.innerText = "- " + timestamp.toLocaleTimeString(); - - return timeStampDiv; -} - function updateMetrics( mainMetricsDiv: HTMLDivElement, markMetricsDiv: HTMLDivElement, @@ -100,6 +81,7 @@ export class MessageContainer { private readonly messageDiv: HTMLDivElement; private readonly timestampDiv: HTMLDivElement; private readonly iconDiv?: HTMLDivElement; + private nameSpan: HTMLSpanElement; private metricsDiv?: { mainMetricsDiv: HTMLDivElement; @@ -159,14 +141,36 @@ export class MessageContainer { if (this.action !== undefined && !Array.isArray(this.action)) { label.setAttribute( "action-data", - JSON.stringify(this.action, undefined, 2), + "
" +
+                        JSON.stringify(this.action, undefined, 2) +
+                        "
", ); + + // mark the span as clickable + this.nameSpan.classList.add("clickable"); } this.iconDiv.innerText = this.sourceIcon; } } + private createTimestampDiv(timestamp: Date, className: string) { + const timeStampDiv = document.createElement("div"); + timeStampDiv.classList.add(className); + + timeStampDiv.appendChild(this.nameSpan); // name placeholder + + const dateSpan = document.createElement("span"); + dateSpan.className = "timestring"; + dateSpan.setAttribute("data", timestamp.toString()); + + timeStampDiv.appendChild(dateSpan); // time string + + dateSpan.innerText = "- " + timestamp.toLocaleTimeString(); + + return timeStampDiv; + } + constructor( private chatView: ChatView, private settingsView: SettingsView, @@ -181,7 +185,25 @@ export class MessageContainer { const div = document.createElement("div"); div.className = `chat-message-container-${classNameSuffix}`; - const timestampDiv = createTimestampDiv( + // create the name placeholder + this.nameSpan = document.createElement("span"); + this.nameSpan.className = "agent-name"; + this.nameSpan.addEventListener("click", () => { + swapContent(this.nameSpan, this.messageDiv); + // const data: string = this.nameSpan.getAttribute("action-data") ?? ""; + // const originalMessage: string = this.messageDiv.innerHTML; + + // if (this.messageDiv.classList.contains("chat-message-action-data")) { + // this.messageDiv.classList.remove("chat-message-action-data"); + // } else { + // this.messageDiv.classList.add("chat-message-action-data"); + // } + + // this.nameSpan.setAttribute("action-data", originalMessage); + // this.messageDiv.innerHTML = data; + }); + + const timestampDiv = this.createTimestampDiv( new Date(), `chat-timestamp-${classNameSuffix}`, ); diff --git a/ts/packages/shell/src/renderer/src/setContent.ts b/ts/packages/shell/src/renderer/src/setContent.ts index 7287b1c8..29b17ab1 100644 --- a/ts/packages/shell/src/renderer/src/setContent.ts +++ b/ts/packages/shell/src/renderer/src/setContent.ts @@ -229,3 +229,26 @@ export function setContent( const parser = new DOMParser(); return parser.parseFromString(contentHtml, "text/html").body.innerText; } + +/** + * Takes the "action-data" attribute from the source element and places it as the html + * of the target element. + * @param sourceElement The source element. + * @param targetElement The target element. + */ +export function swapContent( + sourceElement: HTMLElement, + targetElement: HTMLElement, +) { + const data: string = sourceElement.getAttribute("action-data") ?? ""; + const originalMessage: string = targetElement.innerHTML; + + if (targetElement.classList.contains("chat-message-action-data")) { + targetElement.classList.remove("chat-message-action-data"); + } else { + targetElement.classList.add("chat-message-action-data"); + } + + sourceElement.setAttribute("action-data", originalMessage); + targetElement.innerHTML = data; +} diff --git a/ts/packages/shell/src/renderer/src/webSocketAPI.ts b/ts/packages/shell/src/renderer/src/webSocketAPI.ts index b1a15b34..7a16ef79 100644 --- a/ts/packages/shell/src/renderer/src/webSocketAPI.ts +++ b/ts/packages/shell/src/renderer/src/webSocketAPI.ts @@ -66,6 +66,10 @@ export const webapi: ClientAPI = { // TODO: implement proper message rehydration on mobile fnMap.set("chat-history", callback); }, + onFileSelected(callback) { + // TODO: implement image selection on mobile device + fnMap.set("file-selected", callback); + }, registerClientIO(clientIO: ClientIO) { if (clientIORegistered) { throw new Error("ClientIO already registered"); diff --git a/ts/packages/shell/test/hostWindow.spec.ts b/ts/packages/shell/test/hostWindow.spec.ts index 0084bf27..c4ad21f8 100644 --- a/ts/packages/shell/test/hostWindow.spec.ts +++ b/ts/packages/shell/test/hostWindow.spec.ts @@ -12,6 +12,7 @@ import { getAppPath, getLastAgentMessage, sendUserRequest, + sendUserRequestAndWaitForCompletion, sendUserRequestAndWaitForResponse, sendUserRequestFast, startShell, @@ -138,8 +139,6 @@ test.describe("Shell interface tests", () => { test("send button state", async ({}, testInfo) => { console.log(`Running test '${testInfo.title}`); - let agentMessageCount = 0; - // start the app const mainWindow = await startShell(); @@ -162,4 +161,58 @@ test.describe("Shell interface tests", () => { // close the application await exitApplication(mainWindow); }); + + test("command backstack", async ({}, testInfo) => { + console.log(`Running test '${testInfo.title}`); + + // start the app + const mainWindow = await startShell(); + + // issue some commands + const commands: string[] = ["@history", "@help", "@config agent"]; + for (let i = 0; i < commands.length; i++) { + await sendUserRequestAndWaitForCompletion(commands[i], mainWindow); + } + + // get the input box + const element = await mainWindow.waitForSelector("#phraseDiv"); + + // go through the command back stack to the end and make sure we get the expected + // results. (command and cursor location) + for (let i = commands.length - 1; i >= -1; i--) { + // press the up arrow + await element.press("ArrowUp"); + + // make sure that the text box now has the proper command + const text = await element.innerText(); + + // when we get to the end and hit up again it should still have the last command + let cmd = commands[i]; + if (i < 0) { + cmd = commands[0]; + } + + expect(text, "Wrong back stack command found!").toBe(cmd); + } + + // now reverse the process + for (let i = 1; i <= commands.length; i++) { + // press the up arrow + await element.press("ArrowDown"); + + // make sure that the text box now has the proper command + const text = await element.innerText(); + + // when we get to the end and hit up again it should still have the last command + let cmd = commands[i]; + if (i == commands.length) { + cmd = ""; + } + + expect(text, "Wrong back stack command found!").toBe(cmd); + } + + // close the application + await exitApplication(mainWindow); + }); }); diff --git a/ts/packages/shell/test/listAgent.spec.ts b/ts/packages/shell/test/listAgent.spec.ts index 6cb4bcd4..470c87a2 100644 --- a/ts/packages/shell/test/listAgent.spec.ts +++ b/ts/packages/shell/test/listAgent.spec.ts @@ -47,18 +47,4 @@ test.describe("List Agent Tests", () => { ], ); }); - - // test("delete_list", async ({}, testInfo) => { - // console.log(`Running test '${testInfo.title}`); - - // await testUserRequest( - // [ - // "delete the shopping list", - // "is there a shopping list?" - // ], - // [ - // "Cleared list: shopping", - // "List 'shopping' is empty" - // ]); - // }); });