Action-data display is now inline with the messages.

ts/packages/shell/src/main/index.ts

@@ -698,18 +698,22 @@ async function initialize() {
     });
 
     ipcMain.on("open-image-file", async () => {
-        // TODO: imeplement
-        const result = await dialog.showOpenDialog(mainWindow);
+        const result = await dialog.showOpenDialog(mainWindow, {
+            filters: [
+                {
+                    name: "Image files",
+                    extensions: ["png", "jpg", "jpeg", "gif"],
+                },
+            ],
+        });
+
         if (result && !result.canceled) {
             let paths = result.filePaths;
             if (paths && paths.length > 0) {
-                const content = readFileSync(paths[0], "utf-8").toString();
-                console.log(content);
-                return content;
+                const content = readFileSync(paths[0], "base64");
+                chatView.webContents.send("file-selected", paths[0], content);
             }
         }
-
-        return null;
     });
 
     ipcMain.on(

ts/packages/shell/src/preload/electronTypes.ts

@@ -75,6 +75,13 @@ export interface ClientAPI {
     onChatHistory(
         callback: (e: Electron.IpcRendererEvent, chatHistory: string) => void,
     ): void;
+    onFileSelected(
+        callback: (
+            e: Electron.IpcRendererEvent,
+            fileName: string,
+            fileContent: string,
+        ) => void,
+    ): void;
     registerClientIO(clientIO: ClientIO);
 }
 

ts/packages/shell/src/preload/index.ts

@@ -46,6 +46,9 @@ const api: ClientAPI = {
     onChatHistory(callback) {
         ipcRenderer.on("chat-history", callback);
     },
+    onFileSelected(callback) {
+        ipcRenderer.on("file-selected", callback);
+    },
     registerClientIO: (clientIO: ClientIO) => {
         if (clientIORegistered) {
             throw new Error("ClientIO already registered");

ts/packages/shell/src/renderer/assets/styles.less

@@ -476,6 +476,12 @@ table.table-message td {
   margin-right: 8px;
 }
 
+.agent-name:hover .clickable {
+  cursor: pointer;
+  text-decoration: underline;
+  font-weight: bolder;
+}
+
 .agent-name[action-data]:hover:after {
   content: attr(action-data);
   position: absolute;
@@ -510,17 +516,8 @@ table.table-message td {
   right: 0;
 }
 
-.chat-message-explained[data-expl]:hover:after {
-  content: attr(data-expl);
-  position: absolute;
-  top: 0;
-  right: 0;
+.chat-message-action-data {
   white-space: nowrap;
-  background-color: lavender;
-  color: gray;
-  height: 100%;
-  border-radius: 5px;
-  margin-right: 5px;
   font-size: 80%;
 }
 
@@ -555,8 +552,8 @@ table.table-message td {
 }
 
 .chat-input-image {
-  max-width: 320px;
-  max-height: 320px;
+  max-width: 100%;
+  max-height: 100%;
   width: auto;
   height: auto;
   overflow: auto;

ts/packages/shell/src/renderer/src/chatInput.ts

@@ -263,18 +263,6 @@ export class ChatInput {
             e.preventDefault();
         };
 
-        // this.fileInput = document.createElement("input");
-        // this.fileInput.type = "file";
-        // this.fileInput.classList.add("chat-message-hidden");
-        // this.fileInput.id = "image_upload";
-        // this.inputContainer.append(this.fileInput);
-        // this.fileInput.accept = "image/*,.jpg,.png,.gif";
-        // this.fileInput.onchange = () => {
-        //     if (this.fileInput.files && this.fileInput.files?.length > 0) {
-        //         this.loadImageFile(this.fileInput.files[0]);
-        //     }
-        // };
-
         this.micButton = document.createElement("button");
         this.micButton.appendChild(iconMicrophone());
         this.micButton.id = buttonId;
@@ -348,15 +336,27 @@ export class ChatInput {
         this.inputContainer.appendChild(this.sendButton);
     }
 
+    /**
+     * Loads the contents of the supplied image into the input text box.
+     * @param file The file whose contents to load
+     */
     async loadImageFile(file: File) {
         let buffer: ArrayBuffer = await file.arrayBuffer();
 
-        let dropImg: HTMLImageElement = document.createElement("img");
-        let mimeType = file.name
+        this.loadImageContent(file.name, _arrayBufferToBase64(buffer));
+    }
+
+    /**
+     * Creates and sets an image in the input text area.
+     * @param mimeType The mime type of the supplied image content
+     * @param content The base64 encoded image content
+     */
+    public async loadImageContent(fileName: string, content: string) {
+        let mimeType = fileName
             .toLowerCase()
-            .substring(file.name.lastIndexOf(".") + 1, file.name.length);
+            .substring(fileName.lastIndexOf(".") + 1, fileName.length);
 
-        if (file.name.toLowerCase().endsWith(".jpg")) {
+        if (fileName.toLowerCase().endsWith(".jpg")) {
             mimeType = "jpeg";
         }
 
@@ -365,13 +365,15 @@ export class ChatInput {
             "jpeg",
             "png",
         ]);
+
         if (!supportedMimeTypes.has(mimeType)) {
-            console.log(`Unsupported MIME type for '${file.name}'`);
+            console.log(`Unsupported MIME type for '${fileName}'`);
             this.textarea.getTextEntry().innerText = `Unsupported file type '${mimeType}'. Supported types: ${Array.from(supportedMimeTypes).toString()}`;
             return;
         }
-        dropImg.src =
-            `data:image/${mimeType};base64,` + _arrayBufferToBase64(buffer);
+
+        let dropImg: HTMLImageElement = document.createElement("img");
+        dropImg.src = `data:image/${mimeType};base64,` + content;
 
         dropImg.className = "chat-input-dropImage";
 

ts/packages/shell/src/renderer/src/main.ts

@@ -22,6 +22,7 @@ import { createWebSocket, webapi, webdispatcher } from "./webSocketAPI";
 import * as jose from "jose";
 import { AppAgentEvent } from "@typeagent/agent-sdk";
 import { ClientIO, Dispatcher } from "agent-dispatcher";
+import { swapContent } from "./setContent";
 
 export function getClientAPI(): ClientAPI {
     if (globalThis.api !== undefined) {
@@ -321,10 +322,32 @@ function addEvents(
                         lastSeparatorText!.innerText =
                             getDateDifferenceDescription(new Date(), timeStamp);
                     }
+
+                    // rewire up action-data click handler
+                    const nameDiv = div.querySelector(".agent-name.clickable");
+                    if (nameDiv != null) {
+                        const messageDiv = div.querySelector(
+                            ".chat-message-content",
+                        );
+
+                        if (messageDiv) {
+                            nameDiv.addEventListener("click", () => {
+                                swapContent(
+                                    nameDiv as HTMLSpanElement,
+                                    messageDiv as HTMLDivElement,
+                                );
+                            });
+                        }
+                    }
+
+                    // TODO: wire up any other functionality (player agent?)
                 }
             }
         }
     });
+    api.onFileSelected((_, fileName: string, fileContent: string) => {
+        chatView.chatInput.loadImageContent(fileName, fileContent);
+    });
 }
 
 function showNotifications(

ts/packages/shell/src/renderer/src/messageContainer.ts

@@ -14,31 +14,12 @@ import {
 } from "agent-dispatcher";
 
 import { ChoicePanel, InputChoice } from "./choicePanel";
-import { setContent } from "./setContent";
+import { setContent, swapContent } from "./setContent";
 import { ChatView } from "./chatView";
 import { iconCheckMarkCircle, iconRoadrunner, iconX } from "./icon";
 import { TemplateEditor } from "./templateEditor";
 import { SettingsView } from "./settingsView";
 
-function createTimestampDiv(timestamp: Date, className: string) {
-    const timeStampDiv = document.createElement("div");
-    timeStampDiv.classList.add(className);
-
-    const nameSpan = document.createElement("span");
-    nameSpan.className = "agent-name";
-    timeStampDiv.appendChild(nameSpan); // name placeholder
-
-    const dateSpan = document.createElement("span");
-    dateSpan.className = "timestring";
-    dateSpan.setAttribute("data", timestamp.toString());
-
-    timeStampDiv.appendChild(dateSpan); // time string
-
-    dateSpan.innerText = "- " + timestamp.toLocaleTimeString();
-
-    return timeStampDiv;
-}
-
 function updateMetrics(
     mainMetricsDiv: HTMLDivElement,
     markMetricsDiv: HTMLDivElement,
@@ -100,6 +81,7 @@ export class MessageContainer {
     private readonly messageDiv: HTMLDivElement;
     private readonly timestampDiv: HTMLDivElement;
     private readonly iconDiv?: HTMLDivElement;
+    private nameSpan: HTMLSpanElement;
 
     private metricsDiv?: {
         mainMetricsDiv: HTMLDivElement;
@@ -159,14 +141,36 @@ export class MessageContainer {
             if (this.action !== undefined && !Array.isArray(this.action)) {
                 label.setAttribute(
                     "action-data",
-                    JSON.stringify(this.action, undefined, 2),
+                    "<pre>" +
+                        JSON.stringify(this.action, undefined, 2) +
+                        "</pre>",
                 );
+
+                // mark the span as clickable
+                this.nameSpan.classList.add("clickable");
             }
 
             this.iconDiv.innerText = this.sourceIcon;
         }
     }
 
+    private createTimestampDiv(timestamp: Date, className: string) {
+        const timeStampDiv = document.createElement("div");
+        timeStampDiv.classList.add(className);
+
+        timeStampDiv.appendChild(this.nameSpan); // name placeholder
+
+        const dateSpan = document.createElement("span");
+        dateSpan.className = "timestring";
+        dateSpan.setAttribute("data", timestamp.toString());
+
+        timeStampDiv.appendChild(dateSpan); // time string
+
+        dateSpan.innerText = "- " + timestamp.toLocaleTimeString();
+
+        return timeStampDiv;
+    }
+
     constructor(
         private chatView: ChatView,
         private settingsView: SettingsView,
@@ -181,7 +185,25 @@ export class MessageContainer {
         const div = document.createElement("div");
         div.className = `chat-message-container-${classNameSuffix}`;
 
-        const timestampDiv = createTimestampDiv(
+        // create the name placeholder
+        this.nameSpan = document.createElement("span");
+        this.nameSpan.className = "agent-name";
+        this.nameSpan.addEventListener("click", () => {
+            swapContent(this.nameSpan, this.messageDiv);
+            // const data: string = this.nameSpan.getAttribute("action-data") ?? "";
+            // const originalMessage: string = this.messageDiv.innerHTML;
+
+            // if (this.messageDiv.classList.contains("chat-message-action-data")) {
+            //     this.messageDiv.classList.remove("chat-message-action-data");
+            // } else {
+            //     this.messageDiv.classList.add("chat-message-action-data");
+            // }
+
+            // this.nameSpan.setAttribute("action-data", originalMessage);
+            // this.messageDiv.innerHTML = data;
+        });
+
+        const timestampDiv = this.createTimestampDiv(
             new Date(),
             `chat-timestamp-${classNameSuffix}`,
         );

ts/packages/shell/src/renderer/src/setContent.ts

@@ -229,3 +229,26 @@ export function setContent(
     const parser = new DOMParser();
     return parser.parseFromString(contentHtml, "text/html").body.innerText;
 }
+
+/**
+ * Takes the "action-data" attribute from the source element and places it as the html
+ * of the target element.
+ * @param sourceElement The source element.
+ * @param targetElement The target element.
+ */
+export function swapContent(
+    sourceElement: HTMLElement,
+    targetElement: HTMLElement,
+) {
+    const data: string = sourceElement.getAttribute("action-data") ?? "";
+    const originalMessage: string = targetElement.innerHTML;
+
+    if (targetElement.classList.contains("chat-message-action-data")) {
+        targetElement.classList.remove("chat-message-action-data");
+    } else {
+        targetElement.classList.add("chat-message-action-data");
+    }
+
+    sourceElement.setAttribute("action-data", originalMessage);
+    targetElement.innerHTML = data;
+}

ts/packages/shell/src/renderer/src/webSocketAPI.ts

@@ -66,6 +66,10 @@ export const webapi: ClientAPI = {
         // TODO: implement proper message rehydration on mobile
         fnMap.set("chat-history", callback);
     },
+    onFileSelected(callback) {
+        // TODO: implement image selection on mobile device
+        fnMap.set("file-selected", callback);
+    },
     registerClientIO(clientIO: ClientIO) {
         if (clientIORegistered) {
             throw new Error("ClientIO already registered");