Wiring asset's mimeType to TemplatePart #4605

Wiring asset's mimeType to TemplatePart #4605

> It's a good idea to open an issue first for discussion.

- [x] Tests pass
- [ ] Appropriate changes to documentation are included in the PR

---------

Co-authored-by: Leksyk <[email protected]>

Author: Leksyk

packages/breadboard/src/utils/template.ts

@@ -32,6 +32,7 @@ export type TemplatePart = {
    * dangling autowire or missing asset
    */
   invalid?: true;
+  mimeType?: string;
 };
 
 export type TemplatePartTransformCallback = (
@@ -171,8 +172,9 @@ class Template {
     return `{${JSON.stringify(part)}}`;
   }
 
-  static preamble({ type, path }: TemplatePart) {
-    return `{{"type": ${JSON.stringify(type)}, "path": ${JSON.stringify(path)}, "title": "`;
+  static preamble({ type, path, mimeType }: TemplatePart) {
+    const maybeMimeType = mimeType ? `"mimeType": "${mimeType}"` : "";
+    return `{{"type": ${JSON.stringify(type)}, "path": ${JSON.stringify(path)}, ${maybeMimeType} "title": "`;
   }
 
   static postamble() {

packages/shared-ui/src/elements/fast-access-menu/fast-access-menu.ts

@@ -13,7 +13,11 @@ import { SignalWatcher } from "@lit-labs/signals";
 import { css, html, LitElement, PropertyValues } from "lit";
 import { customElement, property, state } from "lit/decorators.js";
 import { Component, FastAccess, GraphAsset, Tool } from "../../state";
-import { GraphIdentifier, NodeIdentifier } from "@breadboard-ai/types";
+import {
+  GraphIdentifier,
+  LLMContent,
+  NodeIdentifier,
+} from "@breadboard-ai/types";
 import {
   FastAccessDismissedEvent,
   FastAccessSelectEvent,
@@ -337,6 +341,15 @@ export class FastAccessMenu extends SignalWatcher(LitElement) {
     return value;
   }
 
+  #getMimeType(data: LLMContent[]): string | undefined {
+    for (const element of data) {
+      for (const part of element.parts) {
+        if ("inlineData" in part && part.inlineData.mimeType)
+          return part.inlineData.mimeType;
+      }
+    }
+  }
+
   #emitCurrentItem() {
     let idx = this.selectedIndex;
     if (idx < this.#items.assets.length) {
@@ -345,7 +358,8 @@ export class FastAccessMenu extends SignalWatcher(LitElement) {
         new FastAccessSelectEvent(
           asset.path,
           asset.metadata?.title ?? "Untitled asset",
-          "asset"
+          "asset",
+          this.#getMimeType(asset.data)
         )
       );
       return;

packages/shared-ui/src/elements/input/text-editor/text-editor.ts

@@ -260,7 +260,7 @@ export class TextEditor extends LitElement {
     selection.addRange(this.#lastRange);
   }
 
-  #add(path: string, title: string, type: TemplatePartType) {
+  #add(path: string, title: string, type: TemplatePartType, mimeType?: string) {
     if (!this.#editorRef.value) {
       return null;
     }
@@ -282,7 +282,12 @@ export class TextEditor extends LitElement {
       label.classList.add(type);
       label.dataset.path = path;
 
-      preambleText.textContent = Template.preamble({ title, path, type });
+      preambleText.textContent = Template.preamble({
+        title,
+        path,
+        type,
+        mimeType,
+      });
       postamableText.textContent = Template.postamble();
       titleText.textContent = title;
       titleText.classList.add("visible");
@@ -774,7 +779,7 @@ export class TextEditor extends LitElement {
         @bbfastaccessselect=${(evt: FastAccessSelectEvent) => {
           this.#hideFastAccess();
           this.#restoreLastRange();
-          this.#add(evt.path, evt.title, evt.accessType);
+          this.#add(evt.path, evt.title, evt.accessType, evt.mimeType);
 
           this.#captureEditorValue();
         }}

packages/shared-ui/src/events/events.ts

@@ -1331,7 +1331,8 @@ export class FastAccessSelectEvent extends Event {
   constructor(
     public readonly path: string,
     public readonly title: string,
-    public readonly accessType: "asset" | "tool" | "in"
+    public readonly accessType: "asset" | "tool" | "in",
+    public readonly mimeType?: string
   ) {
     super(FastAccessSelectEvent.eventName, { ...eventInit });
   }