Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

list expander api #4993

Merged
merged 6 commits into from
Mar 21, 2025
+87 −82
Merged

list expander api #4993

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
99a5f39
Prepare to make `ListExpander` the main API for lists.
dglazkov Mar 21, 2025
2cb6e4c
Teach "Edit Image" about lists.
dglazkov Mar 21, 2025
5ad88bc
Teach RenderOutputs about lists.
dglazkov Mar 21, 2025
a46dbdd
Teach video generator about lists.
dglazkov Mar 21, 2025
ae27d2e
Teach Audio Generation about lists.
dglazkov Mar 21, 2025
1fd311a
docs(changeset): Teach various steps about lists.
dglazkov Mar 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/fancy-sides-punch.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"@breadboard-ai/a2": minor
---

Teach various steps about lists.
20 changes: 10 additions & 10 deletions packages/a2/bgl/a2.bgl.json
View file

Large diffs are not rendered by default.

136 changes: 68 additions & 68 deletions packages/a2/bgl/audio-generator.bgl.json
View file
Original file line number Diff line number Diff line change
@@ -1,74 +1,74 @@
{
"title": "A2 Audio Generation",
"description": "",
"version": "0.0.1",
"nodes": [],
"edges": [],
"metadata": {
"comments": [
{
"id": "comment-cc94afe8",
"text": "Intentionally Left Blank",
"metadata": {
"title": "Comment",
"visual": {
"x": 531,
"y": 374,
"collapsed": "expanded",
"outputHeight": 0
}
}
}
],
"visual": {
"presentation": {
"themes": {
"5f3ca599-8fee-46fb-951f-0d47b16a6d56": {
"themeColors": {
"primaryColor": "#246db5",
"secondaryColor": "#5cadff",
"backgroundColor": "#ffffff",
"textColor": "#1a1a1a",
"primaryTextColor": "#ffffff"
},
"template": "basic",
"splashScreen": {
"storedData": {
"handle": "/images/app/generic-flow.jpg",
"mimeType": "image/jpeg"
}
}
}
},
"theme": "5f3ca599-8fee-46fb-951f-0d47b16a6d56"
"title": "A2 Audio Generation",
"description": "",
"version": "0.0.1",
"nodes": [],
"edges": [],
"metadata": {
"comments": [
{
"id": "comment-cc94afe8",
"text": "Intentionally Left Blank",
"metadata": {
"title": "Comment",
"visual": {
"x": 531,
"y": 374,
"collapsed": "expanded",
"outputHeight": 0
}
}
}
],
"visual": {
"presentation": {
"themes": {
"5f3ca599-8fee-46fb-951f-0d47b16a6d56": {
"themeColors": {
"primaryColor": "#246db5",
"secondaryColor": "#5cadff",
"backgroundColor": "#ffffff",
"textColor": "#1a1a1a",
"primaryTextColor": "#ffffff"
},
"template": "basic",
"splashScreen": {
"storedData": {
"handle": "/images/app/generic-flow.jpg",
"mimeType": "image/jpeg"
}
}
}
},
"tags": [
"published",
"tool",
"component"
]
},
"modules": {
"main": {
"code": "/**\n * @fileoverview Generates audio output using supplied context.\n */\nimport gemini, { defaultSafetySettings, } from \"./a2/gemini\";\nimport { err, ok, llm, toLLMContent, toLLMContentInline, toText, } from \"./a2/utils\";\nimport {} from \"./a2/common\";\nimport { executeStep, } from \"./a2/step-executor\";\nexport { invoke as default, describe };\nasync function callAudioGen(prompt) {\n const executionInputs = {};\n const encodedPrompt = btoa(unescape(encodeURIComponent(prompt)));\n executionInputs[\"text_to_speak\"] = {\n chunks: [\n {\n mimetype: \"text/plain\",\n data: encodedPrompt,\n },\n ],\n };\n const inputParameters = [\"text_to_speak\"];\n const body = {\n planStep: {\n stepName: \"GenerateAudio\",\n modelApi: \"tts\",\n inputParameters: inputParameters,\n systemPrompt: \"\",\n },\n execution_inputs: executionInputs,\n };\n const response = await executeStep(body);\n if (!ok(response)) {\n return toLLMContent(\"TTS generation failed: \" + response.$error);\n }\n let returnVal;\n for (let value of Object.values(response.executionOutputs)) {\n const mimetype = value.chunks[0].mimetype;\n if (mimetype.startsWith(\"audio\")) {\n returnVal = toLLMContentInline(mimetype, value.chunks[0].data);\n }\n }\n if (!returnVal) {\n return toLLMContent(\"Error: No audio returned from backend\");\n }\n return returnVal;\n}\nasync function invoke({ context, }) {\n // 1) Get last LLMContent from input.\n const prompt = context && Array.isArray(context) && context.length > 0\n ? context.at(-1)\n : undefined;\n if (!prompt) {\n return err(\"Must supply context as input\");\n }\n prompt.role = \"user\";\n // 2) Call backend to generate audio.\n const content = await callAudioGen(toText(prompt));\n return { context: [content] };\n}\nasync function describe() {\n return {\n inputSchema: {\n type: \"object\",\n properties: {\n context: {\n type: \"array\",\n items: { type: \"object\", behavior: [\"llm-content\"] },\n title: \"Context in\",\n behavior: [\"main-port\"],\n },\n },\n additionalProperties: false,\n },\n outputSchema: {\n type: \"object\",\n properties: {\n context: {\n type: \"array\",\n items: { type: \"object\", behavior: [\"llm-content\"] },\n title: \"Context out\",\n behavior: [\"hint-audio\"],\n },\n },\n additionalProperties: false,\n },\n title: \"Make Audio V2 [WIP]\",\n metadata: {\n icon: \"generative-audio\",\n tags: [\"quick-access\", \"generative\", \"experimental\"],\n order: 3,\n },\n };\n}\n",
"metadata": {
"title": "main",
"source": {
"code": "/**\n * @fileoverview Generates audio output using supplied context.\n */\n\nimport gemini, {\n defaultSafetySettings,\n type GeminiOutputs,\n type GeminiInputs,\n} from \"./a2/gemini\";\nimport {\n err,\n ok,\n llm,\n toLLMContent,\n toLLMContentInline,\n toText,\n} from \"./a2/utils\";\nimport { type DescriberResult } from \"./a2/common\";\nimport {\n type ContentMap,\n type ExecuteStepRequest,\n executeStep,\n} from \"./a2/step-executor\";\n\ntype AudioGeneratorInputs = {\n context: LLMContent[];\n};\n\ntype AudioGeneratorOutputs = {\n context: LLMContent[] | DescriberResult;\n};\n\nexport { invoke as default, describe };\n\nasync function callAudioGen(prompt: string): Promise<LLMContent> {\n const executionInputs: ContentMap = {};\n const encodedPrompt = btoa(unescape(encodeURIComponent(prompt)));\n executionInputs[\"text_to_speak\"] = {\n chunks: [\n {\n mimetype: \"text/plain\",\n data: encodedPrompt,\n },\n ],\n };\n const inputParameters: string[] = [\"text_to_speak\"];\n const body = {\n planStep: {\n stepName: \"GenerateAudio\",\n modelApi: \"tts\",\n inputParameters: inputParameters,\n systemPrompt: \"\",\n },\n execution_inputs: executionInputs,\n } satisfies ExecuteStepRequest;\n const response = await executeStep(body);\n if (!ok(response)) {\n return toLLMContent(\"TTS generation failed: \" + response.$error);\n }\n\n let returnVal;\n for (let value of Object.values(response.executionOutputs)) {\n const mimetype = value.chunks[0].mimetype;\n if (mimetype.startsWith(\"audio\")) {\n returnVal = toLLMContentInline(mimetype, value.chunks[0].data);\n }\n }\n if (!returnVal) {\n return toLLMContent(\"Error: No audio returned from backend\");\n }\n return returnVal;\n}\n\nasync function invoke({\n context,\n}: AudioGeneratorInputs): Promise<Outcome<AudioGeneratorOutputs>> {\n // 1) Get last LLMContent from input.\n const prompt =\n context && Array.isArray(context) && context.length > 0\n ? context.at(-1)!\n : undefined;\n if (!prompt) {\n return err(\"Must supply context as input\");\n }\n prompt.role = \"user\";\n\n // 2) Call backend to generate audio.\n const content = await callAudioGen(toText(prompt));\n\n return { context: [content] };\n}\n\nasync function describe() {\n return {\n inputSchema: {\n type: \"object\",\n properties: {\n context: {\n type: \"array\",\n items: { type: \"object\", behavior: [\"llm-content\"] },\n title: \"Context in\",\n behavior: [\"main-port\"],\n },\n },\n additionalProperties: false,\n } satisfies Schema,\n outputSchema: {\n type: \"object\",\n properties: {\n context: {\n type: \"array\",\n items: { type: \"object\", behavior: [\"llm-content\"] },\n title: \"Context out\",\n behavior: [\"hint-audio\"],\n },\n },\n additionalProperties: false,\n } satisfies Schema,\n title: \"Make Audio V2 [WIP]\",\n metadata: {\n icon: \"generative-audio\",\n tags: [\"quick-access\", \"generative\", \"experimental\"],\n order: 3,\n },\n };\n}\n",
"language": "typescript"
},
"description": "Generates audio output using supplied context.",
"runnable": true
}
}
},
"imports": {
"a2": {
"url": "./a2.bgl.json"
}
"theme": "5f3ca599-8fee-46fb-951f-0d47b16a6d56"
}
},
"exports": [
"#module:main"
"tags": [
"published",
"tool",
"component"
]
},
"modules": {
"main": {
"code": "/**\n * @fileoverview Generates audio output using supplied context.\n */\nimport gemini, { defaultSafetySettings, } from \"./a2/gemini\";\nimport { err, ok, llm, toLLMContent, toLLMContentInline, toText, } from \"./a2/utils\";\nimport {} from \"./a2/common\";\nimport { executeStep, } from \"./a2/step-executor\";\nimport { ListExpander } from \"./a2/lists\";\nexport { invoke as default, describe };\nasync function callAudioGen(prompt) {\n const executionInputs = {};\n const encodedPrompt = btoa(unescape(encodeURIComponent(prompt)));\n executionInputs[\"text_to_speak\"] = {\n chunks: [\n {\n mimetype: \"text/plain\",\n data: encodedPrompt,\n },\n ],\n };\n const inputParameters = [\"text_to_speak\"];\n const body = {\n planStep: {\n stepName: \"GenerateAudio\",\n modelApi: \"tts\",\n inputParameters: inputParameters,\n systemPrompt: \"\",\n },\n execution_inputs: executionInputs,\n };\n const response = await executeStep(body);\n if (!ok(response)) {\n return toLLMContent(\"TTS generation failed: \" + response.$error);\n }\n let returnVal;\n for (let value of Object.values(response.executionOutputs)) {\n const mimetype = value.chunks[0].mimetype;\n if (mimetype.startsWith(\"audio\")) {\n returnVal = toLLMContentInline(mimetype, value.chunks[0].data);\n }\n }\n if (!returnVal) {\n return toLLMContent(\"Error: No audio returned from backend\");\n }\n return returnVal;\n}\nasync function invoke({ context, }) {\n const results = await new ListExpander(llm ``.asContent(), context).map(async (_i, itemContext) => {\n // 1) Get last LLMContent from input.\n const prompt = itemContext && Array.isArray(itemContext) && itemContext.length > 0\n ? context.at(-1)\n : undefined;\n if (!prompt) {\n return err(\"Must supply context as input\");\n }\n prompt.role = \"user\";\n // 2) Call backend to generate audio.\n return callAudioGen(toText(prompt));\n });\n if (!ok(results))\n return results;\n return { context: results };\n}\nasync function describe() {\n return {\n inputSchema: {\n type: \"object\",\n properties: {\n context: {\n type: \"array\",\n items: { type: \"object\", behavior: [\"llm-content\"] },\n title: \"Context in\",\n behavior: [\"main-port\"],\n },\n },\n additionalProperties: false,\n },\n outputSchema: {\n type: \"object\",\n properties: {\n context: {\n type: \"array\",\n items: { type: \"object\", behavior: [\"llm-content\"] },\n title: \"Context out\",\n behavior: [\"hint-audio\"],\n },\n },\n additionalProperties: false,\n },\n title: \"Make Audio V2 [WIP]\",\n metadata: {\n icon: \"generative-audio\",\n tags: [\"quick-access\", \"generative\", \"experimental\"],\n order: 3,\n },\n };\n}\n",
"metadata": {
"title": "main",
"source": {
"code": "/**\n * @fileoverview Generates audio output using supplied context.\n */\n\nimport gemini, {\n defaultSafetySettings,\n type GeminiOutputs,\n type GeminiInputs,\n} from \"./a2/gemini\";\nimport {\n err,\n ok,\n llm,\n toLLMContent,\n toLLMContentInline,\n toText,\n} from \"./a2/utils\";\nimport { type DescriberResult } from \"./a2/common\";\nimport {\n type ContentMap,\n type ExecuteStepRequest,\n executeStep,\n} from \"./a2/step-executor\";\nimport { ListExpander } from \"./a2/lists\";\n\ntype AudioGeneratorInputs = {\n context: LLMContent[];\n};\n\ntype AudioGeneratorOutputs = {\n context: LLMContent[] | DescriberResult;\n};\n\nexport { invoke as default, describe };\n\nasync function callAudioGen(prompt: string): Promise<LLMContent> {\n const executionInputs: ContentMap = {};\n const encodedPrompt = btoa(unescape(encodeURIComponent(prompt)));\n executionInputs[\"text_to_speak\"] = {\n chunks: [\n {\n mimetype: \"text/plain\",\n data: encodedPrompt,\n },\n ],\n };\n const inputParameters: string[] = [\"text_to_speak\"];\n const body = {\n planStep: {\n stepName: \"GenerateAudio\",\n modelApi: \"tts\",\n inputParameters: inputParameters,\n systemPrompt: \"\",\n },\n execution_inputs: executionInputs,\n } satisfies ExecuteStepRequest;\n const response = await executeStep(body);\n if (!ok(response)) {\n return toLLMContent(\"TTS generation failed: \" + response.$error);\n }\n\n let returnVal;\n for (let value of Object.values(response.executionOutputs)) {\n const mimetype = value.chunks[0].mimetype;\n if (mimetype.startsWith(\"audio\")) {\n returnVal = toLLMContentInline(mimetype, value.chunks[0].data);\n }\n }\n if (!returnVal) {\n return toLLMContent(\"Error: No audio returned from backend\");\n }\n return returnVal;\n}\n\nasync function invoke({\n context,\n}: AudioGeneratorInputs): Promise<Outcome<AudioGeneratorOutputs>> {\n const results = await new ListExpander(llm``.asContent(), context).map(\n async (_i, itemContext) => {\n // 1) Get last LLMContent from input.\n const prompt =\n itemContext && Array.isArray(itemContext) && itemContext.length > 0\n ? context.at(-1)!\n : undefined;\n if (!prompt) {\n return err(\"Must supply context as input\");\n }\n prompt.role = \"user\";\n\n // 2) Call backend to generate audio.\n return callAudioGen(toText(prompt));\n }\n );\n if (!ok(results)) return results;\n return { context: results };\n}\n\nasync function describe() {\n return {\n inputSchema: {\n type: \"object\",\n properties: {\n context: {\n type: \"array\",\n items: { type: \"object\", behavior: [\"llm-content\"] },\n title: \"Context in\",\n behavior: [\"main-port\"],\n },\n },\n additionalProperties: false,\n } satisfies Schema,\n outputSchema: {\n type: \"object\",\n properties: {\n context: {\n type: \"array\",\n items: { type: \"object\", behavior: [\"llm-content\"] },\n title: \"Context out\",\n behavior: [\"hint-audio\"],\n },\n },\n additionalProperties: false,\n } satisfies Schema,\n title: \"Make Audio V2 [WIP]\",\n metadata: {\n icon: \"generative-audio\",\n tags: [\"quick-access\", \"generative\", \"experimental\"],\n order: 3,\n },\n };\n}\n",
"language": "typescript"
},
"description": "Generates audio output using supplied context.",
"runnable": true
}
}
},
"imports": {
"a2": {
"url": "./a2.bgl.json"
}
},
"exports": [
"#module:main"
]
}
4 changes: 2 additions & 2 deletions packages/a2/bgl/listification-test.bgl.json
View file

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions packages/a2/bgl/video-generator.bgl.json
View file
Original file line number Diff line number Diff line change
@@ -51,11 +51,11 @@
},
"modules": {
"main": {
"code": "/**\n * @fileoverview Generates video output using supplied context.\n */\nimport gemini, { defaultSafetySettings, } from \"./a2/gemini\";\nimport { err, ok, llm, toTextConcat, joinContent, toLLMContent, toLLMContentInline, toText, extractInlineData, extractTextData, } from \"./a2/utils\";\nimport { Template } from \"./a2/template\";\nimport { ToolManager } from \"./a2/tool-manager\";\nimport {} from \"./a2/common\";\nimport {} from \"./a2/common\";\nimport { ArgumentNameGenerator } from \"./a2/introducer\";\nimport { executeStep, } from \"./a2/step-executor\";\nexport { invoke as default, describe };\nasync function callVideoGen(prompt) {\n const executionInputs = {};\n const encodedPrompt = btoa(unescape(encodeURIComponent(prompt)));\n executionInputs[\"video_description\"] = {\n chunks: [\n {\n mimetype: \"text/plain\",\n data: encodedPrompt,\n },\n ],\n };\n const inputParameters = [\"video_description\"];\n const body = {\n planStep: {\n stepName: \"GenerateVideo\",\n modelApi: \"generate_video\",\n inputParameters: inputParameters,\n systemPrompt: \"\",\n },\n execution_inputs: executionInputs,\n };\n // TODO(askerryryan): Remove when stable.\n console.log(\"REQUEST:\");\n console.log(body);\n const response = await executeStep(body);\n console.log(\"RESPONSE:\");\n console.log(response);\n if (!ok(response)) {\n return toLLMContent(\"Video generation failed: \" + response.$error);\n }\n let returnVal;\n for (let value of Object.values(response.executionOutputs)) {\n const mimetype = value.chunks[0].mimetype;\n if (mimetype.startsWith(\"video\")) {\n returnVal = toLLMContentInline(mimetype, value.chunks[0].data);\n }\n }\n if (!returnVal) {\n return toLLMContent(\"Error: No video returned from backend\");\n }\n return returnVal;\n}\nasync function invoke({ context, instruction, ...params }) {\n context ??= [];\n let instructionText = \"\";\n if (instruction) {\n instructionText = toText(instruction).trim();\n }\n console.log(\"context\");\n console.log(context);\n console.log(\"instruction\");\n console.log(instruction);\n // 1) Extract any image and text data from context (with history).\n let imageContext = extractInlineData(context);\n const textContext = extractTextData(context);\n // 2) Substitute variables and magic image reference.\n // Note: it is important that images are not subsituted in here as they will\n // not be handled properly. At this point, only text variables should be left.\n const toolManager = new ToolManager(new ArgumentNameGenerator());\n const substituting = await new Template(toLLMContent(instructionText)).substitute(params, async ({ path: url }) => toolManager.addTool(url));\n if (!ok(substituting)) {\n return substituting;\n }\n // 3) Extract image and text data from (non-history) references.\n const refImages = extractInlineData([substituting]);\n const refText = extractTextData([substituting]);\n // 4) Combine with whatever data was extracted from context.\n // Validate that we did not find any images, given this isn't supported yet.\n imageContext = imageContext.concat(refImages);\n if (imageContext.length != 0) {\n return {\n context: [\n toLLMContent(\"Video generation does not yet support image input. Coming soon.\"),\n ],\n };\n }\n const combinedInstruction = toTextConcat(joinContent(toTextConcat(refText), textContext, false));\n if (!combinedInstruction) {\n return {\n context: [toLLMContent(\"A video instruction must be provided.\")],\n };\n }\n console.log(\"PROMPT: \", combinedInstruction);\n // 2) Call backend to generate video.\n const content = await callVideoGen(combinedInstruction);\n return { context: [content] };\n}\nasync function describe({ inputs: { instruction } }) {\n const template = new Template(instruction);\n return {\n inputSchema: {\n type: \"object\",\n properties: {\n context: {\n type: \"array\",\n items: { type: \"object\", behavior: [\"llm-content\"] },\n title: \"Context in\",\n behavior: [\"main-port\"],\n },\n instruction: {\n type: \"object\",\n behavior: [\"llm-content\", \"config\", \"hint-preview\"],\n title: \"Instruction\",\n description: \"Instructions for how to render the video. Use @ to reference upstream steps.\",\n },\n ...template.schemas(),\n },\n behavior: [\"at-wireable\"],\n ...template.requireds(),\n },\n outputSchema: {\n type: \"object\",\n properties: {\n context: {\n type: \"array\",\n items: { type: \"object\", behavior: [\"llm-content\"] },\n title: \"Context out\",\n behavior: [\"hint-audio\"],\n },\n },\n additionalProperties: false,\n },\n title: \"Make Video [WIP]\",\n metadata: {\n icon: \"generative-audio\",\n tags: [\"quick-access\", \"generative\", \"experimental\"],\n order: 3,\n },\n };\n}\n",
"code": "/**\n * @fileoverview Generates video output using supplied context.\n */\nimport gemini, { defaultSafetySettings, } from \"./a2/gemini\";\nimport { err, ok, llm, toTextConcat, joinContent, toLLMContent, toLLMContentInline, toText, extractInlineData, extractTextData, } from \"./a2/utils\";\nimport { Template } from \"./a2/template\";\nimport { ToolManager } from \"./a2/tool-manager\";\nimport {} from \"./a2/common\";\nimport {} from \"./a2/common\";\nimport { ArgumentNameGenerator } from \"./a2/introducer\";\nimport { ListExpander } from \"./a2/lists\";\nimport { executeStep, } from \"./a2/step-executor\";\nexport { invoke as default, describe };\nasync function callVideoGen(prompt) {\n const executionInputs = {};\n const encodedPrompt = btoa(unescape(encodeURIComponent(prompt)));\n executionInputs[\"video_description\"] = {\n chunks: [\n {\n mimetype: \"text/plain\",\n data: encodedPrompt,\n },\n ],\n };\n const inputParameters = [\"video_description\"];\n const body = {\n planStep: {\n stepName: \"GenerateVideo\",\n modelApi: \"generate_video\",\n inputParameters: inputParameters,\n systemPrompt: \"\",\n },\n execution_inputs: executionInputs,\n };\n // TODO(askerryryan): Remove when stable.\n console.log(\"REQUEST:\");\n console.log(body);\n const response = await executeStep(body);\n console.log(\"RESPONSE:\");\n console.log(response);\n if (!ok(response)) {\n return toLLMContent(\"Video generation failed: \" + response.$error);\n }\n let returnVal;\n for (let value of Object.values(response.executionOutputs)) {\n const mimetype = value.chunks[0].mimetype;\n if (mimetype.startsWith(\"video\")) {\n returnVal = toLLMContentInline(mimetype, value.chunks[0].data);\n }\n }\n if (!returnVal) {\n return toLLMContent(\"Error: No video returned from backend\");\n }\n return returnVal;\n}\nasync function invoke({ context, instruction, ...params }) {\n context ??= [];\n let instructionText = \"\";\n if (instruction) {\n instructionText = toText(instruction).trim();\n }\n // 2) Substitute variables and magic image reference.\n // Note: it is important that images are not subsituted in here as they will\n // not be handled properly. At this point, only text variables should be left.\n const toolManager = new ToolManager(new ArgumentNameGenerator());\n const substituting = await new Template(toLLMContent(instructionText)).substitute(params, async ({ path: url }) => toolManager.addTool(url));\n if (!ok(substituting)) {\n return substituting;\n }\n console.log(\"context\");\n console.log(context);\n console.log(\"instruction\");\n console.log(instruction);\n const results = await new ListExpander(substituting, context).map(async (itemInstruction, itemContext) => {\n // 1) Extract any image and text data from context (with history).\n let imageContext = extractInlineData(context);\n const textContext = extractTextData(context);\n // 3) Extract image and text data from (non-history) references.\n const refImages = extractInlineData([substituting]);\n const refText = extractTextData([substituting]);\n // 4) Combine with whatever data was extracted from context.\n // Validate that we did not find any images, given this isn't supported yet.\n imageContext = imageContext.concat(refImages);\n if (imageContext.length != 0) {\n return toLLMContent(\"Video generation does not yet support image input. Coming soon.\");\n }\n const combinedInstruction = toTextConcat(joinContent(toTextConcat(refText), textContext, false));\n if (!combinedInstruction) {\n return toLLMContent(\"A video instruction must be provided.\");\n }\n console.log(\"PROMPT: \", combinedInstruction);\n // 2) Call backend to generate video.\n const content = await callVideoGen(combinedInstruction);\n return content;\n });\n if (!ok(results))\n return results;\n return { context: results };\n}\nasync function describe({ inputs: { instruction } }) {\n const template = new Template(instruction);\n return {\n inputSchema: {\n type: \"object\",\n properties: {\n context: {\n type: \"array\",\n items: { type: \"object\", behavior: [\"llm-content\"] },\n title: \"Context in\",\n behavior: [\"main-port\"],\n },\n instruction: {\n type: \"object\",\n behavior: [\"llm-content\", \"config\", \"hint-preview\"],\n title: \"Instruction\",\n description: \"Instructions for how to render the video. Use @ to reference upstream steps.\",\n },\n ...template.schemas(),\n },\n behavior: [\"at-wireable\"],\n ...template.requireds(),\n },\n outputSchema: {\n type: \"object\",\n properties: {\n context: {\n type: \"array\",\n items: { type: \"object\", behavior: [\"llm-content\"] },\n title: \"Context out\",\n behavior: [\"hint-audio\"],\n },\n },\n additionalProperties: false,\n },\n title: \"Make Video [WIP]\",\n metadata: {\n icon: \"generative-audio\",\n tags: [\"quick-access\", \"generative\", \"experimental\"],\n order: 3,\n },\n };\n}\n",
"metadata": {
"title": "main",
"source": {
"code": "/**\n * @fileoverview Generates video output using supplied context.\n */\n\nimport gemini, {\n defaultSafetySettings,\n type GeminiOutputs,\n type GeminiInputs,\n} from \"./a2/gemini\";\nimport {\n err,\n ok,\n llm,\n toTextConcat,\n joinContent,\n toLLMContent,\n toLLMContentInline,\n toText,\n extractInlineData,\n extractTextData,\n} from \"./a2/utils\";\nimport { Template } from \"./a2/template\";\nimport { ToolManager } from \"./a2/tool-manager\";\nimport { type Params } from \"./a2/common\";\nimport { type DescriberResult } from \"./a2/common\";\nimport { ArgumentNameGenerator } from \"./a2/introducer\";\n\nimport {\n type ContentMap,\n type ExecuteStepRequest,\n executeStep,\n} from \"./a2/step-executor\";\n\ntype VideoGeneratorInputs = {\n context: LLMContent[];\n instruction?: LLMContent;\n};\n\ntype VideoGeneratorOutputs = {\n context: LLMContent[] | DescriberResult;\n};\n\nexport { invoke as default, describe };\n\nasync function callVideoGen(prompt: string): Promise<LLMContent> {\n const executionInputs: ContentMap = {};\n const encodedPrompt = btoa(unescape(encodeURIComponent(prompt)));\n executionInputs[\"video_description\"] = {\n chunks: [\n {\n mimetype: \"text/plain\",\n data: encodedPrompt,\n },\n ],\n };\n const inputParameters: string[] = [\"video_description\"];\n const body = {\n planStep: {\n stepName: \"GenerateVideo\",\n modelApi: \"generate_video\",\n inputParameters: inputParameters,\n systemPrompt: \"\",\n },\n execution_inputs: executionInputs,\n } satisfies ExecuteStepRequest;\n // TODO(askerryryan): Remove when stable.\n console.log(\"REQUEST:\");\n console.log(body);\n const response = await executeStep(body);\n console.log(\"RESPONSE:\");\n console.log(response);\n if (!ok(response)) {\n return toLLMContent(\"Video generation failed: \" + response.$error);\n }\n\n let returnVal;\n for (let value of Object.values(response.executionOutputs)) {\n const mimetype = value.chunks[0].mimetype;\n if (mimetype.startsWith(\"video\")) {\n returnVal = toLLMContentInline(mimetype, value.chunks[0].data);\n }\n }\n if (!returnVal) {\n return toLLMContent(\"Error: No video returned from backend\");\n }\n return returnVal;\n}\n\nasync function invoke({\n context,\n instruction,\n ...params\n}: VideoGeneratorInputs): Promise<Outcome<VideoGeneratorOutputs>> {\n context ??= [];\n let instructionText = \"\";\n if (instruction) {\n instructionText = toText(instruction).trim();\n }\n console.log(\"context\");\n console.log(context);\n console.log(\"instruction\");\n console.log(instruction);\n\n // 1) Extract any image and text data from context (with history).\n let imageContext = extractInlineData(context);\n const textContext = extractTextData(context);\n\n // 2) Substitute variables and magic image reference.\n // Note: it is important that images are not subsituted in here as they will\n // not be handled properly. At this point, only text variables should be left.\n const toolManager = new ToolManager(new ArgumentNameGenerator());\n const substituting = await new Template(\n toLLMContent(instructionText)\n ).substitute(params, async ({ path: url }) => toolManager.addTool(url));\n if (!ok(substituting)) {\n return substituting;\n }\n\n // 3) Extract image and text data from (non-history) references.\n const refImages = extractInlineData([substituting]);\n const refText = extractTextData([substituting]);\n\n // 4) Combine with whatever data was extracted from context.\n // Validate that we did not find any images, given this isn't supported yet.\n imageContext = imageContext.concat(refImages);\n if (imageContext.length != 0) {\n return {\n context: [\n toLLMContent(\n \"Video generation does not yet support image input. Coming soon.\"\n ),\n ],\n };\n }\n const combinedInstruction = toTextConcat(\n joinContent(toTextConcat(refText), textContext, false)\n );\n if (!combinedInstruction) {\n return {\n context: [toLLMContent(\"A video instruction must be provided.\")],\n };\n }\n\n console.log(\"PROMPT: \", combinedInstruction);\n\n // 2) Call backend to generate video.\n const content = await callVideoGen(combinedInstruction);\n\n return { context: [content] };\n}\n\ntype DescribeInputs = {\n inputs: {\n instruction?: LLMContent;\n };\n};\n\nasync function describe({ inputs: { instruction } }: DescribeInputs) {\n const template = new Template(instruction);\n return {\n inputSchema: {\n type: \"object\",\n properties: {\n context: {\n type: \"array\",\n items: { type: \"object\", behavior: [\"llm-content\"] },\n title: \"Context in\",\n behavior: [\"main-port\"],\n },\n instruction: {\n type: \"object\",\n behavior: [\"llm-content\", \"config\", \"hint-preview\"],\n title: \"Instruction\",\n description:\n \"Instructions for how to render the video. Use @ to reference upstream steps.\",\n },\n ...template.schemas(),\n },\n behavior: [\"at-wireable\"],\n ...template.requireds(),\n } satisfies Schema,\n outputSchema: {\n type: \"object\",\n properties: {\n context: {\n type: \"array\",\n items: { type: \"object\", behavior: [\"llm-content\"] },\n title: \"Context out\",\n behavior: [\"hint-audio\"],\n },\n },\n additionalProperties: false,\n } satisfies Schema,\n title: \"Make Video [WIP]\",\n metadata: {\n icon: \"generative-audio\",\n tags: [\"quick-access\", \"generative\", \"experimental\"],\n order: 3,\n },\n };\n}\n",
"code": "/**\n * @fileoverview Generates video output using supplied context.\n */\n\nimport gemini, {\n defaultSafetySettings,\n type GeminiOutputs,\n type GeminiInputs,\n} from \"./a2/gemini\";\nimport {\n err,\n ok,\n llm,\n toTextConcat,\n joinContent,\n toLLMContent,\n toLLMContentInline,\n toText,\n extractInlineData,\n extractTextData,\n} from \"./a2/utils\";\nimport { Template } from \"./a2/template\";\nimport { ToolManager } from \"./a2/tool-manager\";\nimport { type Params } from \"./a2/common\";\nimport { type DescriberResult } from \"./a2/common\";\nimport { ArgumentNameGenerator } from \"./a2/introducer\";\nimport { ListExpander } from \"./a2/lists\";\n\nimport {\n type ContentMap,\n type ExecuteStepRequest,\n executeStep,\n} from \"./a2/step-executor\";\n\ntype VideoGeneratorInputs = {\n context: LLMContent[];\n instruction?: LLMContent;\n};\n\ntype VideoGeneratorOutputs = {\n context: LLMContent[] | DescriberResult;\n};\n\nexport { invoke as default, describe };\n\nasync function callVideoGen(prompt: string): Promise<LLMContent> {\n const executionInputs: ContentMap = {};\n const encodedPrompt = btoa(unescape(encodeURIComponent(prompt)));\n executionInputs[\"video_description\"] = {\n chunks: [\n {\n mimetype: \"text/plain\",\n data: encodedPrompt,\n },\n ],\n };\n const inputParameters: string[] = [\"video_description\"];\n const body = {\n planStep: {\n stepName: \"GenerateVideo\",\n modelApi: \"generate_video\",\n inputParameters: inputParameters,\n systemPrompt: \"\",\n },\n execution_inputs: executionInputs,\n } satisfies ExecuteStepRequest;\n // TODO(askerryryan): Remove when stable.\n console.log(\"REQUEST:\");\n console.log(body);\n const response = await executeStep(body);\n console.log(\"RESPONSE:\");\n console.log(response);\n if (!ok(response)) {\n return toLLMContent(\"Video generation failed: \" + response.$error);\n }\n\n let returnVal;\n for (let value of Object.values(response.executionOutputs)) {\n const mimetype = value.chunks[0].mimetype;\n if (mimetype.startsWith(\"video\")) {\n returnVal = toLLMContentInline(mimetype, value.chunks[0].data);\n }\n }\n if (!returnVal) {\n return toLLMContent(\"Error: No video returned from backend\");\n }\n return returnVal;\n}\n\nasync function invoke({\n context,\n instruction,\n ...params\n}: VideoGeneratorInputs): Promise<Outcome<VideoGeneratorOutputs>> {\n context ??= [];\n let instructionText = \"\";\n if (instruction) {\n instructionText = toText(instruction).trim();\n }\n // 2) Substitute variables and magic image reference.\n // Note: it is important that images are not subsituted in here as they will\n // not be handled properly. At this point, only text variables should be left.\n const toolManager = new ToolManager(new ArgumentNameGenerator());\n const substituting = await new Template(\n toLLMContent(instructionText)\n ).substitute(params, async ({ path: url }) => toolManager.addTool(url));\n if (!ok(substituting)) {\n return substituting;\n }\n console.log(\"context\");\n console.log(context);\n console.log(\"instruction\");\n console.log(instruction);\n\n const results = await new ListExpander(substituting, context).map(\n async (itemInstruction, itemContext) => {\n // 1) Extract any image and text data from context (with history).\n let imageContext = extractInlineData(context);\n const textContext = extractTextData(context);\n\n // 3) Extract image and text data from (non-history) references.\n const refImages = extractInlineData([substituting]);\n const refText = extractTextData([substituting]);\n\n // 4) Combine with whatever data was extracted from context.\n // Validate that we did not find any images, given this isn't supported yet.\n imageContext = imageContext.concat(refImages);\n if (imageContext.length != 0) {\n return toLLMContent(\n \"Video generation does not yet support image input. Coming soon.\"\n );\n }\n const combinedInstruction = toTextConcat(\n joinContent(toTextConcat(refText), textContext, false)\n );\n if (!combinedInstruction) {\n return toLLMContent(\"A video instruction must be provided.\");\n }\n\n console.log(\"PROMPT: \", combinedInstruction);\n\n // 2) Call backend to generate video.\n const content = await callVideoGen(combinedInstruction);\n return content;\n }\n );\n if (!ok(results)) return results;\n return { context: results };\n}\n\ntype DescribeInputs = {\n inputs: {\n instruction?: LLMContent;\n };\n};\n\nasync function describe({ inputs: { instruction } }: DescribeInputs) {\n const template = new Template(instruction);\n return {\n inputSchema: {\n type: \"object\",\n properties: {\n context: {\n type: \"array\",\n items: { type: \"object\", behavior: [\"llm-content\"] },\n title: \"Context in\",\n behavior: [\"main-port\"],\n },\n instruction: {\n type: \"object\",\n behavior: [\"llm-content\", \"config\", \"hint-preview\"],\n title: \"Instruction\",\n description:\n \"Instructions for how to render the video. Use @ to reference upstream steps.\",\n },\n ...template.schemas(),\n },\n behavior: [\"at-wireable\"],\n ...template.requireds(),\n } satisfies Schema,\n outputSchema: {\n type: \"object\",\n properties: {\n context: {\n type: \"array\",\n items: { type: \"object\", behavior: [\"llm-content\"] },\n title: \"Context out\",\n behavior: [\"hint-audio\"],\n },\n },\n additionalProperties: false,\n } satisfies Schema,\n title: \"Make Video [WIP]\",\n metadata: {\n icon: \"generative-audio\",\n tags: [\"quick-access\", \"generative\", \"experimental\"],\n order: 3,\n },\n };\n}\n",
"language": "typescript"
},
"description": "Generates video output using supplied context.",