From 135afa0be4bb178781094be074e1d387c774e806 Mon Sep 17 00:00:00 2001 From: Hillary Mutisya <150286414+hillary-mutisya@users.noreply.github.com> Date: Thu, 6 Feb 2025 14:10:35 -0800 Subject: [PATCH] Register dynamic agent to try out discovered schema (#675) --- .../src/agent/commerce/actionHandler.mts | 29 ++++- .../agent/commerce/schema/pageComponents.mts | 16 +++ .../src/agent/commerce/schema/userActions.mts | 6 + .../src/agent/discovery/actionHandler.mts | 39 ++++++- .../discovery/schema/discoveryActions.mts | 4 + .../agent/discovery/schema/pageComponents.mts | 6 + .../discovery/schema/userActionsPool.mts | 97 +++++++--------- .../discovery/tempAgentActionHandler.mts | 104 ++++++++++++++++++ .../src/agent/discovery/translator.mts | 19 +++- .../src/agent/instacart/actionHandler.mts | 5 - 10 files changed, 261 insertions(+), 64 deletions(-) create mode 100644 ts/packages/agents/browser/src/agent/discovery/tempAgentActionHandler.mts diff --git a/ts/packages/agents/browser/src/agent/commerce/actionHandler.mts b/ts/packages/agents/browser/src/agent/commerce/actionHandler.mts index 680e7ec89..b9f9db8aa 100644 --- a/ts/packages/agents/browser/src/agent/commerce/actionHandler.mts +++ b/ts/packages/agents/browser/src/agent/commerce/actionHandler.mts @@ -9,6 +9,8 @@ import { ProductDetailsHeroTile, ProductTile, SearchInput, + ShoppingCartButton, + ShoppingCartDetails, StoreLocation, } from "./schema/pageComponents.mjs"; import { ShoppingActions } from "./schema/userActions.mjs"; @@ -46,6 +48,9 @@ export async function handleCommerceAction( case "findNearbyStoreAction": await handleFindNearbyStore(action); break; + case "viewShoppingCartAction": + await handleViewShoppingCart(action); + break; } async function getComponentFromPage( @@ -64,7 +69,7 @@ export async function handleCommerceAction( ); if (!response.success) { - console.error("Attempt to get product tilefailed"); + console.error(`Attempt to get ${componentType} failed`); console.error(response.message); return; } @@ -73,6 +78,14 @@ export async function handleCommerceAction( return response.data; } + async function followLink(linkSelector: string | undefined) { + if (!linkSelector) return; + + await browser.clickOn(linkSelector); + await browser.awaitPageInteraction(); + await browser.awaitPageLoad(); + } + async function searchForProduct(productName: string) { const selector = (await getComponentFromPage("SearchInput")) as SearchInput; const searchSelector = selector.cssSelector; @@ -138,5 +151,19 @@ export async function handleCommerceAction( } } + async function handleViewShoppingCart(action: any) { + const cartButton = (await getComponentFromPage( + "ShoppingCartButton", + )) as ShoppingCartButton; + console.log(cartButton); + + await followLink(cartButton?.detailsLinkCssSelector); + + const cartDetails = (await getComponentFromPage( + "ShoppingCartDetails", + )) as ShoppingCartDetails; + console.log(cartDetails); + } + return message; } diff --git a/ts/packages/agents/browser/src/agent/commerce/schema/pageComponents.mts b/ts/packages/agents/browser/src/agent/commerce/schema/pageComponents.mts index fe5f72efc..78a442763 100644 --- a/ts/packages/agents/browser/src/agent/commerce/schema/pageComponents.mts +++ b/ts/packages/agents/browser/src/agent/commerce/schema/pageComponents.mts @@ -53,3 +53,19 @@ export type LocationInStore = { physicalLocationInStore: string; numberInStock?: string; }; + +// The shopping cart button on the page +export type ShoppingCartButton = { + label: string; + detailsLinkCssSelector: string; +}; + +export type ShoppingCartDetails = { + storeName: string; + deliveryInformation: string; + totalAmount: string; + + productsInCart?: ProductTile[]; + + relatedProducts?: ProductTile[]; +}; diff --git a/ts/packages/agents/browser/src/agent/commerce/schema/userActions.mts b/ts/packages/agents/browser/src/agent/commerce/schema/userActions.mts index 2c5849717..8fa18e22b 100644 --- a/ts/packages/agents/browser/src/agent/commerce/schema/userActions.mts +++ b/ts/packages/agents/browser/src/agent/commerce/schema/userActions.mts @@ -8,6 +8,11 @@ export type AddToCartAction = { }; }; +// This allows you to view the shopping cart contents +export type ViewShoppingCartAction = { + actionName: "viewShoppingCartAction"; +}; + export type FindNearbyStoreAction = { actionName: "findNearbyStoreAction"; }; @@ -38,6 +43,7 @@ export type SelectSearchResult = { export type ShoppingActions = | AddToCartAction + | ViewShoppingCartAction | FindNearbyStoreAction | GetLocationInStore | SearchForProductAction diff --git a/ts/packages/agents/browser/src/agent/discovery/actionHandler.mts b/ts/packages/agents/browser/src/agent/discovery/actionHandler.mts index dcc0ef582..23c9094f7 100644 --- a/ts/packages/agents/browser/src/agent/discovery/actionHandler.mts +++ b/ts/packages/agents/browser/src/agent/discovery/actionHandler.mts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { ActionContext } from "@typeagent/agent-sdk"; +import { ActionContext, AppAgentManifest } from "@typeagent/agent-sdk"; import { BrowserActionContext } from "../actionHandler.mjs"; import { BrowserConnector } from "../browserConnector.mjs"; import { createDiscoveryPageTranslator } from "./translator.mjs"; @@ -15,6 +15,8 @@ import path from "path"; import fs from "fs"; import { fileURLToPath } from "url"; import { UserActionsList } from "./schema/userActionsPool.mjs"; +import { PageDescription } from "./schema/pageSummary.mjs"; +import { createTempAgentForSchema } from "./tempAgentActionHandler.mjs"; import { SchemaDiscoveryActions } from "./schema/discoveryActions.mjs"; export async function handleSchemaDiscoveryAction( @@ -58,9 +60,13 @@ export async function handleSchemaDiscoveryAction( screenshot, ); + let schemaDescription = + "A schema that enables interactions with the current page"; if (summaryResponse.success) { pageSummary = "Page summary: \n" + JSON.stringify(summaryResponse.data, null, 2); + schemaDescription += (summaryResponse.data as PageDescription) + .description; } const timerName = `Analyzing page actions`; @@ -76,6 +82,7 @@ export async function handleSchemaDiscoveryAction( if (!response.success) { console.error("Attempt to get page actions failed"); console.error(response.message); + message = "Action could not be completed"; return; } @@ -92,6 +99,27 @@ export async function handleSchemaDiscoveryAction( const schema = await getDynamicSchema(actionNames); message += `\n =========== \n Discovered actions schema: \n ${schema} `; + if (action.parameters.registerAgent) { + const manifest: AppAgentManifest = { + emojiChar: "🚧", + description: schemaDescription, + schema: { + description: schemaDescription, + schemaType: "DynamicUserPageActions", + schemaFile: { content: schema, type: "ts" }, + }, + }; + + // register agent after request is processed to avoid a deadlock + setTimeout(async () => { + await context.sessionContext.addDynamicAgent( + "tempPageSchema", + manifest, + createTempAgentForSchema(browser, agent, context), + ); + }, 500); + } + return response.data; } @@ -128,9 +156,13 @@ export async function handleSchemaDiscoveryAction( typeDefinitions.map((definition) => sc.ref(definition)), ); const entry = sc.type("DynamicUserPageActions", union); + entry.exported = true; const actionSchemas = new Map(); const order = new Map(); - const schema = await generateActionSchema({ entry, actionSchemas, order }); + const schema = await generateActionSchema( + { entry, actionSchemas, order }, + { exact: true }, + ); return schema; } @@ -144,6 +176,7 @@ export async function handleSchemaDiscoveryAction( if (!response.success) { console.error("Attempt to get page summary failed"); console.error(response.message); + message = "Action could not be completed"; return; } @@ -161,6 +194,7 @@ export async function handleSchemaDiscoveryAction( if (!response.success) { console.error("Attempt to get page layout failed"); console.error(response.message); + message = "Action could not be completed"; return; } @@ -184,6 +218,7 @@ export async function handleSchemaDiscoveryAction( if (!response.success) { console.error("Attempt to get page layout failed"); console.error(response.message); + message = "Action could not be completed"; return; } diff --git a/ts/packages/agents/browser/src/agent/discovery/schema/discoveryActions.mts b/ts/packages/agents/browser/src/agent/discovery/schema/discoveryActions.mts index d1025078d..0f2d2f5ec 100644 --- a/ts/packages/agents/browser/src/agent/discovery/schema/discoveryActions.mts +++ b/ts/packages/agents/browser/src/agent/discovery/schema/discoveryActions.mts @@ -7,6 +7,10 @@ export type FindPageComponents = { export type FindUserActions = { actionName: "findUserActions"; + parameters: { + registerAgent: boolean; + agentName?: string; + }; }; export type GetPageType = { diff --git a/ts/packages/agents/browser/src/agent/discovery/schema/pageComponents.mts b/ts/packages/agents/browser/src/agent/discovery/schema/pageComponents.mts index 7a62e47e8..ce1e2c08f 100644 --- a/ts/packages/agents/browser/src/agent/discovery/schema/pageComponents.mts +++ b/ts/packages/agents/browser/src/agent/discovery/schema/pageComponents.mts @@ -53,3 +53,9 @@ export type LocationInStore = { physicalLocationInStore: string; numberInStock?: string; }; + +export type NavigationLink = { + // CSS Selector for the link + title: string; + linkCssSelector: string; +}; diff --git a/ts/packages/agents/browser/src/agent/discovery/schema/userActionsPool.mts b/ts/packages/agents/browser/src/agent/discovery/schema/userActionsPool.mts index 501284083..8b707dd79 100644 --- a/ts/packages/agents/browser/src/agent/discovery/schema/userActionsPool.mts +++ b/ts/packages/agents/browser/src/agent/discovery/schema/userActionsPool.mts @@ -2,19 +2,31 @@ // Licensed under the MIT License. export type AddToCartAction = { - actionName: "AddToCartAction"; + actionName: "addToCartAction"; parameters: { productName: string; }; }; +export type RemoveFromCartAction = { + actionName: "removeFromCartAction"; + parameters: { + productName: string; + }; +}; + +// This allows you to view the shopping cart contents +export type ViewShoppingCartAction = { + actionName: "viewShoppingCartAction"; +}; + export type FindNearbyStoreAction = { - actionName: "FindNearbyStoreAction"; + actionName: "findNearbyStoreAction"; }; // Use this action for user queries such as "where is product X in the store" export type GetLocationInStore = { - actionName: "GetLocationInStore"; + actionName: "getLocationInStore"; parameters: { productName: string; }; @@ -22,97 +34,72 @@ export type GetLocationInStore = { // IMPORTANT: Use this action when the user query involves search for products on an e-commerce store, such as "aaa batteries" export type SearchForProductAction = { - actionName: "SearchForProductAction"; + actionName: "searchForProductAction"; parameters: { productName: string; + selectionCriteria?: string; }; }; // This allows users to select individual results on the search results page. export type SelectSearchResult = { - actionName: "SelectSearchResult"; + actionName: "selectSearchResult"; parameters: { position: number; productName?: string; }; }; -export type NavigateToHomePage = { - actionName: "NavigateToHomePage"; - parameters: { - linkCssSelector: string; - }; -}; - -// Follow a link to view a store landing page -export type NavigateToStorePage = { - actionName: "NavigateToStorePage"; - parameters: { - linkCssSelector: string; - }; -}; - -// Follow a link to view a product details page -export type NavigateToProductPage = { - actionName: "NavigateToProductPage"; +export type NavigateToPage = { + actionName: "navigateToPage"; parameters: { - linkCssSelector: string; + keywords: string; }; }; -// Follow a link to view a recipe details page. This link is typically named "Recipe" or "Recipes" -export type NavigateToRecipePage = { - actionName: "NavigateToRecipePage"; +export type BrowseProductCategoriesAction = { + actionName: "browseProductCategoriesAction"; parameters: { - linkCssSelector: string; + categoryName?: string; }; }; -export type NavigateToListPage = { - actionName: "NavigateToListPage"; +// This allows users to filter products based on a criteria such as price, size, shipping options etc. +export type FilterProductsAction = { + actionName: "filterProductsAction"; parameters: { - linkCssSelector: string; + filterCriteria: string; }; }; -// Navigate to the "Buy it again" page. This page may also be called Past Orders. -export type NavigateToBuyItAgainPage = { - actionName: "NavigateToBuyItAgainPage"; +export type SignUpForNewsletterAction = { + actionName: "signUpForNewsletterAction"; parameters: { - linkCssSelector: string; + emailAddress: string; }; }; -// This link opens the shopping cart. Its usually indicated by a cart or bag icon. -export type NavigateToShoppingCartPage = { - actionName: "NavigateToShoppingCartPage"; - parameters: { - linkCssSelector: string; - }; -}; - -export type NavigateToOtherPage = { - actionName: "NavigateToOtherPage"; +// Follow a link to view a product details page +export type NavigateToProductPage = { + actionName: "navigateToProductPage"; parameters: { - pageType: string; - linkCssSelector: string; + productName: string; }; }; export type UserPageActions = | AddToCartAction + | BrowseProductCategoriesAction + | FilterProductsAction | FindNearbyStoreAction | GetLocationInStore + | NavigateToPage + | NavigateToProductPage + | RemoveFromCartAction | SearchForProductAction | SelectSearchResult - | NavigateToBuyItAgainPage - | NavigateToHomePage - | NavigateToListPage - | NavigateToOtherPage - | NavigateToProductPage - | NavigateToRecipePage - | NavigateToShoppingCartPage - | NavigateToStorePage; + | SignUpForNewsletterAction + | ViewShoppingCartAction; export type UserActionsList = { actions: UserPageActions[]; diff --git a/ts/packages/agents/browser/src/agent/discovery/tempAgentActionHandler.mts b/ts/packages/agents/browser/src/agent/discovery/tempAgentActionHandler.mts new file mode 100644 index 000000000..11429a793 --- /dev/null +++ b/ts/packages/agents/browser/src/agent/discovery/tempAgentActionHandler.mts @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { AppAgent } from "@typeagent/agent-sdk"; +import { BrowserConnector } from "../browserConnector.mjs"; +import { + BrowseProductCategoriesAction, + NavigateToPage, +} from "./schema/userActionsPool.mjs"; +import { handleCommerceAction } from "../commerce/actionHandler.mjs"; +import { NavigationLink } from "./schema/pageComponents.mjs"; + +export function createTempAgentForSchema( + browser: BrowserConnector, + agent: any, + context: any, +): AppAgent { + return { + async executeAction(action: any, tempContext: any): Promise { + console.log(`Executing action: ${action.actionName}`); + switch (action.actionName) { + case "addToCartAction": + case "viewShoppingCartAction": + case "findNearbyStoreAction": + case "getLocationInStore": + case "searchForProductAction": + case "selectSearchResult": + handleCommerceAction(action, context); + case "browseProductCategoriesAction": + handleBrowseProductCategory(action); + break; + case "filterProductsAction": + break; + case "navigateToPage": + handleNavigateToPage(action); + break; + case "navigateToProductPage": + break; + case "removeFromCartAction": + break; + case "signUpForNewsletterAction": + break; + } + }, + }; + + async function getComponentFromPage( + componentType: string, + selectionCondition?: string, + ) { + const htmlFragments = await browser.getHtmlFragments(); + const timerName = `getting ${componentType} section`; + + console.time(timerName); + const response = await agent.getPageComponentSchema( + componentType, + selectionCondition, + htmlFragments, + undefined, + ); + + if (!response.success) { + console.error(`Attempt to get ${componentType} failed`); + console.error(response.message); + return; + } + + console.timeEnd(timerName); + return response.data; + } + + async function followLink(linkSelector: string | undefined) { + if (!linkSelector) return; + + await browser.clickOn(linkSelector); + await browser.awaitPageInteraction(); + await browser.awaitPageLoad(); + } + + async function handleNavigateToPage(action: NavigateToPage) { + const link = (await getComponentFromPage( + "NavigationLink", + `link text ${action.parameters.keywords}`, + )) as NavigationLink; + console.log(link); + + await followLink(link?.linkCssSelector); + } + + async function handleBrowseProductCategory( + action: BrowseProductCategoriesAction, + ) { + let linkText = action.parameters.categoryName + ? `link text ${action.parameters.categoryName}` + : ""; + const link = (await getComponentFromPage( + "NavigationLink", + linkText, + )) as NavigationLink; + console.log(link); + + await followLink(link?.linkCssSelector); + } +} diff --git a/ts/packages/agents/browser/src/agent/discovery/translator.mts b/ts/packages/agents/browser/src/agent/discovery/translator.mts index 22b7f0509..dcafa40a8 100644 --- a/ts/packages/agents/browser/src/agent/discovery/translator.mts +++ b/ts/packages/agents/browser/src/agent/discovery/translator.mts @@ -231,7 +231,24 @@ export class SchemaDiscoveryAgent { fragments?: HtmlFragments[], screenshot?: string, ) { - const bootstrapTranslator = this.getBootstrapTranslator(componentTypeName); + const packageRoot = path.join("..", "..", ".."); + const componentsSchema = await fs.promises.readFile( + fileURLToPath( + new URL( + path.join( + packageRoot, + "./src/agent/discovery/schema/pageComponents.mts", + ), + import.meta.url, + ), + ), + "utf8", + ); + + const bootstrapTranslator = this.getBootstrapTranslator( + componentTypeName, + componentsSchema, + ); const promptSections = this.getCssSelectorForElementPrompt( bootstrapTranslator, diff --git a/ts/packages/agents/browser/src/agent/instacart/actionHandler.mts b/ts/packages/agents/browser/src/agent/instacart/actionHandler.mts index fd9aee431..0f5428c88 100644 --- a/ts/packages/agents/browser/src/agent/instacart/actionHandler.mts +++ b/ts/packages/agents/browser/src/agent/instacart/actionHandler.mts @@ -35,17 +35,12 @@ export async function handleInstacartAction( case "addToCartAction": await handleAddToCart(action); break; - case "getShoppingCartAction": - await handleGetCart(action); - break; - break; case "getShoppingCartAction": await handleGetCart(action); break; case "addToListAction": await handleAddToList(action); break; - break; case "findNearbyStoreAction": await handleFindStores(action); break;