Register dynamic agent to try out discovered schema (#675)

ts/packages/agents/browser/src/agent/commerce/actionHandler.mts

@@ -9,6 +9,8 @@ import {
   ProductDetailsHeroTile,
   ProductTile,
   SearchInput,
+  ShoppingCartButton,
+  ShoppingCartDetails,
   StoreLocation,
 } from "./schema/pageComponents.mjs";
 import { ShoppingActions } from "./schema/userActions.mjs";
@@ -46,6 +48,9 @@ export async function handleCommerceAction(
     case "findNearbyStoreAction":
       await handleFindNearbyStore(action);
       break;
+    case "viewShoppingCartAction":
+      await handleViewShoppingCart(action);
+      break;
   }
 
   async function getComponentFromPage(
@@ -64,7 +69,7 @@ export async function handleCommerceAction(
     );
 
     if (!response.success) {
-      console.error("Attempt to get product tilefailed");
+      console.error(`Attempt to get ${componentType} failed`);
       console.error(response.message);
       return;
     }
@@ -73,6 +78,14 @@ export async function handleCommerceAction(
     return response.data;
   }
 
+  async function followLink(linkSelector: string | undefined) {
+    if (!linkSelector) return;
+
+    await browser.clickOn(linkSelector);
+    await browser.awaitPageInteraction();
+    await browser.awaitPageLoad();
+  }
+
   async function searchForProduct(productName: string) {
     const selector = (await getComponentFromPage("SearchInput")) as SearchInput;
     const searchSelector = selector.cssSelector;
@@ -138,5 +151,19 @@ export async function handleCommerceAction(
     }
   }
 
+  async function handleViewShoppingCart(action: any) {
+    const cartButton = (await getComponentFromPage(
+      "ShoppingCartButton",
+    )) as ShoppingCartButton;
+    console.log(cartButton);
+
+    await followLink(cartButton?.detailsLinkCssSelector);
+
+    const cartDetails = (await getComponentFromPage(
+      "ShoppingCartDetails",
+    )) as ShoppingCartDetails;
+    console.log(cartDetails);
+  }
+
   return message;
 }

ts/packages/agents/browser/src/agent/commerce/schema/pageComponents.mts

@@ -53,3 +53,19 @@ export type LocationInStore = {
   physicalLocationInStore: string;
   numberInStock?: string;
 };
+
+// The shopping cart button on the page
+export type ShoppingCartButton = {
+  label: string;
+  detailsLinkCssSelector: string;
+};
+
+export type ShoppingCartDetails = {
+  storeName: string;
+  deliveryInformation: string;
+  totalAmount: string;
+
+  productsInCart?: ProductTile[];
+
+  relatedProducts?: ProductTile[];
+};

ts/packages/agents/browser/src/agent/commerce/schema/userActions.mts

@@ -8,6 +8,11 @@ export type AddToCartAction = {
   };
 };
 
+// This allows you to view the shopping cart contents
+export type ViewShoppingCartAction = {
+  actionName: "viewShoppingCartAction";
+};
+
 export type FindNearbyStoreAction = {
   actionName: "findNearbyStoreAction";
 };
@@ -38,6 +43,7 @@ export type SelectSearchResult = {
 
 export type ShoppingActions =
   | AddToCartAction
+  | ViewShoppingCartAction
   | FindNearbyStoreAction
   | GetLocationInStore
   | SearchForProductAction

ts/packages/agents/browser/src/agent/discovery/actionHandler.mts

@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-import { ActionContext } from "@typeagent/agent-sdk";
+import { ActionContext, AppAgentManifest } from "@typeagent/agent-sdk";
 import { BrowserActionContext } from "../actionHandler.mjs";
 import { BrowserConnector } from "../browserConnector.mjs";
 import { createDiscoveryPageTranslator } from "./translator.mjs";
@@ -15,6 +15,8 @@ import path from "path";
 import fs from "fs";
 import { fileURLToPath } from "url";
 import { UserActionsList } from "./schema/userActionsPool.mjs";
+import { PageDescription } from "./schema/pageSummary.mjs";
+import { createTempAgentForSchema } from "./tempAgentActionHandler.mjs";
 import { SchemaDiscoveryActions } from "./schema/discoveryActions.mjs";
 
 export async function handleSchemaDiscoveryAction(
@@ -58,9 +60,13 @@ export async function handleSchemaDiscoveryAction(
       screenshot,
     );
 
+    let schemaDescription =
+      "A schema that enables interactions with the current page";
     if (summaryResponse.success) {
       pageSummary =
         "Page summary: \n" + JSON.stringify(summaryResponse.data, null, 2);
+      schemaDescription += (summaryResponse.data as PageDescription)
+        .description;
     }
 
     const timerName = `Analyzing page actions`;
@@ -76,6 +82,7 @@ export async function handleSchemaDiscoveryAction(
     if (!response.success) {
       console.error("Attempt to get page actions failed");
       console.error(response.message);
+      message = "Action could not be completed";
       return;
     }
 
@@ -92,6 +99,27 @@ export async function handleSchemaDiscoveryAction(
     const schema = await getDynamicSchema(actionNames);
     message += `\n =========== \n Discovered actions schema: \n ${schema} `;
 
+    if (action.parameters.registerAgent) {
+      const manifest: AppAgentManifest = {
+        emojiChar: "🚧",
+        description: schemaDescription,
+        schema: {
+          description: schemaDescription,
+          schemaType: "DynamicUserPageActions",
+          schemaFile: { content: schema, type: "ts" },
+        },
+      };
+
+      // register agent after request is processed to avoid a deadlock
+      setTimeout(async () => {
+        await context.sessionContext.addDynamicAgent(
+          "tempPageSchema",
+          manifest,
+          createTempAgentForSchema(browser, agent, context),
+        );
+      }, 500);
+    }
+
     return response.data;
   }
 
@@ -128,9 +156,13 @@ export async function handleSchemaDiscoveryAction(
       typeDefinitions.map((definition) => sc.ref(definition)),
     );
     const entry = sc.type("DynamicUserPageActions", union);
+    entry.exported = true;
     const actionSchemas = new Map<string, ActionSchemaTypeDefinition>();
     const order = new Map<string, number>();
-    const schema = await generateActionSchema({ entry, actionSchemas, order });
+    const schema = await generateActionSchema(
+      { entry, actionSchemas, order },
+      { exact: true },
+    );
 
     return schema;
   }
@@ -144,6 +176,7 @@ export async function handleSchemaDiscoveryAction(
     if (!response.success) {
       console.error("Attempt to get page summary failed");
       console.error(response.message);
+      message = "Action could not be completed";
       return;
     }
 
@@ -161,6 +194,7 @@ export async function handleSchemaDiscoveryAction(
     if (!response.success) {
       console.error("Attempt to get page layout failed");
       console.error(response.message);
+      message = "Action could not be completed";
       return;
     }
 
@@ -184,6 +218,7 @@ export async function handleSchemaDiscoveryAction(
     if (!response.success) {
       console.error("Attempt to get page layout failed");
       console.error(response.message);
+      message = "Action could not be completed";
       return;
     }
 

ts/packages/agents/browser/src/agent/discovery/schema/discoveryActions.mts

@@ -7,6 +7,10 @@ export type FindPageComponents = {
 
 export type FindUserActions = {
   actionName: "findUserActions";
+  parameters: {
+    registerAgent: boolean;
+    agentName?: string;
+  };
 };
 
 export type GetPageType = {

ts/packages/agents/browser/src/agent/discovery/schema/pageComponents.mts

@@ -53,3 +53,9 @@ export type LocationInStore = {
   physicalLocationInStore: string;
   numberInStock?: string;
 };
+
+export type NavigationLink = {
+  // CSS Selector for the link
+  title: string;
+  linkCssSelector: string;
+};

ts/packages/agents/browser/src/agent/discovery/schema/userActionsPool.mts

@@ -2,117 +2,104 @@
 // Licensed under the MIT License.
 
 export type AddToCartAction = {
-  actionName: "AddToCartAction";
+  actionName: "addToCartAction";
   parameters: {
     productName: string;
   };
 };
 
+export type RemoveFromCartAction = {
+  actionName: "removeFromCartAction";
+  parameters: {
+    productName: string;
+  };
+};
+
+// This allows you to view the shopping cart contents
+export type ViewShoppingCartAction = {
+  actionName: "viewShoppingCartAction";
+};
+
 export type FindNearbyStoreAction = {
-  actionName: "FindNearbyStoreAction";
+  actionName: "findNearbyStoreAction";
 };
 
 // Use this action for user queries such as "where is product X in the store"
 export type GetLocationInStore = {
-  actionName: "GetLocationInStore";
+  actionName: "getLocationInStore";
   parameters: {
     productName: string;
   };
 };
 
 // IMPORTANT: Use this action when the user query involves search for products on an e-commerce store, such as "aaa batteries"
 export type SearchForProductAction = {
-  actionName: "SearchForProductAction";
+  actionName: "searchForProductAction";
   parameters: {
     productName: string;
+    selectionCriteria?: string;
   };
 };
 
 // This allows users to select individual results on the search results page.
 export type SelectSearchResult = {
-  actionName: "SelectSearchResult";
+  actionName: "selectSearchResult";
   parameters: {
     position: number;
     productName?: string;
   };
 };
 
-export type NavigateToHomePage = {
-  actionName: "NavigateToHomePage";
-  parameters: {
-    linkCssSelector: string;
-  };
-};
-
-// Follow a link to view  a store landing page
-export type NavigateToStorePage = {
-  actionName: "NavigateToStorePage";
-  parameters: {
-    linkCssSelector: string;
-  };
-};
-
-// Follow a link to view  a product details page
-export type NavigateToProductPage = {
-  actionName: "NavigateToProductPage";
+export type NavigateToPage = {
+  actionName: "navigateToPage";
   parameters: {
-    linkCssSelector: string;
+    keywords: string;
   };
 };
 
-// Follow a link to view  a recipe details page. This link is typically named "Recipe" or "Recipes"
-export type NavigateToRecipePage = {
-  actionName: "NavigateToRecipePage";
+export type BrowseProductCategoriesAction = {
+  actionName: "browseProductCategoriesAction";
   parameters: {
-    linkCssSelector: string;
+    categoryName?: string;
   };
 };
 
-export type NavigateToListPage = {
-  actionName: "NavigateToListPage";
+// This allows users to filter products based on a criteria such as price, size, shipping options etc.
+export type FilterProductsAction = {
+  actionName: "filterProductsAction";
   parameters: {
-    linkCssSelector: string;
+    filterCriteria: string;
   };
 };
 
-// Navigate to the "Buy it again" page. This page may also be called Past Orders.
-export type NavigateToBuyItAgainPage = {
-  actionName: "NavigateToBuyItAgainPage";
+export type SignUpForNewsletterAction = {
+  actionName: "signUpForNewsletterAction";
   parameters: {
-    linkCssSelector: string;
+    emailAddress: string;
   };
 };
 
-// This link opens the shopping cart. Its usually indicated by a cart or bag icon.
-export type NavigateToShoppingCartPage = {
-  actionName: "NavigateToShoppingCartPage";
-  parameters: {
-    linkCssSelector: string;
-  };
-};
-
-export type NavigateToOtherPage = {
-  actionName: "NavigateToOtherPage";
+// Follow a link to view  a product details page
+export type NavigateToProductPage = {
+  actionName: "navigateToProductPage";
   parameters: {
-    pageType: string;
-    linkCssSelector: string;
+    productName: string;
   };
 };
 
 export type UserPageActions =
   | AddToCartAction
+  | BrowseProductCategoriesAction
+  | FilterProductsAction
   | FindNearbyStoreAction
   | GetLocationInStore
+  | NavigateToPage
+  | NavigateToProductPage
+  | RemoveFromCartAction
   | SearchForProductAction
   | SelectSearchResult
-  | NavigateToBuyItAgainPage
-  | NavigateToHomePage
-  | NavigateToListPage
-  | NavigateToOtherPage
-  | NavigateToProductPage
-  | NavigateToRecipePage
-  | NavigateToShoppingCartPage
-  | NavigateToStorePage;
+  | SignUpForNewsletterAction
+  | ViewShoppingCartAction;
 
 export type UserActionsList = {
   actions: UserPageActions[];