From aa283a32da50b97455788376d7a9691365d0a788 Mon Sep 17 00:00:00 2001 From: Victor Dibia Date: Sun, 9 Feb 2025 18:22:51 -0800 Subject: [PATCH 01/10] improve docs, add example of custom model clients with custom agents and serialization #5450 --- .../user-guide/agentchat-user-guide/index.md | 4 +- .../tutorial/custom-agents.ipynb | 389 +++++++++++++++++- 2 files changed, 391 insertions(+), 2 deletions(-) diff --git a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/index.md b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/index.md index e83288338dc7..aa3aee477d8b 100644 --- a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/index.md +++ b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/index.md @@ -101,7 +101,7 @@ tutorial/agents tutorial/teams tutorial/human-in-the-loop tutorial/termination -tutorial/custom-agents + tutorial/state ``` @@ -111,11 +111,13 @@ tutorial/state :hidden: :caption: Advanced +tutorial/custom-agents selector-group-chat swarm magentic-one memory serialize-components + ``` ```{toctree} diff --git a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/custom-agents.ipynb b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/custom-agents.ipynb index 5b8c4e7f24fa..76319974ca98 100644 --- a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/custom-agents.ipynb +++ b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/custom-agents.ipynb @@ -287,6 +287,393 @@ "From the output, we can see that the agents have successfully transformed the input integer\n", "from 10 to 25 by choosing appropriate agents that apply the arithmetic operations in sequence." ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using Custom Model Clients in Custom Agents\n", + "\n", + "One of the key features of the AssistantAgent preset in AgentChat is that it takes a `model_client` argument and can use it in responding to messages. However, in some cases, you may want your agent to use a custom model client that is not currently supported (see [supported model clients](https://microsoft.github.io/autogen/dev/user-guide/core-user-guide/components/model-clients.html)). \n", + "\n", + "You can accomplish this with a custom agent that implements your custom model client.\n", + "In the example below, we will walk through an example of a custom agent that uses the Google Gemini SDK directly to respond to messages.\n", + "\n", + "> **Note:** You will need to install the Google Gemini SDK to run this example. You can install it using the following command: \n", + "\n", + "```bash\n", + "pip install google-genai\n", + "``` " + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "metadata": {}, + "outputs": [], + "source": [ + "# !pip install google-genai\n", + "import os\n", + "from typing import AsyncGenerator, Sequence\n", + "\n", + "from autogen_agentchat.agents import BaseChatAgent\n", + "from autogen_agentchat.base import Response\n", + "from autogen_agentchat.messages import AgentEvent, ChatMessage\n", + "from autogen_core import CancellationToken\n", + "from autogen_core.model_context import UnboundedChatCompletionContext\n", + "from autogen_core.models import AssistantMessage, RequestUsage, UserMessage\n", + "from google import genai\n", + "from google.genai import types\n", + "\n", + "\n", + "class GeminiAssistantAgent(BaseChatAgent):\n", + " def __init__(\n", + " self,\n", + " name: str,\n", + " description: str = \"An agent that provides assistance with ability to use tools.\",\n", + " model: str = \"gemini-1.5-flash-002\",\n", + " api_key: str = os.environ[\"GEMINI_API_KEY\"],\n", + " system_message: str = \"You are a helpful assistant that can respond to messages. Reply with TERMINATE when the task has been completed.\",\n", + " ):\n", + " super().__init__(name=name, description=description)\n", + " self._model_context = UnboundedChatCompletionContext()\n", + " self._model_client = genai.Client(api_key=api_key)\n", + " self._system_message = system_message\n", + " self._model = model\n", + "\n", + " @property\n", + " def produced_message_types(self) -> Sequence[type[ChatMessage]]:\n", + " return (TextMessage,)\n", + "\n", + " async def on_messages(self, messages: Sequence[ChatMessage], cancellation_token: CancellationToken) -> Response:\n", + " async for message in self.on_messages_stream(messages, cancellation_token):\n", + " if isinstance(message, Response):\n", + " return message\n", + " raise AssertionError(\"The stream should have returned the final result.\")\n", + "\n", + " async def on_messages_stream(\n", + " self, messages: Sequence[ChatMessage], cancellation_token: CancellationToken\n", + " ) -> AsyncGenerator[AgentEvent | ChatMessage | Response, None]:\n", + " # Add messages to the model context\n", + " for msg in messages:\n", + " await self._model_context.add_message(UserMessage(content=msg.content, source=msg.source))\n", + "\n", + " # Get conversation history\n", + " history = [msg.source + \": \" + msg.content + \"\\n\" for msg in await self._model_context.get_messages()]\n", + "\n", + " # Generate response using Gemini\n", + " response = self._model_client.models.generate_content(\n", + " model=self._model,\n", + " contents=f\"History: {history}\\nGiven the history, please provide a response\",\n", + " config=types.GenerateContentConfig(\n", + " system_instruction=self._system_message,\n", + " temperature=0.3,\n", + " ),\n", + " )\n", + "\n", + " # Create usage metadata\n", + " usage = RequestUsage(\n", + " prompt_tokens=response.usage_metadata.prompt_token_count,\n", + " completion_tokens=response.usage_metadata.candidates_token_count,\n", + " )\n", + "\n", + " # Add response to model context\n", + " await self._model_context.add_message(AssistantMessage(content=response.text, source=self.name))\n", + "\n", + " # Yield the final response\n", + " yield Response(\n", + " chat_message=TextMessage(content=response.text, source=self.name, models_usage=usage),\n", + " inner_messages=[],\n", + " )\n", + "\n", + " async def on_reset(self, cancellation_token: CancellationToken) -> None:\n", + " \"\"\"Reset the assistant by clearing the model context.\"\"\"\n", + " await self._model_context.clear()" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "---------- user ----------\n", + "What is the capital of New York?\n", + "---------- gemini_assistant ----------\n", + "Albany\n", + "TERMINATE\n", + "\n" + ] + }, + { + "data": { + "text/plain": [ + "TaskResult(messages=[TextMessage(source='user', models_usage=None, content='What is the capital of New York?', type='TextMessage'), TextMessage(source='gemini_assistant', models_usage=RequestUsage(prompt_tokens=46, completion_tokens=5), content='Albany\\nTERMINATE\\n', type='TextMessage')], stop_reason=None)" + ] + }, + "execution_count": 59, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "gemini_assistant = GeminiAssistantAgent(\"gemini_assistant\")\n", + "await Console(gemini_assistant.run_stream(task=\"What is the capital of New York?\"))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the example above, we have chosen to provide `model`, `api_key` and `system_message` as arguments - you can choose to provide any other arguments that are required by the model client you are using or fits with your application design. \n", + "\n", + "Now, let us explore how to use this custom agent as part of a team in AgentChat." + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "---------- user ----------\n", + "Write a Haiku poem with 4 lines about the fall season.\n", + "---------- primary ----------\n", + "Crisp leaves dance and swirl, \n", + "Golden hues paint the still air, \n", + "Whispers of the breeze, \n", + "Autumn's quiet embrace.\n", + "---------- gemini_critic ----------\n", + "The haiku has four lines, not three. A haiku must have three lines with a 5-7-5 syllable structure. The poem is lovely, but needs to be edited to fit the haiku form.\n", + "\n", + "---------- primary ----------\n", + "Thank you for your feedback! Here’s a revised three-line haiku about fall, adhering to the 5-7-5 syllable structure:\n", + "\n", + "Crisp leaves tumble down, \n", + "Golden hues brush the cool air, \n", + "Autumn whispers peace.\n", + "---------- gemini_critic ----------\n", + "The revised haiku is much improved. It correctly follows the 5-7-5 syllable structure and captures the essence of autumn. APPROVE\n", + "\n" + ] + }, + { + "data": { + "text/plain": [ + "TaskResult(messages=[TextMessage(source='user', models_usage=None, content='Write a Haiku poem with 4 lines about the fall season.', type='TextMessage'), TextMessage(source='primary', models_usage=RequestUsage(prompt_tokens=33, completion_tokens=31), content=\"Crisp leaves dance and swirl, \\nGolden hues paint the still air, \\nWhispers of the breeze, \\nAutumn's quiet embrace.\", type='TextMessage'), TextMessage(source='gemini_critic', models_usage=RequestUsage(prompt_tokens=86, completion_tokens=45), content='The haiku has four lines, not three. A haiku must have three lines with a 5-7-5 syllable structure. The poem is lovely, but needs to be edited to fit the haiku form.\\n', type='TextMessage'), TextMessage(source='primary', models_usage=RequestUsage(prompt_tokens=126, completion_tokens=52), content='Thank you for your feedback! Here’s a revised three-line haiku about fall, adhering to the 5-7-5 syllable structure:\\n\\nCrisp leaves tumble down, \\nGolden hues brush the cool air, \\nAutumn whispers peace.', type='TextMessage'), TextMessage(source='gemini_critic', models_usage=RequestUsage(prompt_tokens=200, completion_tokens=31), content='The revised haiku is much improved. It correctly follows the 5-7-5 syllable structure and captures the essence of autumn. APPROVE\\n', type='TextMessage')], stop_reason=\"Text 'APPROVE' mentioned\")" + ] + }, + "execution_count": 69, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from autogen_agentchat.agents import AssistantAgent\n", + "from autogen_agentchat.base import TaskResult\n", + "from autogen_agentchat.conditions import TextMentionTermination\n", + "from autogen_agentchat.teams import RoundRobinGroupChat\n", + "from autogen_agentchat.ui import Console\n", + "from autogen_core import CancellationToken\n", + "\n", + "# Create the primary agent.\n", + "primary_agent = AssistantAgent(\n", + " \"primary\",\n", + " model_client=OpenAIChatCompletionClient(model=\"gpt-4o-mini\"),\n", + " system_message=\"You are a helpful AI assistant.\",\n", + ")\n", + "\n", + "# Create a critic agent based on our new GeminiAssistantAgent.\n", + "critic_agent = GeminiAssistantAgent(\n", + " \"gemini_critic\",\n", + " system_message=\"Provide constructive feedback. Respond with 'APPROVE' to when your feedbacks are addressed.\",\n", + ")\n", + "\n", + "\n", + "# Define a termination condition that stops the task if the critic approves or after 10 messages.\n", + "text_termination = TextMentionTermination(\"APPROVE\") | MaxMessageTermination(10)\n", + "\n", + "# Create a team with the primary and critic agents.\n", + "team = RoundRobinGroupChat([primary_agent, critic_agent], termination_condition=text_termination)\n", + "\n", + "await Console(team.run_stream(task=\"Write a Haiku poem with 4 lines about the fall season.\"))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In section above, we show several very important concepts:\n", + "- We have developed a custom agent that uses the Google Gemini SDK to respond to messages. \n", + "- We show that this custom agent can be used as part of the broader AgentChat ecosystem - in this case as a participant in a {py:class}`~autogen_agentchat.teams.RoundRobinGroupChat` as long as it inherits from {py:class}`~autogen_agentchat.agents.BaseChatAgent`.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Making the Custom Agent Declarative \n", + "\n", + "The Autogen framework provides a [Component](https://microsoft.github.io/autogen/dev/user-guide/core-user-guide/framework/component-config.html) interface for making the configuration of components serializable to a declarative format. This is useful for saving and loading configurations, and for sharing configurations with others. \n", + "\n", + "We accomplish this by inheriting from the `Component` class and implementing the `_from_config` and `_to_config` methods.\n", + "The declarative class can be serialized to a JSON format using the `dump_component` method, and deserialized from a JSON format using the `load_component` method." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from typing import AsyncGenerator, Sequence\n", + "\n", + "from autogen_agentchat.agents import BaseChatAgent\n", + "from autogen_agentchat.base import Response\n", + "from autogen_agentchat.messages import AgentEvent, ChatMessage\n", + "from autogen_core import CancellationToken, Component\n", + "from pydantic import BaseModel\n", + "\n", + "\n", + "class GeminiAssistantAgentConfig(BaseModel):\n", + " name: str\n", + " description: str = \"An agent that provides assistance with ability to use tools.\"\n", + " model: str = \"gemini-1.5-flash-002\"\n", + " system_message: str | None = None\n", + "\n", + "\n", + "class GeminiAssistant(BaseChatAgent, Component[GeminiAssistantAgentConfig]):\n", + " component_config_schema = GeminiAssistantAgentConfig\n", + " # component_provider_override = \"mypackage.agents.GeminiAssistant\"\n", + "\n", + " def __init__(\n", + " self,\n", + " name: str,\n", + " description: str = \"An agent that provides assistance with ability to use tools.\",\n", + " model: str = \"gemini-1.5-flash-002\",\n", + " api_key: str = os.environ[\"GEMINI_API_KEY\"],\n", + " system_message: str = \"You are a helpful assistant that can respond to messages. Reply with TERMINATE when the task has been completed.\",\n", + " ):\n", + " super().__init__(name=name, description=description)\n", + " self._model_context = UnboundedChatCompletionContext()\n", + " self._model_client = genai.Client(api_key=api_key)\n", + " self._system_message = system_message\n", + " self._model = model\n", + "\n", + " @property\n", + " def produced_message_types(self) -> Sequence[type[ChatMessage]]:\n", + " return (TextMessage,)\n", + "\n", + " async def on_messages(self, messages: Sequence[ChatMessage], cancellation_token: CancellationToken) -> Response:\n", + " async for message in self.on_messages_stream(messages, cancellation_token):\n", + " if isinstance(message, Response):\n", + " return message\n", + " raise AssertionError(\"The stream should have returned the final result.\")\n", + "\n", + " async def on_messages_stream(\n", + " self, messages: Sequence[ChatMessage], cancellation_token: CancellationToken\n", + " ) -> AsyncGenerator[AgentEvent | ChatMessage | Response, None]:\n", + " # Add messages to the model context\n", + " for msg in messages:\n", + " await self._model_context.add_message(UserMessage(content=msg.content, source=msg.source))\n", + "\n", + " # Get conversation history\n", + " history = [msg.source + \": \" + msg.content + \"\\n\" for msg in await self._model_context.get_messages()]\n", + "\n", + " # Generate response using Gemini\n", + " response = self._model_client.models.generate_content(\n", + " model=self._model,\n", + " contents=f\"History: {history}\\nGiven the history, please provide a response\",\n", + " config=types.GenerateContentConfig(\n", + " system_instruction=self._system_message,\n", + " temperature=0.3,\n", + " ),\n", + " )\n", + "\n", + " # Create usage metadata\n", + " usage = RequestUsage(\n", + " prompt_tokens=response.usage_metadata.prompt_token_count,\n", + " completion_tokens=response.usage_metadata.candidates_token_count,\n", + " )\n", + "\n", + " # Add response to model context\n", + " await self._model_context.add_message(AssistantMessage(content=response.text, source=self.name))\n", + "\n", + " # Yield the final response\n", + " yield Response(\n", + " chat_message=TextMessage(content=response.text, source=self.name, models_usage=usage),\n", + " inner_messages=[],\n", + " )\n", + "\n", + " async def on_reset(self, cancellation_token: CancellationToken) -> None:\n", + " \"\"\"Reset the assistant by clearing the model context.\"\"\"\n", + " await self._model_context.clear()\n", + "\n", + " @classmethod\n", + " def _from_config(cls, config: GeminiAssistantAgentConfig) -> \"GeminiAssistant\":\n", + " return cls(\n", + " name=config.name, description=config.description, model=config.model, system_message=config.system_message\n", + " )\n", + "\n", + " def _to_config(self) -> GeminiAssistantAgentConfig:\n", + " return GeminiAssistantAgentConfig(\n", + " name=self.name,\n", + " description=self.description,\n", + " model=self._model,\n", + " system_message=self._system_message,\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now that we have the required methods implemented, we can now load and dump the custom agent to and from a JSON format, and then load the agent from the JSON format.\n", + " \n", + " > Note: You should set the `component_provider_override` class variable to the full path of the module containing the custom agent class e.g., (mypackage.agents.GeminiAssistantAgent). This is used by `load_component` method to determine how to instantiate the class." + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"provider\": \"__main__.GeminiAssistant\",\n", + " \"component_type\": \"agent\",\n", + " \"version\": 1,\n", + " \"component_version\": 1,\n", + " \"description\": null,\n", + " \"label\": \"GeminiAssistant\",\n", + " \"config\": {\n", + " \"name\": \"gemini_assistant\",\n", + " \"description\": \"An agent that provides assistance with ability to use tools.\",\n", + " \"model\": \"gemini-1.5-flash-002\",\n", + " \"system_message\": \"You are a helpful assistant that can respond to messages. Reply with TERMINATE when the task has been completed.\"\n", + " }\n", + "}\n", + "<__main__.GeminiAssistant object at 0x14ab6d890>\n" + ] + } + ], + "source": [ + "gemini_assistant = GeminiAssistant(\"gemini_assistant\")\n", + "config = gemini_assistant.dump_component()\n", + "print(config.model_dump_json(indent=2))\n", + "loaded_agent = GeminiAssistant.load_component(config)\n", + "print(loaded_agent)" + ] } ], "metadata": { @@ -305,7 +692,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.5" + "version": "3.11.9" } }, "nbformat": 4, From da34b533755f4451ba4bc84250fdfe384b1a16d0 Mon Sep 17 00:00:00 2001 From: Victor Dibia Date: Sun, 9 Feb 2025 18:35:17 -0800 Subject: [PATCH 02/10] minor update --- .../tutorial/custom-agents.ipynb | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/custom-agents.ipynb b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/custom-agents.ipynb index 76319974ca98..a4797de78261 100644 --- a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/custom-agents.ipynb +++ b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/custom-agents.ipynb @@ -435,7 +435,7 @@ }, { "cell_type": "code", - "execution_count": 69, + "execution_count": 71, "metadata": {}, "outputs": [ { @@ -445,31 +445,31 @@ "---------- user ----------\n", "Write a Haiku poem with 4 lines about the fall season.\n", "---------- primary ----------\n", - "Crisp leaves dance and swirl, \n", - "Golden hues paint the still air, \n", - "Whispers of the breeze, \n", - "Autumn's quiet embrace.\n", + "Leaves dance in the breeze, \n", + "Crimson and gold softly fall, \n", + "Whispers of crisp air, \n", + "Nature's quilt, a warm call.\n", "---------- gemini_critic ----------\n", - "The haiku has four lines, not three. A haiku must have three lines with a 5-7-5 syllable structure. The poem is lovely, but needs to be edited to fit the haiku form.\n", + "The haiku is lovely and evocative, but it has four lines instead of three. To be a true haiku, one line needs to be removed. Consider which line contributes least to the overall image and remove that. The imagery is strong, though.\n", "\n", "---------- primary ----------\n", - "Thank you for your feedback! Here’s a revised three-line haiku about fall, adhering to the 5-7-5 syllable structure:\n", + "Thank you for the feedback! Here’s a revised version of the haiku with three lines:\n", "\n", - "Crisp leaves tumble down, \n", - "Golden hues brush the cool air, \n", - "Autumn whispers peace.\n", + "Leaves dance in the breeze, \n", + "Crimson and gold softly fall, \n", + "Nature's quilt, a warm call.\n", "---------- gemini_critic ----------\n", - "The revised haiku is much improved. It correctly follows the 5-7-5 syllable structure and captures the essence of autumn. APPROVE\n", + "The revised haiku is much improved! Removing the \"Whispers of crisp air\" line maintains the strong imagery while adhering to the three-line structure. APPROVE\n", "\n" ] }, { "data": { "text/plain": [ - "TaskResult(messages=[TextMessage(source='user', models_usage=None, content='Write a Haiku poem with 4 lines about the fall season.', type='TextMessage'), TextMessage(source='primary', models_usage=RequestUsage(prompt_tokens=33, completion_tokens=31), content=\"Crisp leaves dance and swirl, \\nGolden hues paint the still air, \\nWhispers of the breeze, \\nAutumn's quiet embrace.\", type='TextMessage'), TextMessage(source='gemini_critic', models_usage=RequestUsage(prompt_tokens=86, completion_tokens=45), content='The haiku has four lines, not three. A haiku must have three lines with a 5-7-5 syllable structure. The poem is lovely, but needs to be edited to fit the haiku form.\\n', type='TextMessage'), TextMessage(source='primary', models_usage=RequestUsage(prompt_tokens=126, completion_tokens=52), content='Thank you for your feedback! Here’s a revised three-line haiku about fall, adhering to the 5-7-5 syllable structure:\\n\\nCrisp leaves tumble down, \\nGolden hues brush the cool air, \\nAutumn whispers peace.', type='TextMessage'), TextMessage(source='gemini_critic', models_usage=RequestUsage(prompt_tokens=200, completion_tokens=31), content='The revised haiku is much improved. It correctly follows the 5-7-5 syllable structure and captures the essence of autumn. APPROVE\\n', type='TextMessage')], stop_reason=\"Text 'APPROVE' mentioned\")" + "TaskResult(messages=[TextMessage(source='user', models_usage=None, content='Write a Haiku poem with 4 lines about the fall season.', type='TextMessage'), TextMessage(source='primary', models_usage=RequestUsage(prompt_tokens=33, completion_tokens=31), content=\"Leaves dance in the breeze, \\nCrimson and gold softly fall, \\nWhispers of crisp air, \\nNature's quilt, a warm call.\", type='TextMessage'), TextMessage(source='gemini_critic', models_usage=RequestUsage(prompt_tokens=88, completion_tokens=53), content='The haiku is lovely and evocative, but it has four lines instead of three. To be a true haiku, one line needs to be removed. Consider which line contributes least to the overall image and remove that. The imagery is strong, though.\\n', type='TextMessage'), TextMessage(source='primary', models_usage=RequestUsage(prompt_tokens=133, completion_tokens=43), content=\"Thank you for the feedback! Here’s a revised version of the haiku with three lines:\\n\\nLeaves dance in the breeze, \\nCrimson and gold softly fall, \\nNature's quilt, a warm call.\", type='TextMessage'), TextMessage(source='gemini_critic', models_usage=RequestUsage(prompt_tokens=203, completion_tokens=35), content='The revised haiku is much improved! Removing the \"Whispers of crisp air\" line maintains the strong imagery while adhering to the three-line structure. APPROVE\\n', type='TextMessage')], stop_reason=\"Text 'APPROVE' mentioned\")" ] }, - "execution_count": 69, + "execution_count": 71, "metadata": {}, "output_type": "execute_result" } @@ -490,7 +490,7 @@ ")\n", "\n", "# Create a critic agent based on our new GeminiAssistantAgent.\n", - "critic_agent = GeminiAssistantAgent(\n", + "gemini_critic_agent = GeminiAssistantAgent(\n", " \"gemini_critic\",\n", " system_message=\"Provide constructive feedback. Respond with 'APPROVE' to when your feedbacks are addressed.\",\n", ")\n", @@ -500,7 +500,7 @@ "text_termination = TextMentionTermination(\"APPROVE\") | MaxMessageTermination(10)\n", "\n", "# Create a team with the primary and critic agents.\n", - "team = RoundRobinGroupChat([primary_agent, critic_agent], termination_condition=text_termination)\n", + "team = RoundRobinGroupChat([primary_agent, gemini_critic_agent], termination_condition=text_termination)\n", "\n", "await Console(team.run_stream(task=\"Write a Haiku poem with 4 lines about the fall season.\"))" ] From f9037d5caa32bfd3bc1c8e9bb572771d36d5a803 Mon Sep 17 00:00:00 2001 From: Victor Dibia Date: Sun, 9 Feb 2025 18:35:56 -0800 Subject: [PATCH 03/10] minor updates --- .../agentchat-user-guide/tutorial/custom-agents.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/custom-agents.ipynb b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/custom-agents.ipynb index a4797de78261..f23ff9686f3f 100644 --- a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/custom-agents.ipynb +++ b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/custom-agents.ipynb @@ -435,7 +435,7 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -497,10 +497,10 @@ "\n", "\n", "# Define a termination condition that stops the task if the critic approves or after 10 messages.\n", - "text_termination = TextMentionTermination(\"APPROVE\") | MaxMessageTermination(10)\n", + "termination = TextMentionTermination(\"APPROVE\") | MaxMessageTermination(10)\n", "\n", "# Create a team with the primary and critic agents.\n", - "team = RoundRobinGroupChat([primary_agent, gemini_critic_agent], termination_condition=text_termination)\n", + "team = RoundRobinGroupChat([primary_agent, gemini_critic_agent], termination_condition=termination)\n", "\n", "await Console(team.run_stream(task=\"Write a Haiku poem with 4 lines about the fall season.\"))" ] From 9b9a76dffe3af34c495e56021d9d9b95d2310e6c Mon Sep 17 00:00:00 2001 From: Victor Dibia Date: Sun, 9 Feb 2025 20:09:17 -0800 Subject: [PATCH 04/10] mypy fixes --- .../tutorial/custom-agents.ipynb | 78 +++++++++---------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/custom-agents.ipynb b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/custom-agents.ipynb index f23ff9686f3f..26386b661b51 100644 --- a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/custom-agents.ipynb +++ b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/custom-agents.ipynb @@ -308,17 +308,17 @@ }, { "cell_type": "code", - "execution_count": 58, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ "# !pip install google-genai\n", "import os\n", "from typing import AsyncGenerator, Sequence\n", - "\n", + "from autogen_agentchat.ui import Console\n", "from autogen_agentchat.agents import BaseChatAgent\n", "from autogen_agentchat.base import Response\n", - "from autogen_agentchat.messages import AgentEvent, ChatMessage\n", + "from autogen_agentchat.messages import AgentEvent, ChatMessage, TextMessage\n", "from autogen_core import CancellationToken\n", "from autogen_core.model_context import UnboundedChatCompletionContext\n", "from autogen_core.models import AssistantMessage, RequestUsage, UserMessage\n", @@ -333,7 +333,7 @@ " description: str = \"An agent that provides assistance with ability to use tools.\",\n", " model: str = \"gemini-1.5-flash-002\",\n", " api_key: str = os.environ[\"GEMINI_API_KEY\"],\n", - " system_message: str = \"You are a helpful assistant that can respond to messages. Reply with TERMINATE when the task has been completed.\",\n", + " system_message: str | None = \"You are a helpful assistant that can respond to messages. Reply with TERMINATE when the task has been completed.\",\n", " ):\n", " super().__init__(name=name, description=description)\n", " self._model_context = UnboundedChatCompletionContext()\n", @@ -357,10 +357,10 @@ " # Add messages to the model context\n", " for msg in messages:\n", " await self._model_context.add_message(UserMessage(content=msg.content, source=msg.source))\n", + " \n", "\n", " # Get conversation history\n", - " history = [msg.source + \": \" + msg.content + \"\\n\" for msg in await self._model_context.get_messages()]\n", - "\n", + " history = [(msg.source if hasattr(msg, \"source\") else \"system\" ) + \": \" + (msg.content if isinstance(msg.content, str) else \"\") + \"\\n\" for msg in await self._model_context.get_messages()] \n", " # Generate response using Gemini\n", " response = self._model_client.models.generate_content(\n", " model=self._model,\n", @@ -393,7 +393,7 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 29, "metadata": {}, "outputs": [ { @@ -414,7 +414,7 @@ "TaskResult(messages=[TextMessage(source='user', models_usage=None, content='What is the capital of New York?', type='TextMessage'), TextMessage(source='gemini_assistant', models_usage=RequestUsage(prompt_tokens=46, completion_tokens=5), content='Albany\\nTERMINATE\\n', type='TextMessage')], stop_reason=None)" ] }, - "execution_count": 59, + "execution_count": 29, "metadata": {}, "output_type": "execute_result" } @@ -435,7 +435,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "metadata": {}, "outputs": [ { @@ -445,42 +445,41 @@ "---------- user ----------\n", "Write a Haiku poem with 4 lines about the fall season.\n", "---------- primary ----------\n", - "Leaves dance in the breeze, \n", - "Crimson and gold softly fall, \n", - "Whispers of crisp air, \n", - "Nature's quilt, a warm call.\n", + "Leaves of crimson dance, \n", + "Whispers of the crisp cool air, \n", + "Pumpkins grin with light, \n", + "Nature's quilt laid bare.\n", "---------- gemini_critic ----------\n", - "The haiku is lovely and evocative, but it has four lines instead of three. To be a true haiku, one line needs to be removed. Consider which line contributes least to the overall image and remove that. The imagery is strong, though.\n", + "The haiku only has three lines. The fourth line should be removed. Otherwise, it's a good poem.\n", "\n", "---------- primary ----------\n", - "Thank you for the feedback! Here’s a revised version of the haiku with three lines:\n", + "Thank you for your feedback! Here’s the revised haiku with three lines:\n", "\n", - "Leaves dance in the breeze, \n", - "Crimson and gold softly fall, \n", - "Nature's quilt, a warm call.\n", + "Leaves of crimson dance, \n", + "Whispers of the crisp cool air, \n", + "Pumpkins grin with light.\n", "---------- gemini_critic ----------\n", - "The revised haiku is much improved! Removing the \"Whispers of crisp air\" line maintains the strong imagery while adhering to the three-line structure. APPROVE\n", + "APPROVE\n", "\n" ] }, { "data": { "text/plain": [ - "TaskResult(messages=[TextMessage(source='user', models_usage=None, content='Write a Haiku poem with 4 lines about the fall season.', type='TextMessage'), TextMessage(source='primary', models_usage=RequestUsage(prompt_tokens=33, completion_tokens=31), content=\"Leaves dance in the breeze, \\nCrimson and gold softly fall, \\nWhispers of crisp air, \\nNature's quilt, a warm call.\", type='TextMessage'), TextMessage(source='gemini_critic', models_usage=RequestUsage(prompt_tokens=88, completion_tokens=53), content='The haiku is lovely and evocative, but it has four lines instead of three. To be a true haiku, one line needs to be removed. Consider which line contributes least to the overall image and remove that. The imagery is strong, though.\\n', type='TextMessage'), TextMessage(source='primary', models_usage=RequestUsage(prompt_tokens=133, completion_tokens=43), content=\"Thank you for the feedback! Here’s a revised version of the haiku with three lines:\\n\\nLeaves dance in the breeze, \\nCrimson and gold softly fall, \\nNature's quilt, a warm call.\", type='TextMessage'), TextMessage(source='gemini_critic', models_usage=RequestUsage(prompt_tokens=203, completion_tokens=35), content='The revised haiku is much improved! Removing the \"Whispers of crisp air\" line maintains the strong imagery while adhering to the three-line structure. APPROVE\\n', type='TextMessage')], stop_reason=\"Text 'APPROVE' mentioned\")" + "TaskResult(messages=[TextMessage(source='user', models_usage=None, content='Write a Haiku poem with 4 lines about the fall season.', type='TextMessage'), TextMessage(source='primary', models_usage=RequestUsage(prompt_tokens=33, completion_tokens=29), content=\"Leaves of crimson dance, \\nWhispers of the crisp cool air, \\nPumpkins grin with light, \\nNature's quilt laid bare.\", type='TextMessage'), TextMessage(source='gemini_critic', models_usage=RequestUsage(prompt_tokens=87, completion_tokens=26), content=\"The haiku only has three lines. The fourth line should be removed. Otherwise, it's a good poem.\\n\", type='TextMessage'), TextMessage(source='primary', models_usage=RequestUsage(prompt_tokens=100, completion_tokens=38), content='Thank you for your feedback! Here’s the revised haiku with three lines:\\n\\nLeaves of crimson dance, \\nWhispers of the crisp cool air, \\nPumpkins grin with light.', type='TextMessage'), TextMessage(source='gemini_critic', models_usage=RequestUsage(prompt_tokens=170, completion_tokens=3), content='APPROVE\\n', type='TextMessage')], stop_reason=\"Text 'APPROVE' mentioned\")" ] }, - "execution_count": 71, + "execution_count": 30, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "from autogen_agentchat.agents import AssistantAgent\n", - "from autogen_agentchat.base import TaskResult\n", - "from autogen_agentchat.conditions import TextMentionTermination\n", + "from autogen_agentchat.agents import AssistantAgent \n", + "from autogen_agentchat.conditions import TextMentionTermination, MaxMessageTermination\n", "from autogen_agentchat.teams import RoundRobinGroupChat\n", - "from autogen_agentchat.ui import Console\n", - "from autogen_core import CancellationToken\n", + "from autogen_agentchat.ui import Console \n", + "from autogen_ext.models.openai import OpenAIChatCompletionClient\n", "\n", "# Create the primary agent.\n", "primary_agent = AssistantAgent(\n", @@ -540,6 +539,7 @@ "from autogen_agentchat.messages import AgentEvent, ChatMessage\n", "from autogen_core import CancellationToken, Component\n", "from pydantic import BaseModel\n", + "from typing_extensions import Self\n", "\n", "\n", "class GeminiAssistantAgentConfig(BaseModel):\n", @@ -549,9 +549,9 @@ " system_message: str | None = None\n", "\n", "\n", - "class GeminiAssistant(BaseChatAgent, Component[GeminiAssistantAgentConfig]):\n", + "class GeminiAssistantAgent(BaseChatAgent, Component[GeminiAssistantAgentConfig]): # type: ignore[no-redef]\n", " component_config_schema = GeminiAssistantAgentConfig\n", - " # component_provider_override = \"mypackage.agents.GeminiAssistant\"\n", + " # component_provider_override = \"mypackage.agents.GeminiAssistantAgent\"\n", "\n", " def __init__(\n", " self,\n", @@ -559,7 +559,7 @@ " description: str = \"An agent that provides assistance with ability to use tools.\",\n", " model: str = \"gemini-1.5-flash-002\",\n", " api_key: str = os.environ[\"GEMINI_API_KEY\"],\n", - " system_message: str = \"You are a helpful assistant that can respond to messages. Reply with TERMINATE when the task has been completed.\",\n", + " system_message: str | None = \"You are a helpful assistant that can respond to messages. Reply with TERMINATE when the task has been completed.\",\n", " ):\n", " super().__init__(name=name, description=description)\n", " self._model_context = UnboundedChatCompletionContext()\n", @@ -583,9 +583,9 @@ " # Add messages to the model context\n", " for msg in messages:\n", " await self._model_context.add_message(UserMessage(content=msg.content, source=msg.source))\n", - "\n", + " \n", " # Get conversation history\n", - " history = [msg.source + \": \" + msg.content + \"\\n\" for msg in await self._model_context.get_messages()]\n", + " history = [(msg.source if hasattr(msg, \"source\") else \"system\" ) + \": \" + (msg.content if isinstance(msg.content, str) else \"\") + \"\\n\" for msg in await self._model_context.get_messages()]\n", "\n", " # Generate response using Gemini\n", " response = self._model_client.models.generate_content(\n", @@ -617,7 +617,7 @@ " await self._model_context.clear()\n", "\n", " @classmethod\n", - " def _from_config(cls, config: GeminiAssistantAgentConfig) -> \"GeminiAssistant\":\n", + " def _from_config(cls, config: GeminiAssistantAgentConfig) -> Self:\n", " return cls(\n", " name=config.name, description=config.description, model=config.model, system_message=config.system_message\n", " )\n", @@ -637,12 +637,12 @@ "source": [ "Now that we have the required methods implemented, we can now load and dump the custom agent to and from a JSON format, and then load the agent from the JSON format.\n", " \n", - " > Note: You should set the `component_provider_override` class variable to the full path of the module containing the custom agent class e.g., (mypackage.agents.GeminiAssistantAgent). This is used by `load_component` method to determine how to instantiate the class." + " > Note: You should set the `component_provider_override` class variable to the full path of the module containing the custom agent class e.g., (`mypackage.agents.GeminiAssistantAgent`). This is used by `load_component` method to determine how to instantiate the class." ] }, { "cell_type": "code", - "execution_count": 67, + "execution_count": 36, "metadata": {}, "outputs": [ { @@ -650,12 +650,12 @@ "output_type": "stream", "text": [ "{\n", - " \"provider\": \"__main__.GeminiAssistant\",\n", + " \"provider\": \"__main__.GeminiAssistantAgent\",\n", " \"component_type\": \"agent\",\n", " \"version\": 1,\n", " \"component_version\": 1,\n", " \"description\": null,\n", - " \"label\": \"GeminiAssistant\",\n", + " \"label\": \"GeminiAssistantAgent\",\n", " \"config\": {\n", " \"name\": \"gemini_assistant\",\n", " \"description\": \"An agent that provides assistance with ability to use tools.\",\n", @@ -663,15 +663,15 @@ " \"system_message\": \"You are a helpful assistant that can respond to messages. Reply with TERMINATE when the task has been completed.\"\n", " }\n", "}\n", - "<__main__.GeminiAssistant object at 0x14ab6d890>\n" + "<__main__.GeminiAssistantAgent object at 0x11a9a4f10>\n" ] } ], "source": [ - "gemini_assistant = GeminiAssistant(\"gemini_assistant\")\n", + "gemini_assistant = GeminiAssistantAgent(\"gemini_assistant\")\n", "config = gemini_assistant.dump_component()\n", "print(config.model_dump_json(indent=2))\n", - "loaded_agent = GeminiAssistant.load_component(config)\n", + "loaded_agent = GeminiAssistantAgent.load_component(config)\n", "print(loaded_agent)" ] } From eb36cda5f1e0d7887264e5a0514d93d5d48eb3f5 Mon Sep 17 00:00:00 2001 From: Victor Dibia Date: Sun, 9 Feb 2025 20:14:47 -0800 Subject: [PATCH 05/10] format fixes --- .../tutorial/custom-agents.ipynb | 38 ++++++++++++------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/custom-agents.ipynb b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/custom-agents.ipynb index 26386b661b51..ba1a7caadd1c 100644 --- a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/custom-agents.ipynb +++ b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/custom-agents.ipynb @@ -315,10 +315,10 @@ "# !pip install google-genai\n", "import os\n", "from typing import AsyncGenerator, Sequence\n", - "from autogen_agentchat.ui import Console\n", + "\n", "from autogen_agentchat.agents import BaseChatAgent\n", "from autogen_agentchat.base import Response\n", - "from autogen_agentchat.messages import AgentEvent, ChatMessage, TextMessage\n", + "from autogen_agentchat.messages import AgentEvent, ChatMessage\n", "from autogen_core import CancellationToken\n", "from autogen_core.model_context import UnboundedChatCompletionContext\n", "from autogen_core.models import AssistantMessage, RequestUsage, UserMessage\n", @@ -333,7 +333,8 @@ " description: str = \"An agent that provides assistance with ability to use tools.\",\n", " model: str = \"gemini-1.5-flash-002\",\n", " api_key: str = os.environ[\"GEMINI_API_KEY\"],\n", - " system_message: str | None = \"You are a helpful assistant that can respond to messages. Reply with TERMINATE when the task has been completed.\",\n", + " system_message: str\n", + " | None = \"You are a helpful assistant that can respond to messages. Reply with TERMINATE when the task has been completed.\",\n", " ):\n", " super().__init__(name=name, description=description)\n", " self._model_context = UnboundedChatCompletionContext()\n", @@ -357,10 +358,15 @@ " # Add messages to the model context\n", " for msg in messages:\n", " await self._model_context.add_message(UserMessage(content=msg.content, source=msg.source))\n", - " \n", "\n", " # Get conversation history\n", - " history = [(msg.source if hasattr(msg, \"source\") else \"system\" ) + \": \" + (msg.content if isinstance(msg.content, str) else \"\") + \"\\n\" for msg in await self._model_context.get_messages()] \n", + " history = [\n", + " (msg.source if hasattr(msg, \"source\") else \"system\")\n", + " + \": \"\n", + " + (msg.content if isinstance(msg.content, str) else \"\")\n", + " + \"\\n\"\n", + " for msg in await self._model_context.get_messages()\n", + " ]\n", " # Generate response using Gemini\n", " response = self._model_client.models.generate_content(\n", " model=self._model,\n", @@ -475,11 +481,10 @@ } ], "source": [ - "from autogen_agentchat.agents import AssistantAgent \n", - "from autogen_agentchat.conditions import TextMentionTermination, MaxMessageTermination\n", + "from autogen_agentchat.agents import AssistantAgent\n", + "from autogen_agentchat.conditions import TextMentionTermination\n", "from autogen_agentchat.teams import RoundRobinGroupChat\n", - "from autogen_agentchat.ui import Console \n", - "from autogen_ext.models.openai import OpenAIChatCompletionClient\n", + "from autogen_agentchat.ui import Console\n", "\n", "# Create the primary agent.\n", "primary_agent = AssistantAgent(\n", @@ -549,7 +554,7 @@ " system_message: str | None = None\n", "\n", "\n", - "class GeminiAssistantAgent(BaseChatAgent, Component[GeminiAssistantAgentConfig]): # type: ignore[no-redef]\n", + "class GeminiAssistantAgent(BaseChatAgent, Component[GeminiAssistantAgentConfig]): # type: ignore[no-redef]\n", " component_config_schema = GeminiAssistantAgentConfig\n", " # component_provider_override = \"mypackage.agents.GeminiAssistantAgent\"\n", "\n", @@ -559,7 +564,8 @@ " description: str = \"An agent that provides assistance with ability to use tools.\",\n", " model: str = \"gemini-1.5-flash-002\",\n", " api_key: str = os.environ[\"GEMINI_API_KEY\"],\n", - " system_message: str | None = \"You are a helpful assistant that can respond to messages. Reply with TERMINATE when the task has been completed.\",\n", + " system_message: str\n", + " | None = \"You are a helpful assistant that can respond to messages. Reply with TERMINATE when the task has been completed.\",\n", " ):\n", " super().__init__(name=name, description=description)\n", " self._model_context = UnboundedChatCompletionContext()\n", @@ -583,9 +589,15 @@ " # Add messages to the model context\n", " for msg in messages:\n", " await self._model_context.add_message(UserMessage(content=msg.content, source=msg.source))\n", - " \n", + "\n", " # Get conversation history\n", - " history = [(msg.source if hasattr(msg, \"source\") else \"system\" ) + \": \" + (msg.content if isinstance(msg.content, str) else \"\") + \"\\n\" for msg in await self._model_context.get_messages()]\n", + " history = [\n", + " (msg.source if hasattr(msg, \"source\") else \"system\")\n", + " + \": \"\n", + " + (msg.content if isinstance(msg.content, str) else \"\")\n", + " + \"\\n\"\n", + " for msg in await self._model_context.get_messages()\n", + " ]\n", "\n", " # Generate response using Gemini\n", " response = self._model_client.models.generate_content(\n", From 01e3fc0c71427e05c4f144d98741dbcbfdbef454 Mon Sep 17 00:00:00 2001 From: Victor Dibia Date: Mon, 10 Feb 2025 15:10:07 -0800 Subject: [PATCH 06/10] tutorial doc location refactor and lint fixes --- python/packages/autogen-core/docs/src/conf.py | 1 + .../{tutorial => }/custom-agents.ipynb | 82 ++++++++++++------- .../user-guide/agentchat-user-guide/index.md | 2 +- 3 files changed, 55 insertions(+), 30 deletions(-) rename python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/{tutorial => }/custom-agents.ipynb (85%) diff --git a/python/packages/autogen-core/docs/src/conf.py b/python/packages/autogen-core/docs/src/conf.py index 5c4fcbb0ccfb..bfb8123da32b 100644 --- a/python/packages/autogen-core/docs/src/conf.py +++ b/python/packages/autogen-core/docs/src/conf.py @@ -178,6 +178,7 @@ "user-guide/core-user-guide/framework/command-line-code-executors.ipynb": "user-guide/core-user-guide/components/command-line-code-executors.ipynb", "user-guide/core-user-guide/framework/model-clients.ipynb": "user-guide/core-user-guide/components/model-clients.ipynb", "user-guide/core-user-guide/framework/tools.ipynb": "user-guide/core-user-guide/components/tools.ipynb", + "user-guide/agentchat-user-guide/tutorial/custom-agents.ipynb": "user-guide/agentchat-user-guide/custom-agents.ipynb", } diff --git a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/custom-agents.ipynb b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/custom-agents.ipynb similarity index 85% rename from python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/custom-agents.ipynb rename to python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/custom-agents.ipynb index ba1a7caadd1c..ee7c987fe68a 100644 --- a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/custom-agents.ipynb +++ b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/custom-agents.ipynb @@ -294,12 +294,13 @@ "source": [ "## Using Custom Model Clients in Custom Agents\n", "\n", - "One of the key features of the AssistantAgent preset in AgentChat is that it takes a `model_client` argument and can use it in responding to messages. However, in some cases, you may want your agent to use a custom model client that is not currently supported (see [supported model clients](https://microsoft.github.io/autogen/dev/user-guide/core-user-guide/components/model-clients.html)). \n", + "One of the key features of the {py:class}`~autogen_agentchat.agents.AssistantAgent` preset in AgentChat is that it takes a `model_client` argument and can use it in responding to messages. However, in some cases, you may want your agent to use a custom model client that is not currently supported (see [supported model clients](https://microsoft.github.io/autogen/dev/user-guide/core-user-guide/components/model-clients.html)) or custom model behaviours. \n", "\n", - "You can accomplish this with a custom agent that implements your custom model client.\n", - "In the example below, we will walk through an example of a custom agent that uses the Google Gemini SDK directly to respond to messages.\n", + "You can accomplish this with a custom agent that implements *your custom model client*.\n", "\n", - "> **Note:** You will need to install the Google Gemini SDK to run this example. You can install it using the following command: \n", + "In the example below, we will walk through an example of a custom agent that uses the [Google Gemini SDK](https://github.com/googleapis/python-genai) directly to respond to messages.\n", + "\n", + "> **Note:** You will need to install the [Google Gemini SDK](https://github.com/googleapis/python-genai) to run this example. You can install it using the following command: \n", "\n", "```bash\n", "pip install google-genai\n", @@ -308,7 +309,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 37, "metadata": {}, "outputs": [], "source": [ @@ -347,10 +348,15 @@ " return (TextMessage,)\n", "\n", " async def on_messages(self, messages: Sequence[ChatMessage], cancellation_token: CancellationToken) -> Response:\n", + " final_response = None\n", " async for message in self.on_messages_stream(messages, cancellation_token):\n", " if isinstance(message, Response):\n", - " return message\n", - " raise AssertionError(\"The stream should have returned the final result.\")\n", + " final_response = message\n", + "\n", + " if final_response is None:\n", + " raise AssertionError(\"The stream should have returned the final result.\")\n", + "\n", + " return final_response\n", "\n", " async def on_messages_stream(\n", " self, messages: Sequence[ChatMessage], cancellation_token: CancellationToken\n", @@ -399,7 +405,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 38, "metadata": {}, "outputs": [ { @@ -420,7 +426,7 @@ "TaskResult(messages=[TextMessage(source='user', models_usage=None, content='What is the capital of New York?', type='TextMessage'), TextMessage(source='gemini_assistant', models_usage=RequestUsage(prompt_tokens=46, completion_tokens=5), content='Albany\\nTERMINATE\\n', type='TextMessage')], stop_reason=None)" ] }, - "execution_count": 29, + "execution_count": 38, "metadata": {}, "output_type": "execute_result" } @@ -441,7 +447,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 39, "metadata": {}, "outputs": [ { @@ -451,31 +457,31 @@ "---------- user ----------\n", "Write a Haiku poem with 4 lines about the fall season.\n", "---------- primary ----------\n", - "Leaves of crimson dance, \n", - "Whispers of the crisp cool air, \n", - "Pumpkins grin with light, \n", - "Nature's quilt laid bare.\n", + "Crimson leaves cascade, \n", + "Whispering winds sing of change, \n", + "Chill wraps the fading, \n", + "Nature's quilt, rich and warm.\n", "---------- gemini_critic ----------\n", - "The haiku only has three lines. The fourth line should be removed. Otherwise, it's a good poem.\n", + "The poem is good, but it has four lines instead of three. A haiku must have three lines with a 5-7-5 syllable structure. The content is evocative of autumn, but the form is incorrect. Please revise to adhere to the haiku's syllable structure.\n", "\n", "---------- primary ----------\n", - "Thank you for your feedback! Here’s the revised haiku with three lines:\n", + "Thank you for your feedback! Here’s a revised haiku that follows the 5-7-5 syllable structure:\n", "\n", - "Leaves of crimson dance, \n", - "Whispers of the crisp cool air, \n", - "Pumpkins grin with light.\n", + "Crimson leaves drift down, \n", + "Chill winds whisper through the gold, \n", + "Autumn’s breath is near.\n", "---------- gemini_critic ----------\n", - "APPROVE\n", + "The revised haiku is much improved. It correctly follows the 5-7-5 syllable structure and maintains the evocative imagery of autumn. APPROVE\n", "\n" ] }, { "data": { "text/plain": [ - "TaskResult(messages=[TextMessage(source='user', models_usage=None, content='Write a Haiku poem with 4 lines about the fall season.', type='TextMessage'), TextMessage(source='primary', models_usage=RequestUsage(prompt_tokens=33, completion_tokens=29), content=\"Leaves of crimson dance, \\nWhispers of the crisp cool air, \\nPumpkins grin with light, \\nNature's quilt laid bare.\", type='TextMessage'), TextMessage(source='gemini_critic', models_usage=RequestUsage(prompt_tokens=87, completion_tokens=26), content=\"The haiku only has three lines. The fourth line should be removed. Otherwise, it's a good poem.\\n\", type='TextMessage'), TextMessage(source='primary', models_usage=RequestUsage(prompt_tokens=100, completion_tokens=38), content='Thank you for your feedback! Here’s the revised haiku with three lines:\\n\\nLeaves of crimson dance, \\nWhispers of the crisp cool air, \\nPumpkins grin with light.', type='TextMessage'), TextMessage(source='gemini_critic', models_usage=RequestUsage(prompt_tokens=170, completion_tokens=3), content='APPROVE\\n', type='TextMessage')], stop_reason=\"Text 'APPROVE' mentioned\")" + "TaskResult(messages=[TextMessage(source='user', models_usage=None, content='Write a Haiku poem with 4 lines about the fall season.', type='TextMessage'), TextMessage(source='primary', models_usage=RequestUsage(prompt_tokens=33, completion_tokens=31), content=\"Crimson leaves cascade, \\nWhispering winds sing of change, \\nChill wraps the fading, \\nNature's quilt, rich and warm.\", type='TextMessage'), TextMessage(source='gemini_critic', models_usage=RequestUsage(prompt_tokens=86, completion_tokens=60), content=\"The poem is good, but it has four lines instead of three. A haiku must have three lines with a 5-7-5 syllable structure. The content is evocative of autumn, but the form is incorrect. Please revise to adhere to the haiku's syllable structure.\\n\", type='TextMessage'), TextMessage(source='primary', models_usage=RequestUsage(prompt_tokens=141, completion_tokens=49), content='Thank you for your feedback! Here’s a revised haiku that follows the 5-7-5 syllable structure:\\n\\nCrimson leaves drift down, \\nChill winds whisper through the gold, \\nAutumn’s breath is near.', type='TextMessage'), TextMessage(source='gemini_critic', models_usage=RequestUsage(prompt_tokens=211, completion_tokens=32), content='The revised haiku is much improved. It correctly follows the 5-7-5 syllable structure and maintains the evocative imagery of autumn. APPROVE\\n', type='TextMessage')], stop_reason=\"Text 'APPROVE' mentioned\")" ] }, - "execution_count": 30, + "execution_count": 39, "metadata": {}, "output_type": "execute_result" } @@ -524,7 +530,7 @@ "source": [ "## Making the Custom Agent Declarative \n", "\n", - "The Autogen framework provides a [Component](https://microsoft.github.io/autogen/dev/user-guide/core-user-guide/framework/component-config.html) interface for making the configuration of components serializable to a declarative format. This is useful for saving and loading configurations, and for sharing configurations with others. \n", + "Autogen provides a [Component](https://microsoft.github.io/autogen/dev/user-guide/core-user-guide/framework/component-config.html) interface for making the configuration of components serializable to a declarative format. This is useful for saving and loading configurations, and for sharing configurations with others. \n", "\n", "We accomplish this by inheriting from the `Component` class and implementing the `_from_config` and `_to_config` methods.\n", "The declarative class can be serialized to a JSON format using the `dump_component` method, and deserialized from a JSON format using the `load_component` method." @@ -532,7 +538,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 40, "metadata": {}, "outputs": [], "source": [ @@ -578,10 +584,15 @@ " return (TextMessage,)\n", "\n", " async def on_messages(self, messages: Sequence[ChatMessage], cancellation_token: CancellationToken) -> Response:\n", + " final_response = None\n", " async for message in self.on_messages_stream(messages, cancellation_token):\n", " if isinstance(message, Response):\n", - " return message\n", - " raise AssertionError(\"The stream should have returned the final result.\")\n", + " final_response = message\n", + "\n", + " if final_response is None:\n", + " raise AssertionError(\"The stream should have returned the final result.\")\n", + "\n", + " return final_response\n", "\n", " async def on_messages_stream(\n", " self, messages: Sequence[ChatMessage], cancellation_token: CancellationToken\n", @@ -649,12 +660,13 @@ "source": [ "Now that we have the required methods implemented, we can now load and dump the custom agent to and from a JSON format, and then load the agent from the JSON format.\n", " \n", - " > Note: You should set the `component_provider_override` class variable to the full path of the module containing the custom agent class e.g., (`mypackage.agents.GeminiAssistantAgent`). This is used by `load_component` method to determine how to instantiate the class." + " > Note: You should set the `component_provider_override` class variable to the full path of the module containing the custom agent class e.g., (`mypackage.agents.GeminiAssistantAgent`). This is used by `load_component` method to determine how to instantiate the class. \n", + " " ] }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 41, "metadata": {}, "outputs": [ { @@ -675,7 +687,7 @@ " \"system_message\": \"You are a helpful assistant that can respond to messages. Reply with TERMINATE when the task has been completed.\"\n", " }\n", "}\n", - "<__main__.GeminiAssistantAgent object at 0x11a9a4f10>\n" + "<__main__.GeminiAssistantAgent object at 0x11a5c5a90>\n" ] } ], @@ -686,6 +698,18 @@ "loaded_agent = GeminiAssistantAgent.load_component(config)\n", "print(loaded_agent)" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Next Steps \n", + "\n", + "So far, we have seen how to create custom agents, add custom model clients to agents, and make custom agents declarative. There are a few ways in which this basic sample can be extended:\n", + "\n", + "- Extend the Gemini model client to handle function calling similar to the {py:class}`~autogen_agentchat.agents.AssistantAgent` class. https://ai.google.dev/gemini-api/docs/function-calling \n", + "- Implement a package wit a custom agent and experiment with using it's declarative format in a tool like [AutoGen Studio](https://microsoft.github.io/autogen/dev/user-guide/autogenstudio-user-guide/index.html)." + ] } ], "metadata": { diff --git a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/index.md b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/index.md index aa3aee477d8b..b7d124f9e67e 100644 --- a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/index.md +++ b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/index.md @@ -111,7 +111,7 @@ tutorial/state :hidden: :caption: Advanced -tutorial/custom-agents +custom-agents selector-group-chat swarm magentic-one From 2243f91324fcfd75b8057b6489fe9aebac729899 Mon Sep 17 00:00:00 2001 From: Victor Dibia Date: Mon, 10 Feb 2025 15:13:06 -0800 Subject: [PATCH 07/10] add gemini deps and uv lock update --- python/packages/autogen-ext/pyproject.toml | 4 ++++ python/uv.lock | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/python/packages/autogen-ext/pyproject.toml b/python/packages/autogen-ext/pyproject.toml index f3e49b6ce992..298a60ff9017 100644 --- a/python/packages/autogen-ext/pyproject.toml +++ b/python/packages/autogen-ext/pyproject.toml @@ -70,6 +70,10 @@ semantic-kernel-core = [ "semantic-kernel>=1.17.1", ] +gemini = [ + "google-genai>=1.0.0", +] + semantic-kernel-google = [ "semantic-kernel[google]>=1.17.1", ] diff --git a/python/uv.lock b/python/uv.lock index b4b1e170df60..4d64c23cfb57 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -593,6 +593,9 @@ file-surfer = [ { name = "autogen-agentchat" }, { name = "markitdown" }, ] +gemini = [ + { name = "google-genai" }, +] graphrag = [ { name = "graphrag" }, ] @@ -696,6 +699,7 @@ requires-dist = [ { name = "diskcache", marker = "extra == 'diskcache'", specifier = ">=5.6.3" }, { name = "docker", marker = "extra == 'docker'", specifier = "~=7.0" }, { name = "ffmpeg-python", marker = "extra == 'video-surfer'" }, + { name = "google-genai", marker = "extra == 'gemini'", specifier = ">=1.0.0" }, { name = "graphrag", marker = "extra == 'graphrag'", specifier = ">=1.0.1" }, { name = "grpcio", marker = "extra == 'grpc'", specifier = "~=1.70.0" }, { name = "ipykernel", marker = "extra == 'jupyter-executor'", specifier = ">=6.29.5" }, @@ -2215,6 +2219,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/fb/54deefe679b7d1c1cc81d83396fcf28ad1a66d213bddeb275a8d28665918/google_crc32c-1.6.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18e311c64008f1f1379158158bb3f0c8d72635b9eb4f9545f8cf990c5668e59d", size = 27866 }, ] +[[package]] +name = "google-genai" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth" }, + { name = "pydantic" }, + { name = "requests" }, + { name = "websockets" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/6b/30/25443d2dec5fe4efd9f05440e13735ae68b19de2dea043b3b83a91a4e14b/google_genai-1.1.0-py3-none-any.whl", hash = "sha256:c48ac44612ad6aadc0bf96b12fa4314756baa16382c890fff793bcb53e9a9cc8", size = 130299 }, +] + [[package]] name = "google-generativeai" version = "0.8.4" From aab4268ede261f4f10bace8f5f18a7a9f490ac9c Mon Sep 17 00:00:00 2001 From: Victor Dibia Date: Mon, 10 Feb 2025 15:17:10 -0800 Subject: [PATCH 08/10] update migration guide with tutorial link --- .../agentchat-user-guide/migration-guide.md | 54 ++++++++++--------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/migration-guide.md b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/migration-guide.md index fdb3ca2152d2..a7987be74bf6 100644 --- a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/migration-guide.md +++ b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/migration-guide.md @@ -33,29 +33,35 @@ We provide a detailed guide on how to migrate your existing codebase from `v0.2` See each feature below for detailed information on how to migrate. -- [Model Client](#model-client) -- [Model Client for OpenAI-Compatible APIs](#model-client-for-openai-compatible-apis) -- [Model Client Cache](#model-client-cache) -- [Assistant Agent](#assistant-agent) -- [Multi-Modal Agent](#multi-modal-agent) -- [User Proxy](#user-proxy) -- [Conversable Agent and Register Reply](#conversable-agent-and-register-reply) -- [Save and Load Agent State](#save-and-load-agent-state) -- [Two-Agent Chat](#two-agent-chat) -- [Tool Use](#tool-use) -- [Chat Result](#chat-result) -- [Conversion between v0.2 and v0.4 Messages](#conversion-between-v02-and-v04-messages) -- [Group Chat](#group-chat) -- [Group Chat with Resume](#group-chat-with-resume) -- [Save and Load Group Chat State](#save-and-load-group-chat-state) -- [Group Chat with Tool Use](#group-chat-with-tool-use) -- [Group Chat with Custom Selector (Stateflow)](#group-chat-with-custom-selector-stateflow) -- [Nested Chat](#nested-chat) -- [Sequential Chat](#sequential-chat) -- [GPTAssistantAgent](#gptassistantagent) -- [Long-Context Handling](#long-context-handling) -- [Observability and Control](#observability-and-control) -- [Code Executors](#code-executors) +- [Migration Guide for v0.2 to v0.4](#migration-guide-for-v02-to-v04) + - [What is `v0.4`?](#what-is-v04) + - [New to AutoGen?](#new-to-autogen) + - [What's in this guide?](#whats-in-this-guide) + - [Model Client](#model-client) + - [Use component config](#use-component-config) + - [Use model client class directly](#use-model-client-class-directly) + - [Model Client for OpenAI-Compatible APIs](#model-client-for-openai-compatible-apis) + - [Model Client Cache](#model-client-cache) + - [Assistant Agent](#assistant-agent) + - [Multi-Modal Agent](#multi-modal-agent) + - [User Proxy](#user-proxy) + - [Conversable Agent and Register Reply](#conversable-agent-and-register-reply) + - [Save and Load Agent State](#save-and-load-agent-state) + - [Two-Agent Chat](#two-agent-chat) + - [Tool Use](#tool-use) + - [Chat Result](#chat-result) + - [Conversion between v0.2 and v0.4 Messages](#conversion-between-v02-and-v04-messages) + - [Group Chat](#group-chat) + - [Group Chat with Resume](#group-chat-with-resume) + - [Save and Load Group Chat State](#save-and-load-group-chat-state) + - [Group Chat with Tool Use](#group-chat-with-tool-use) + - [Group Chat with Custom Selector (Stateflow)](#group-chat-with-custom-selector-stateflow) + - [Nested Chat](#nested-chat) + - [Sequential Chat](#sequential-chat) + - [GPTAssistantAgent](#gptassistantagent) + - [Long Context Handling](#long-context-handling) + - [Observability and Control](#observability-and-control) + - [Code Executors](#code-executors) The following features currently in `v0.2` will be provided in the future releases of `v0.4.*` versions: @@ -417,7 +423,7 @@ class CustomAgent(BaseChatAgent): ``` You can then use the custom agent in the same way as the {py:class}`~autogen_agentchat.agents.AssistantAgent`. -See [Custom Agent Tutorial](./tutorial/custom-agents.ipynb) +See [Custom Agent Tutorial](./custom-agents.ipynb) for more details. ## Save and Load Agent State From 3bf5fb06a692be2d9405436bb33979b0db353e46 Mon Sep 17 00:00:00 2001 From: Victor Dibia Date: Mon, 10 Feb 2025 15:30:03 -0800 Subject: [PATCH 09/10] docs link refs updated --- .../src/user-guide/agentchat-user-guide/migration-guide.md | 2 +- .../user-guide/agentchat-user-guide/tutorial/messages.ipynb | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/migration-guide.md b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/migration-guide.md index a7987be74bf6..de9991609258 100644 --- a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/migration-guide.md +++ b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/migration-guide.md @@ -423,7 +423,7 @@ class CustomAgent(BaseChatAgent): ``` You can then use the custom agent in the same way as the {py:class}`~autogen_agentchat.agents.AssistantAgent`. -See [Custom Agent Tutorial](./custom-agents.ipynb) +See [Custom Agent Tutorial](custom-agents.ipynb) for more details. ## Save and Load Agent State diff --git a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/messages.ipynb b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/messages.ipynb index 25dc78641980..a3f5a25519dd 100644 --- a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/messages.ipynb +++ b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/messages.ipynb @@ -97,7 +97,7 @@ "\n", "Examples of these include {py:class}`~autogen_agentchat.messages.ToolCallRequestEvent`, which indicates that a request was made to call a tool, and {py:class}`~autogen_agentchat.messages.ToolCallExecutionEvent`, which contains the results of tool calls.\n", "\n", - "Typically, events are created by the agent itself and are contained in the {py:attr}`~autogen_agentchat.base.Response.inner_messages` field of the {py:class}`~autogen_agentchat.base.Response` returned from {py:class}`~autogen_agentchat.base.ChatAgent.on_messages`. If you are building a custom agent and have events that you want to communicate to other entities (e.g., a UI), you can include these in the {py:attr}`~autogen_agentchat.base.Response.inner_messages` field of the {py:class}`~autogen_agentchat.base.Response`. We will show examples of this in [Custom Agents](./custom-agents.ipynb).\n", + "Typically, events are created by the agent itself and are contained in the {py:attr}`~autogen_agentchat.base.Response.inner_messages` field of the {py:class}`~autogen_agentchat.base.Response` returned from {py:class}`~autogen_agentchat.base.ChatAgent.on_messages`. If you are building a custom agent and have events that you want to communicate to other entities (e.g., a UI), you can include these in the {py:attr}`~autogen_agentchat.base.Response.inner_messages` field of the {py:class}`~autogen_agentchat.base.Response`. We will show examples of this in [Custom Agents](../custom-agents.ipynb).\n", "\n", "\n", "You can read about the full set of messages supported in AgentChat in the {py:mod}`~autogen_agentchat.messages` module. \n", @@ -107,7 +107,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "agnext", "language": "python", "name": "python3" }, @@ -121,7 +121,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.6" + "version": "3.11.9" } }, "nbformat": 4, From 00c5560283f9734d9f8bb18bd1c9d78ffdfa078a Mon Sep 17 00:00:00 2001 From: Victor Dibia Date: Mon, 10 Feb 2025 15:59:10 -0800 Subject: [PATCH 10/10] link fixes --- .../src/user-guide/agentchat-user-guide/custom-agents.ipynb | 2 +- .../tools/semantic_kernel/_kernel_function_from_tool.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/custom-agents.ipynb b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/custom-agents.ipynb index ee7c987fe68a..d738e72b60f8 100644 --- a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/custom-agents.ipynb +++ b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/custom-agents.ipynb @@ -309,7 +309,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ diff --git a/python/packages/autogen-ext/src/autogen_ext/tools/semantic_kernel/_kernel_function_from_tool.py b/python/packages/autogen-ext/src/autogen_ext/tools/semantic_kernel/_kernel_function_from_tool.py index 8125aef9c5ce..0f3f93e2e7fa 100644 --- a/python/packages/autogen-ext/src/autogen_ext/tools/semantic_kernel/_kernel_function_from_tool.py +++ b/python/packages/autogen-ext/src/autogen_ext/tools/semantic_kernel/_kernel_function_from_tool.py @@ -3,8 +3,8 @@ from autogen_core import CancellationToken from autogen_core.tools import BaseTool from pydantic import BaseModel -from semantic_kernel.functions.kernel_parameter_metadata import KernelParameterMetadata from semantic_kernel.functions import KernelFunctionFromMethod, kernel_function +from semantic_kernel.functions.kernel_parameter_metadata import KernelParameterMetadata InputT = TypeVar("InputT", bound=BaseModel) OutputT = TypeVar("OutputT", bound=BaseModel)