Merge remote-tracking branch 'upstream/main' into dotnet_unit_rt

Author: bassmang

python/README.md

@@ -50,12 +50,24 @@ To create a pull request (PR), ensure the following checks are met. You can run
 - Pyright: `poe pyright`
 - Build docs: `poe --directory ./packages/autogen-core/ docs-build`
 - Auto rebuild+serve docs: `poe --directory ./packages/autogen-core/ docs-serve`
+- Check samples in `python/samples`: `poe samples-code-check`
 Alternatively, you can run all the checks with:
 - `poe check`
 
 > [!NOTE]
 > These need to be run in the virtual environment.
 
+### Syncing Dependencies
+
+When you pull new changes, you may need to update the dependencies.
+To do so, first make sure you are in the virtual environment, and then in the `python` directory, run:
+
+```sh
+uv sync --all-extras
+```
+
+This will update the dependencies in the virtual environment.
+
 ### Creating a New Package
 
 To create a new package, similar to `autogen-core` or `autogen-chat`, use the following:

python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py

@@ -42,13 +42,13 @@
     HandoffMessage,
     MemoryQueryEvent,
     ModelClientStreamingChunkEvent,
-    MultiModalMessage,
     TextMessage,
     ToolCallExecutionEvent,
     ToolCallRequestEvent,
     ToolCallSummaryMessage,
 )
 from ..state import AssistantAgentState
+from ..utils import remove_images
 from ._base_chat_agent import BaseChatAgent
 
 event_logger = logging.getLogger(EVENT_LOGGER_NAME)
@@ -64,7 +64,7 @@ class AssistantAgentConfig(BaseModel):
     model_context: ComponentModel | None = None
     description: str
     system_message: str | None = None
-    model_client_stream: bool
+    model_client_stream: bool = False
     reflect_on_tool_use: bool
     tool_call_summary_format: str
 
@@ -375,8 +375,6 @@ async def on_messages_stream(
     ) -> AsyncGenerator[AgentEvent | ChatMessage | Response, None]:
         # Add messages to the model context.
         for msg in messages:
-            if isinstance(msg, MultiModalMessage) and self._model_client.model_info["vision"] is False:
-                raise ValueError("The model does not support vision.")
             if isinstance(msg, HandoffMessage):
                 # Add handoff context to the model context.
                 for context_msg in msg.context:
@@ -398,7 +396,7 @@ async def on_messages_stream(
                     yield memory_query_event_msg
 
         # Generate an inference result based on the current model context.
-        llm_messages = self._system_messages + await self._model_context.get_messages()
+        llm_messages = self._get_compatible_context(self._system_messages + await self._model_context.get_messages())
         model_result: CreateResult | None = None
         if self._model_client_stream:
             # Stream the model client.
@@ -494,7 +492,9 @@ async def on_messages_stream(
 
         if self._reflect_on_tool_use:
             # Generate another inference result based on the tool call and result.
-            llm_messages = self._system_messages + await self._model_context.get_messages()
+            llm_messages = self._get_compatible_context(
+                self._system_messages + await self._model_context.get_messages()
+            )
             reflection_model_result: CreateResult | None = None
             if self._model_client_stream:
                 # Stream the model client.
@@ -575,6 +575,13 @@ async def load_state(self, state: Mapping[str, Any]) -> None:
         # Load the model context state.
         await self._model_context.load_state(assistant_agent_state.llm_context)
 
+    def _get_compatible_context(self, messages: List[LLMMessage]) -> Sequence[LLMMessage]:
+        """Ensure that the messages are compatible with the underlying client, by removing images if needed."""
+        if self._model_client.model_info["vision"]:
+            return messages
+        else:
+            return remove_images(messages)
+
     def _to_config(self) -> AssistantAgentConfig:
         """Convert the assistant agent to a declarative config."""
 

python/packages/autogen-agentchat/src/autogen_agentchat/teams/_group_chat/_magentic_one/_magentic_one_orchestrator.py

@@ -2,7 +2,7 @@
 import logging
 from typing import Any, Dict, List, Mapping
 
-from autogen_core import AgentId, CancellationToken, DefaultTopicId, Image, MessageContext, event, rpc
+from autogen_core import AgentId, CancellationToken, DefaultTopicId, MessageContext, event, rpc
 from autogen_core.models import (
     AssistantMessage,
     ChatCompletionClient,
@@ -24,6 +24,7 @@
     ToolCallSummaryMessage,
 )
 from ....state import MagenticOneOrchestratorState
+from ....utils import content_to_str, remove_images
 from .._base_group_chat_manager import BaseGroupChatManager
 from .._events import (
     GroupChatAgentResponse,
@@ -138,15 +139,17 @@ async def handle_start(self, message: GroupChatStart, ctx: MessageContext) -> No
         # Create the initial task ledger
         #################################
         # Combine all message contents for task
-        self._task = " ".join([self._content_to_str(msg.content) for msg in message.messages])
+        self._task = " ".join([content_to_str(msg.content) for msg in message.messages])
         planning_conversation: List[LLMMessage] = []
 
         # 1. GATHER FACTS
         # create a closed book task and generate a response and update the chat history
         planning_conversation.append(
             UserMessage(content=self._get_task_ledger_facts_prompt(self._task), source=self._name)
         )
-        response = await self._model_client.create(planning_conversation, cancellation_token=ctx.cancellation_token)
+        response = await self._model_client.create(
+            self._get_compatible_context(planning_conversation), cancellation_token=ctx.cancellation_token
+        )
 
         assert isinstance(response.content, str)
         self._facts = response.content
@@ -157,7 +160,9 @@ async def handle_start(self, message: GroupChatStart, ctx: MessageContext) -> No
         planning_conversation.append(
             UserMessage(content=self._get_task_ledger_plan_prompt(self._team_description), source=self._name)
         )
-        response = await self._model_client.create(planning_conversation, cancellation_token=ctx.cancellation_token)
+        response = await self._model_client.create(
+            self._get_compatible_context(planning_conversation), cancellation_token=ctx.cancellation_token
+        )
 
         assert isinstance(response.content, str)
         self._plan = response.content
@@ -281,7 +286,7 @@ async def _orchestrate_step(self, cancellation_token: CancellationToken) -> None
         assert self._max_json_retries > 0
         key_error: bool = False
         for _ in range(self._max_json_retries):
-            response = await self._model_client.create(context, json_output=True)
+            response = await self._model_client.create(self._get_compatible_context(context), json_output=True)
             ledger_str = response.content
             try:
                 assert isinstance(ledger_str, str)
@@ -397,7 +402,9 @@ async def _update_task_ledger(self, cancellation_token: CancellationToken) -> No
         update_facts_prompt = self._get_task_ledger_facts_update_prompt(self._task, self._facts)
         context.append(UserMessage(content=update_facts_prompt, source=self._name))
 
-        response = await self._model_client.create(context, cancellation_token=cancellation_token)
+        response = await self._model_client.create(
+            self._get_compatible_context(context), cancellation_token=cancellation_token
+        )
 
         assert isinstance(response.content, str)
         self._facts = response.content
@@ -407,7 +414,9 @@ async def _update_task_ledger(self, cancellation_token: CancellationToken) -> No
         update_plan_prompt = self._get_task_ledger_plan_update_prompt(self._team_description)
         context.append(UserMessage(content=update_plan_prompt, source=self._name))
 
-        response = await self._model_client.create(context, cancellation_token=cancellation_token)
+        response = await self._model_client.create(
+            self._get_compatible_context(context), cancellation_token=cancellation_token
+        )
 
         assert isinstance(response.content, str)
         self._plan = response.content
@@ -420,7 +429,9 @@ async def _prepare_final_answer(self, reason: str, cancellation_token: Cancellat
         final_answer_prompt = self._get_final_answer_prompt(self._task)
         context.append(UserMessage(content=final_answer_prompt, source=self._name))
 
-        response = await self._model_client.create(context, cancellation_token=cancellation_token)
+        response = await self._model_client.create(
+            self._get_compatible_context(context), cancellation_token=cancellation_token
+        )
         assert isinstance(response.content, str)
         message = TextMessage(content=response.content, source=self._name)
 
@@ -464,15 +475,9 @@ def _thread_to_context(self) -> List[LLMMessage]:
                 context.append(UserMessage(content=m.content, source=m.source))
         return context
 
-    def _content_to_str(self, content: str | List[str | Image]) -> str:
-        """Convert the content to a string."""
-        if isinstance(content, str):
-            return content
+    def _get_compatible_context(self, messages: List[LLMMessage]) -> List[LLMMessage]:
+        """Ensure that the messages are compatible with the underlying client, by removing images if needed."""
+        if self._model_client.model_info["vision"]:
+            return messages
         else:
-            result: List[str] = []
-            for c in content:
-                if isinstance(c, str):
-                    result.append(c)
-                else:
-                    result.append("<image>")
-        return "\n".join(result)
+            return remove_images(messages)

python/packages/autogen-agentchat/src/autogen_agentchat/utils/__init__.py

@@ -0,0 +1,7 @@
+"""
+This module implements various utilities common to AgentChat agents and teams.
+"""
+
+from ._utils import content_to_str, remove_images
+
+__all__ = ["content_to_str", "remove_images"]

python/packages/autogen-agentchat/src/autogen_agentchat/utils/_utils.py

@@ -0,0 +1,38 @@
+from typing import List, Union
+
+from autogen_core import FunctionCall, Image
+from autogen_core.models import FunctionExecutionResult, LLMMessage, UserMessage
+
+# Type aliases for convenience
+_UserContent = Union[str, List[Union[str, Image]]]
+_AssistantContent = Union[str, List[FunctionCall]]
+_FunctionExecutionContent = List[FunctionExecutionResult]
+_SystemContent = str
+
+
+def content_to_str(content: _UserContent | _AssistantContent | _FunctionExecutionContent | _SystemContent) -> str:
+    """Convert the content of an LLMMessage to a string."""
+    if isinstance(content, str):
+        return content
+    else:
+        result: List[str] = []
+        for c in content:
+            if isinstance(c, str):
+                result.append(c)
+            elif isinstance(c, Image):
+                result.append("<image>")
+            else:
+                result.append(str(c))
+
+    return "\n".join(result)
+
+
+def remove_images(messages: List[LLMMessage]) -> List[LLMMessage]:
+    """Remove images from a list of LLMMessages"""
+    str_messages: List[LLMMessage] = []
+    for message in messages:
+        if isinstance(message, UserMessage) and isinstance(message.content, list):
+            str_messages.append(UserMessage(content=content_to_str(message.content), source=message.source))
+        else:
+            str_messages.append(message)
+    return str_messages

python/packages/autogen-agentchat/tests/test_assistant_agent.py

@@ -21,7 +21,15 @@
 from autogen_core import FunctionCall, Image
 from autogen_core.memory import ListMemory, Memory, MemoryContent, MemoryMimeType, MemoryQueryResult
 from autogen_core.model_context import BufferedChatCompletionContext
-from autogen_core.models import CreateResult, FunctionExecutionResult, LLMMessage, RequestUsage
+from autogen_core.models import (
+    AssistantMessage,
+    CreateResult,
+    FunctionExecutionResult,
+    LLMMessage,
+    RequestUsage,
+    SystemMessage,
+    UserMessage,
+)
 from autogen_core.models._model_client import ModelFamily
 from autogen_core.tools import FunctionTool
 from autogen_ext.models.openai import OpenAIChatCompletionClient
@@ -541,15 +549,44 @@ async def test_invalid_model_capabilities() -> None:
                 FunctionTool(_echo_function, description="Echo"),
             ],
         )
+        await agent.run(task=TextMessage(source="user", content="Test"))
 
     with pytest.raises(ValueError):
         agent = AssistantAgent(name="assistant", model_client=model_client, handoffs=["agent2"])
+        await agent.run(task=TextMessage(source="user", content="Test"))
 
-    with pytest.raises(ValueError):
-        agent = AssistantAgent(name="assistant", model_client=model_client)
-        # Generate a random base64 image.
-        img_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4//8/AAX+Av4N70a4AAAAAElFTkSuQmCC"
-        await agent.run(task=MultiModalMessage(source="user", content=["Test", Image.from_base64(img_base64)]))
+
+@pytest.mark.asyncio
+async def test_remove_images(monkeypatch: pytest.MonkeyPatch) -> None:
+    model = "random-model"
+    model_client_1 = OpenAIChatCompletionClient(
+        model=model,
+        api_key="",
+        model_info={"vision": False, "function_calling": False, "json_output": False, "family": ModelFamily.UNKNOWN},
+    )
+    model_client_2 = OpenAIChatCompletionClient(
+        model=model,
+        api_key="",
+        model_info={"vision": True, "function_calling": False, "json_output": False, "family": ModelFamily.UNKNOWN},
+    )
+
+    img_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4//8/AAX+Av4N70a4AAAAAElFTkSuQmCC"
+    messages: List[LLMMessage] = [
+        SystemMessage(content="System.1"),
+        UserMessage(content=["User.1", Image.from_base64(img_base64)], source="user.1"),
+        AssistantMessage(content="Assistant.1", source="assistant.1"),
+        UserMessage(content="User.2", source="assistant.2"),
+    ]
+
+    agent_1 = AssistantAgent(name="assistant_1", model_client=model_client_1)
+    result = agent_1._get_compatible_context(messages)  # type: ignore
+    assert len(result) == 4
+    assert isinstance(result[1].content, str)
+
+    agent_2 = AssistantAgent(name="assistant_2", model_client=model_client_2)
+    result = agent_2._get_compatible_context(messages)  # type: ignore
+    assert len(result) == 4
+    assert isinstance(result[1].content, list)
 
 
 @pytest.mark.asyncio

python/packages/autogen-agentchat/tests/test_utils.py

@@ -0,0 +1,36 @@
+from typing import List
+
+import pytest
+from autogen_agentchat.utils import remove_images
+from autogen_core import Image
+from autogen_core.models import AssistantMessage, LLMMessage, SystemMessage, UserMessage
+
+
+@pytest.mark.asyncio
+async def test_remove_images() -> None:
+    img_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4//8/AAX+Av4N70a4AAAAAElFTkSuQmCC"
+    messages: List[LLMMessage] = [
+        SystemMessage(content="System.1"),
+        UserMessage(content=["User.1", Image.from_base64(img_base64)], source="user.1"),
+        AssistantMessage(content="Assistant.1", source="assistant.1"),
+        UserMessage(content="User.2", source="assistant.2"),
+    ]
+
+    result = remove_images(messages)
+
+    # Check all the invariants
+    assert len(result) == 4
+    assert isinstance(result[0], SystemMessage)
+    assert isinstance(result[1], UserMessage)
+    assert isinstance(result[2], AssistantMessage)
+    assert isinstance(result[3], UserMessage)
+    assert result[0].content == messages[0].content
+    assert result[2].content == messages[2].content
+    assert result[3].content == messages[3].content
+    assert isinstance(messages[2], AssistantMessage)
+    assert isinstance(messages[3], UserMessage)
+    assert result[2].source == messages[2].source
+    assert result[3].source == messages[3].source
+
+    # Check that the image was removed.
+    assert result[1].content == "User.1\n<image>"

python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/termination.ipynb

@@ -10,7 +10,7 @@
                 "\n",
                 "AgentChat supports several termination condition by providing a base {py:class}`~autogen_agentchat.base.TerminationCondition` class and several implementations that inherit from it.\n",
                 "\n",
-                "A termination condition is a callable that takes a sequece of {py:class}`~autogen_agentchat.messages.AgentEvent` or {py:class}`~autogen_agentchat.messages.ChatMessage` objects **since the last time the condition was called**, and returns a {py:class}`~autogen_agentchat.messages.StopMessage` if the conversation should be terminated, or `None` otherwise.\n",
+                "A termination condition is a callable that takes a sequence of {py:class}`~autogen_agentchat.messages.AgentEvent` or {py:class}`~autogen_agentchat.messages.ChatMessage` objects **since the last time the condition was called**, and returns a {py:class}`~autogen_agentchat.messages.StopMessage` if the conversation should be terminated, or `None` otherwise.\n",
                 "Once a termination condition has been reached, it must be reset by calling {py:meth}`~autogen_agentchat.base.TerminationCondition.reset` before it can be used again.\n",
                 "\n",
                 "Some important things to note about termination conditions: \n",

python/packages/autogen-core/src/autogen_core/_serialization.py

@@ -192,6 +192,13 @@ class UnknownPayload:
 
 
 def _type_name(cls: type[Any] | Any) -> str:
+    # If cls is a protobuf, then we need to determine the descriptor
+    if isinstance(cls, type):
+        if issubclass(cls, Message):
+            return cast(str, cls.DESCRIPTOR.full_name)
+    elif isinstance(cls, Message):
+        return cast(str, cls.DESCRIPTOR.full_name)
+
     if isinstance(cls, type):
         return cls.__name__
     else: