Skip to content

Commit 4ccff54

Browse files
cheng-tanafourneyvictordibiasonichi
authored
Logging (microsoft#1146)
* WIP:logging * serialize request, response and client * Fixed code formatting. * Updated to use a global package, and added some test cases. Still very-much a draft. * Update work in progress. * adding cost * log new agent * update log_completion test in test_agent_telemetry * tests * fix formatting * Added additional telemetry for wrappers and clients. * WIP: add test for oai client and oai wrapper table * update test_telemetry * fix format * More tests, update doc and clean up * small fix for session id - moved to start_logging and return from start_logging * update start_logging type to return str, add notebook to demonstrate use of telemetry * add ability to get log dataframe * precommit formatting fixes * formatting fix * Remove pandas dependency from telemetry and only use in notebook * formatting fixes * log query exceptions * fix formatting * fix ci * fix comment - add notebook link in doc and fix groupchat serialization * small fix * do not serialize Agent * formatting * wip * fix test * serialization bug fix for soc moderator * fix test and clean up * wip: add version table * fix test * fix test * fix test * make the logging interface more general and fix client model logging * fix format * fix formatting and tests * fix * fix comment * Renaming telemetry to logging * update notebook * update doc * formatting * formatting and clean up * fix doc * fix link and title * fix notebook format and fix comment * format * try fixing agent test and update migration guide * fix link * debug print * debug * format * add back tests * fix tests --------- Co-authored-by: Adam Fourney <[email protected]> Co-authored-by: Victor Dibia <[email protected]> Co-authored-by: Chi Wang <[email protected]>
1 parent f68c09b commit 4ccff54

16 files changed

+1519
-9
lines changed

‎autogen/agentchat/assistant_agent.py

+3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from typing import Callable, Dict, Literal, Optional, Union
22

33
from .conversable_agent import ConversableAgent
4+
from autogen.runtime_logging import logging_enabled, log_new_agent
45

56

67
class AssistantAgent(ConversableAgent):
@@ -67,6 +68,8 @@ def __init__(
6768
description=description,
6869
**kwargs,
6970
)
71+
if logging_enabled():
72+
log_new_agent(self, locals())
7073

7174
# Update the provided description if None, and we are using the default system_message,
7275
# then use the default description.

‎autogen/agentchat/conversable_agent.py

+4
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
from ..coding.factory import CodeExecutorFactory
1515

1616
from ..oai.client import OpenAIWrapper, ModelClient
17+
from ..runtime_logging import logging_enabled, log_new_agent
1718
from ..cache.cache import Cache
1819
from ..code_utils import (
1920
UNKNOWN,
@@ -145,6 +146,9 @@ def __init__(
145146
self.llm_config.update(llm_config)
146147
self.client = OpenAIWrapper(**self.llm_config)
147148

149+
if logging_enabled():
150+
log_new_agent(self, locals())
151+
148152
# Initialize standalone client cache object.
149153
self.client_cache = None
150154

‎autogen/agentchat/groupchat.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@
99
from ..code_utils import content_str
1010
from .agent import Agent
1111
from .conversable_agent import ConversableAgent
12+
from ..runtime_logging import logging_enabled, log_new_agent
1213
from ..graph_utils import check_graph_validity, invert_disallowed_to_allowed
1314

14-
1515
logger = logging.getLogger(__name__)
1616

1717

@@ -463,6 +463,8 @@ def __init__(
463463
system_message=system_message,
464464
**kwargs,
465465
)
466+
if logging_enabled():
467+
log_new_agent(self, locals())
466468
# Store groupchat
467469
self._groupchat = groupchat
468470

‎autogen/agentchat/user_proxy_agent.py

+4
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from typing import Callable, Dict, List, Literal, Optional, Union
22

33
from .conversable_agent import ConversableAgent
4+
from ..runtime_logging import logging_enabled, log_new_agent
45

56

67
class UserProxyAgent(ConversableAgent):
@@ -93,3 +94,6 @@ def __init__(
9394
if description is not None
9495
else self.DEFAULT_USER_PROXY_AGENT_DESCRIPTIONS[human_input_mode],
9596
)
97+
98+
if logging_enabled():
99+
log_new_agent(self, locals())

‎autogen/logger/__init__.py

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
from .logger_factory import LoggerFactory
2+
from .sqlite_logger import SqliteLogger
3+
4+
__all__ = ("LoggerFactory", "SqliteLogger")

‎autogen/logger/base_logger.py

+101
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
from __future__ import annotations
2+
3+
from abc import ABC, abstractmethod
4+
from typing import Dict, TYPE_CHECKING, Union
5+
import sqlite3
6+
import uuid
7+
8+
from openai import OpenAI, AzureOpenAI
9+
from openai.types.chat import ChatCompletion
10+
11+
if TYPE_CHECKING:
12+
from autogen import ConversableAgent, OpenAIWrapper
13+
14+
15+
class BaseLogger(ABC):
16+
@abstractmethod
17+
def start(self) -> str:
18+
"""
19+
Open a connection to the logging database, and start recording.
20+
21+
Returns:
22+
session_id (str): a unique id for the logging session
23+
"""
24+
...
25+
26+
@abstractmethod
27+
def log_chat_completion(
28+
invocation_id: uuid.UUID,
29+
client_id: int,
30+
wrapper_id: int,
31+
request: Dict,
32+
response: Union[str, ChatCompletion],
33+
is_cached: int,
34+
cost: float,
35+
start_time: str,
36+
) -> None:
37+
"""
38+
Log a chat completion to database.
39+
40+
In AutoGen, chat completions are somewhat complicated because they are handled by the `autogen.oai.OpenAIWrapper` class.
41+
One invocation to `create` can lead to multiple underlying OpenAI calls, depending on the llm_config list used, and
42+
any errors or retries.
43+
44+
Args:
45+
invocation_id (uuid): A unique identifier for the invocation to the OpenAIWrapper.create method call
46+
client_id (int): A unique identifier for the underlying OpenAI client instance
47+
wrapper_id (int): A unique identifier for the OpenAIWrapper instance
48+
request (dict): A dictionary representing the the request or call to the OpenAI client endpoint
49+
response (str or ChatCompletion): The response from OpenAI
50+
is_chached (int): 1 if the response was a cache hit, 0 otherwise
51+
cost(float): The cost for OpenAI response
52+
start_time (str): A string representing the moment the request was initiated
53+
"""
54+
...
55+
56+
@abstractmethod
57+
def log_new_agent(agent: ConversableAgent, init_args: Dict) -> None:
58+
"""
59+
Log the birth of a new agent.
60+
61+
Args:
62+
agent (ConversableAgent): The agent to log.
63+
init_args (dict): The arguments passed to the construct the conversable agent
64+
"""
65+
...
66+
67+
@abstractmethod
68+
def log_new_wrapper(wrapper: OpenAIWrapper, init_args: Dict) -> None:
69+
"""
70+
Log the birth of a new OpenAIWrapper.
71+
72+
Args:
73+
wrapper (OpenAIWrapper): The wrapper to log.
74+
init_args (dict): The arguments passed to the construct the wrapper
75+
"""
76+
...
77+
78+
@abstractmethod
79+
def log_new_client(client: Union[AzureOpenAI, OpenAI], wrapper: OpenAIWrapper, init_args: Dict) -> None:
80+
"""
81+
Log the birth of a new OpenAIWrapper.
82+
83+
Args:
84+
wrapper (OpenAI): The OpenAI client to log.
85+
init_args (dict): The arguments passed to the construct the client
86+
"""
87+
...
88+
89+
@abstractmethod
90+
def stop() -> None:
91+
"""
92+
Close the connection to the logging database, and stop logging.
93+
"""
94+
...
95+
96+
@abstractmethod
97+
def get_connection() -> Union[sqlite3.Connection]:
98+
"""
99+
Return a connection to the logging database.
100+
"""
101+
...

‎autogen/logger/logger_factory.py

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
from typing import Any, Dict, Optional
2+
from autogen.logger.base_logger import BaseLogger
3+
from autogen.logger.sqlite_logger import SqliteLogger
4+
5+
__all__ = ("LoggerFactory",)
6+
7+
8+
class LoggerFactory:
9+
@staticmethod
10+
def get_logger(logger_type: str = "sqlite", config: Optional[Dict[str, Any]] = None) -> BaseLogger:
11+
if config is None:
12+
config = {}
13+
14+
if logger_type == "sqlite":
15+
return SqliteLogger(config)
16+
else:
17+
raise ValueError(f"[logger_factory] Unknown logger type: {logger_type}")

‎autogen/logger/logger_utils.py

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
import datetime
2+
import inspect
3+
from typing import Any, Dict, List, Tuple, Union
4+
5+
__all__ = ("get_current_ts", "to_dict")
6+
7+
8+
def get_current_ts():
9+
return datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S.%f")
10+
11+
12+
def to_dict(
13+
obj: Union[int, float, str, bool, Dict[Any, Any], List[Any], Tuple[Any, ...], Any],
14+
exclude: Tuple[str] = (),
15+
no_recursive: Tuple[str] = (),
16+
) -> Any:
17+
if isinstance(obj, (int, float, str, bool)):
18+
return obj
19+
elif callable(obj):
20+
return inspect.getsource(obj).strip()
21+
elif isinstance(obj, dict):
22+
return {
23+
str(k): to_dict(str(v)) if isinstance(v, no_recursive) else to_dict(v, exclude, no_recursive)
24+
for k, v in obj.items()
25+
if k not in exclude
26+
}
27+
elif isinstance(obj, (list, tuple)):
28+
return [to_dict(str(v)) if isinstance(v, no_recursive) else to_dict(v, exclude, no_recursive) for v in obj]
29+
elif hasattr(obj, "__dict__"):
30+
return {
31+
str(k): to_dict(str(v)) if isinstance(v, no_recursive) else to_dict(v, exclude, no_recursive)
32+
for k, v in vars(obj).items()
33+
if k not in exclude
34+
}
35+
else:
36+
return obj

0 commit comments

Comments
 (0)