1. support concurrency; 2. add detailed exception handling

Author: HenryCai11

data/avalon/dev.json

@@ -14,6 +14,16 @@
         "quest_leader": 0,
         "role_names": ["Servant", "Assassin", "Merlin", "Servant", "Minion"]
     },
+    {
+        "num_players": 5,
+        "quest_leader": 0,
+        "role_names": ["Servant", "Servant", "Assassin", "Merlin", "Minion"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 0,
+        "role_names": ["Servant", "Minion", "Assassin", "Merlin", "Servant"]
+    },
     {
         "num_players": 5,
         "quest_leader": 0,
@@ -28,5 +38,65 @@
         "num_players": 5,
         "quest_leader": 0,
         "role_names": ["Assassin", "Servant", "Servant", "Minion", "Merlin"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 0,
+        "role_names": ["Assassin", "Merlin", "Servant", "Servant", "Minion"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 0,
+        "role_names": ["Assassin", "Minion", "Servant", "Servant", "Merlin"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 1,
+        "role_names": ["Servant", "Merlin", "Servant", "Assassin", "Minion"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 1,
+        "role_names": ["Servant", "Assassin", "Servant", "Merlin", "Minion"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 1,
+        "role_names": ["Servant", "Assassin", "Merlin", "Servant", "Minion"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 1,
+        "role_names": ["Servant", "Servant", "Assassin", "Merlin", "Minion"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 1,
+        "role_names": ["Servant", "Minion", "Assassin", "Merlin", "Servant"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 1,
+        "role_names": ["Assassin", "Servant", "Merlin", "Servant", "Minion"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 1,
+        "role_names": ["Assassin", "Servant", "Servant", "Merlin", "Minion"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 1,
+        "role_names": ["Assassin", "Servant", "Servant", "Minion", "Merlin"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 1,
+        "role_names": ["Assassin", "Merlin", "Servant", "Servant", "Minion"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 1,
+        "role_names": ["Assassin", "Minion", "Servant", "Servant", "Merlin"]
     }
 ]

src/server/tasks/avalon/agents/agent.py

@@ -13,7 +13,7 @@ class Agent:
         - :method:`Agent.vote_on_team`
         - :method:`Agent.vote_on_mission`
     """
-    def __init__(self, id: int, role: int, config: AvalonBasicConfig):
+    def __init__(self, id: int, role: int, config: AvalonBasicConfig) -> None:
         self.id = id
         self.name = f"Player {id}"
         self.role = role
@@ -77,6 +77,6 @@ def get_believed_sides(self, num_players: int) -> List[float]:
             num_players (int): The number of players in the game.
         
         Returns:
-            List[float]: The list of believed sides of all players.
+            List[float]: The list of believed sides (probability) of all players.
         """
         raise NotImplementedError

src/server/tasks/avalon/agents/baseline_agents.py

@@ -78,9 +78,6 @@ async def assassinate(self):
         return random.randint(0, self.config.num_players-1)
 
     async def get_believed_sides(self, **kwargs):
-        '''
-        returns a list of probability of each player being good, where the list is ordered by player id. if player side is known, probability is 0 or 1. otherwise, probability is 0.5
-        '''
         return [0.5 if side == -1 else side for side in self.player_sides]
 
 
@@ -221,7 +218,8 @@ def __init__(self, id: int, name: str, config: AvalonBasicConfig, side: int=1, r
         self.lexigraphic = lexigraphic
 
     def generate_possible_player_sides(self, sides: List[int], num_evils: int):
-        '''
+        '''Util func of NaiveServant
+        
         generates a list of all possible combinations of player sides given a list of known sides and unknown sides recursively as well number of unknown evils
         '''
         out = []

src/server/tasks/avalon/agents/llm_with_discussion.py

@@ -34,10 +34,8 @@ def __repr__(self):
     def see_sides(self, sides):
         self.player_sides = sides
 
-    async def initialize_game_info(self, player_list):
-        r"""Initiliaze the game info for the agent, which includes game introduction, role, and reveal information for different roles.
-
-        """
+    async def initialize_game_info(self, player_list) -> None:
+        """Initiliaze the game info for the agent, which includes game introduction, role, and reveal information for different roles."""
         # Introduction Prompt
         verbal_side = ["Evil", "Good"]
         intro_prompt = INTRODUCTION
@@ -89,7 +87,7 @@ async def initialize_game_info(self, player_list):
             "mode": "system",
         })
 
-    async def summarize(self):
+    async def summarize(self) -> None:
         summary = await self.session.action({
             "role": "user",
             "content": "Please summarize the history. Try to keep all useful information, including your identity, other player's identities, and your observations in the game.",
@@ -104,10 +102,10 @@ async def summarize(self):
         })
         print("History after summarization: ", self.session.get_history())
 
-    async def observe_mission(self, team, mission_id, num_fails, votes, outcome):
+    async def observe_mission(self, team, mission_id, num_fails, votes, outcome) -> None:
         pass
 
-    async def observe_team_result(self, mission_id, team: frozenset, votes: List[int], outcome: bool):
+    async def observe_team_result(self, mission_id, team: frozenset, votes: List[int], outcome: bool) -> None:
         await self.session.action({
             "role": "user",
             "content": verbalize_team_result(team, votes, outcome),
@@ -229,7 +227,6 @@ async def vote_on_mission(self, team, mission_id, discussion_history):
         if isinstance(vote_result, int):
             return vote_result
         else:
-            print(vote_result)
             raise ValueError(
                 "Vote result should be either 0 or 1, instead of {}.".format(type(vote_result))
             )

src/server/tasks/avalon/avalon_exception.py

@@ -0,0 +1,28 @@
+from typing import Union
+from src.typings import AgentBenchException
+
+class AvalonException(AgentBenchException):
+    def __init__(self, reason: str, detail: Union[str, None] = None) -> None:
+        super().__init__()
+        self.reason = reason
+        self.detail = detail
+
+    def __str__(self) -> str:
+        if not self.detail:
+            return "{CLASS_NAME}[{REASON}]".format(
+                CLASS_NAME=self.__class__.__name__, REASON=self.reason
+            )
+        else:
+            return "{CLASS_NAME}[{REASON}]: {DETAIL}".format(
+                CLASS_NAME=self.__class__.__name__,
+                REASON=self.reason,
+                DETAIL=self.detail,
+            )
+
+class AvalonEnvException(AvalonException):
+    def __init__(self, detail: Union[str, None] = None) -> None:
+        super().__init__("Avalon Environment Exception", detail)
+
+class AvalonAgentActionException(AvalonException):
+    def __init__(self, detail: Union[str, None] = None) -> None:
+        super().__init__("Invalid action (result) with retry", detail)

src/server/tasks/avalon/engine.py

@@ -1,6 +1,7 @@
 from typing import ClassVar, List, Optional, Dict
 import numpy as np
 from pydantic import BaseModel
+from .avalon_exception import AvalonEnvException
 
 class AvalonBasicConfig(BaseModel):
     r"""Avalon game configuration
@@ -279,22 +280,22 @@ def choose_quest_team(self, team: frozenset, leader):
         '''
         # check if game ended or not
         if self.done:
-            raise ValueError("Game ended")
+            raise AvalonEnvException("Game ended")
 
         # check if it is team selection phase. if not, raise error
         if self.phase != 0:
-            raise ValueError("Not in team selection phase")
+            raise AvalonEnvException("Not in team selection phase")
 
         # check if team size is valid
         # if np.sum(team) != self.num_players_for_quest[self.round]:
-        #     raise ValueError("Invalid team size")
+        #     raise AvalonEnvException("Invalid team size")
 
         if len(team) != self.num_players_for_quest[self.turn]:
-            raise ValueError("Invalid team size")
+            raise AvalonEnvException("Invalid team size")
 
         # check if leader is quest leader
         if leader != self.quest_leader:
-            raise ValueError("Not quest leader")
+            raise AvalonEnvException("Not quest leader")
 
         self.quest_team = team
 
@@ -317,15 +318,15 @@ def gather_team_votes(self, votes: List):
         '''
         # check if game ended or not
         if self.done:
-            raise ValueError("Game ended")
+            raise AvalonEnvException("Game ended")
 
         # check if it is team voting phase. if not, raise error
         if self.phase != 1:
-            raise ValueError("Not in team voting phase")
+            raise AvalonEnvException("Not in team voting phase")
 
         # check if votes is valid
         if len(votes) != self.num_players:
-            raise ValueError("Invalid number of votes")
+            raise AvalonEnvException("Invalid number of votes")
 
         self.team_votes = votes
 
@@ -358,15 +359,15 @@ def gather_quest_votes(self, votes: List):
         '''
         # check if game ended or not
         if self.done:
-            raise ValueError("Game ended")
+            raise AvalonEnvException("Game ended")
 
         # check if it is quest voting phase. if not, raise error
         if self.phase != 2:
-            raise ValueError("Not in quest voting phase")
+            raise AvalonEnvException("Not in quest voting phase")
 
         # check if votes is valid
         if len(votes) != self.num_players_for_quest[self.turn]:
-            raise ValueError("Invalid number of votes")
+            raise AvalonEnvException("Invalid number of votes")
 
         self.quest_votes = votes
 
@@ -412,19 +413,19 @@ def choose_assassination_target(self, player, target):
         '''
         # check if game ended or not
         if self.done:
-            raise ValueError("Game ended")
+            raise AvalonEnvException("Game ended")
 
         # check if it is assassination phase. if not, raise error
         if self.phase != 3:
-            raise ValueError("Not in assassination phase")
+            raise AvalonEnvException("Not in assassination phase")
 
         # check if player is assassin
         if self.roles[player] != 7:
-            raise ValueError("Not assassin")
+            raise AvalonEnvException("Not assassin")
 
         # check if player is good
         if self.is_good[player]:
-            raise ValueError("Assassin cannot be good")
+            raise AvalonEnvException("Assassin cannot be good")
 
         self.done = True