fix tournament CLI tool (v1.1.5)

ChangeLog.md

@@ -1,5 +1,8 @@
 # ChangeLog
 
+### v1.1.5
+
+- Fix bugs related to the tournament CLI tool where it required to save replays and didn't run Win/Loss/Tie ranking systems
 ### v1.1.4
 
 - Fix bug where lichen could grow to tiles adjacent to factories

luxai_runner/cli.py

@@ -5,7 +5,7 @@
 import numpy as np
 import json
 from luxai_runner.episode import Episode, EpisodeConfig, ReplayConfig
-from luxai_runner.tournament import Tournament
+from luxai_runner.tournament import Tournament, TournamentConfig
 from luxai_runner.logger import Logger
 from omegaconf import OmegaConf
 import sys
@@ -35,6 +35,7 @@ def main():
 
     parser.add_argument("--tournament", help="Turn tournament mode on", action="store_true", default=False)
     parser.add_argument("--tournament_cfg.concurrent", help="Max concurrent number of episodes to run. Recommended to set no higher than the number of CPUs / 2", type=int, default=1)
+    parser.add_argument("--tournament_cfg.ranking_system", help="The ranking system to use. Default is 'elo'. Can be 'elo', 'wins'.", type=str, default="elo")
     parser.add_argument("--skip_validate_action_space", help="Set this for a small performance increase. Note that turning this on means the engine assumes your submitted actions are valid. If your actions are not well formatted there could be errors", action="store_true", default=False)
 
 
@@ -73,7 +74,16 @@ def main():
                     agents.append(agent_file)
             print(f"Found {len(agents)} in {args.players[0]}")
             args.players = agents
-        tourney = Tournament(tournament_config_kwargs=dict(agents=args.players, max_concurrent_episodes=getattr(args, "tournament_cfg.concurrent")), episode_cfg=cfg)
+
+        tournament_config = TournamentConfig()
+        tournament_config.agents = args.players
+        # TODO - in future replace this with OmegaConf or something that can parse these nicely
+        tournament_config.max_concurrent_episodes = getattr(args, "tournament_cfg.concurrent")
+        tournament_config.ranking_system = getattr(args, "tournament_cfg.ranking_system")
+        tourney = Tournament(
+            cfg=tournament_config,
+            episode_cfg=cfg # the base/default episode config
+        )
         # import ipdb;ipdb.set_trace()
         asyncio.run(tourney.run())
         # exit()

luxai_runner/tournament/rankingsystem.py

@@ -13,7 +13,10 @@ def init_rank_state(self) -> Rank:
         pass
     def update(self, rank_1: Rank, rank_2: Rank, rank_1_score: float, rank_2_score: float):
         pass
-
+    def _rank_headers(self) -> str:
+        pass
+    def _rank_info(self, rank: Rank) -> str:
+        pass
 
 @dataclass
 class ELORank(Rank):
@@ -46,29 +49,52 @@ def _expected_score(self, rank_1: ELORank, rank_2: ELORank):
         """
         return 1 / (1 + np.power(10, (rank_2.rating - rank_1.rating) / 400))
 
+    def _rank_headers(self) -> str:
+        return f"{'Rating':8.8}"
+    def _rank_info(self, rank: ELORank) -> str:
+        return f"{str(rank.rating):8.8}"
+
 @dataclass
 class WinLossRank(Rank):
     rating: int
+    wins: int
+    ties: int
+    losses: int
     episodes: int
 
 class WinLoss(RankingSystem):
-    def __init__(self, win_points=3,tie_points=1,loss_points=0) -> None:
+    def __init__(self, win_points=3,tie_points=1,loss_points=0,col_length=7) -> None:
         super().__init__()
+        self.col_length=col_length
         self.win_points=win_points
         self.tie_points=tie_points
         self.loss_points=loss_points
     def init_rank_state(self) -> Rank:
-        return ELORank(rating=0, episodes=0)
-    def update(self, rank_1: ELORank, rank_2: ELORank, rank_1_score, rank_2_score):
+        return WinLossRank(rating=0, episodes=0, wins=0,ties=0,losses=0)
+    def update(self, rank_1: WinLossRank, rank_2: WinLossRank, rank_1_score, rank_2_score):
         # Only implements win/loss/ties
-        if rank_1_score > rank_2_score:
-            rank_1.rating = rank_1.rating + self.win_points
-            rank_2.rating = rank_2.rating + self.loss_points
-        elif rank_1_score < rank_2_score:
-            rank_1.rating = rank_1.rating + self.loss_points
-            rank_2.rating = rank_2.rating + self.win_points
-        else:
+        if rank_1_score == rank_2_score:
+            winner = None
+            loser = None
+            rank_1.ties += 1
+            rank_2.ties += 1
             rank_1.rating = rank_1.rating + self.tie_points
             rank_2.rating = rank_2.rating + self.tie_points
+        else:
+            winner = rank_1
+            loser = rank_2
+            if rank_1_score < rank_2_score:
+                winner = rank_2
+                loser = rank_1
+            # not a tie
+            winner.wins += 1
+            loser.losses += 1
+            winner.rating = winner.rating + self.win_points
+            loser.rating = loser.rating + self.loss_points
+
         rank_1.episodes += 1
-        rank_2.episodes += 1
+        rank_2.episodes += 1
+    def _rank_headers(self) -> str:
+        return f"{'Score':{self.col_length}} | {'Wins':{self.col_length}} | {'Ties':{self.col_length}} | {'Losses':{self.col_length}}"
+    def _rank_info(self, rank: WinLossRank) -> str:
+        return f"{str(rank.rating):{self.col_length}.2} | {str(rank.wins):{self.col_length}.2} | {str(rank.ties):{self.col_length}.2} | {str(rank.losses):{self.col_length}.2}"

luxai_runner/tournament/tournament.py

@@ -3,7 +3,7 @@
 from typing import Dict
 from luxai_runner.episode import Episode, EpisodeConfig
 from luxai_runner.tournament.config import TournamentConfig
-from luxai_runner.tournament.rankingsystem import ELO, Rank
+from luxai_runner.tournament.rankingsystem import ELO, Rank, WinLoss, WinLossRank
 from luxai_runner.tournament import matchmaking
 import copy
 import os.path as osp
@@ -14,17 +14,23 @@ def __init__(self, player_id, file) -> None:
         self.rank: Rank = None
 
 class Tournament:
-    def __init__(self, tournament_config_kwargs = dict(), episode_cfg: EpisodeConfig = None):
+    def __init__(self, cfg: TournamentConfig, episode_cfg: EpisodeConfig = None):
         self.global_id = 0
         self.episode_id = 0
-        self.cfg = TournamentConfig(**tournament_config_kwargs)
+        self.cfg = cfg
         self.eps_cfg = episode_cfg
         min_agents = min(self.cfg.agents_per_episode)
         assert len(self.cfg.agents) >= min_agents
         assert episode_cfg is not None
 
         # init ranking system
-        self.ranking_sys = ELO(K=30,init_rating=1000)
+        if cfg.ranking_system.lower() == "elo":
+            # TODO - allow configuration via CLI
+            self.ranking_sys = ELO(K=30,init_rating=1000)
+        elif cfg.ranking_system.lower() == "wins":
+            self.ranking_sys = WinLoss(win_points=3, tie_points=1, loss_points=0)
+        else:
+            raise Exception(f"'{cfg.ranking_system}' is not a valid ranking system")
 
         self.players: Dict[str, Player] = dict()
         for agent in self.cfg.agents:
@@ -47,8 +53,10 @@ async def _run_episode_cb(a):
             if a in episodes: episodes.discard(a)
             next_players = self.match_making_sys.next_match()
             eps_cfg = copy.deepcopy(self.eps_cfg)
-            save_replay_path_split = osp.splitext(eps_cfg.save_replay_path)
-            eps_cfg.save_replay_path = f"{save_replay_path_split[0]}_{self.episode_id}{save_replay_path_split[1]}"
+            if self.eps_cfg.save_replay_path is not None:
+                # if there is a replay path template, use it and generate a replay path for this episode
+                save_replay_path_split = osp.splitext(eps_cfg.save_replay_path)
+                eps_cfg.save_replay_path = f"{save_replay_path_split[0]}_{self.episode_id}{save_replay_path_split[1]}"
             eps_cfg.players = [self.players[p].file for p in next_players]
             self.episode_id += 1
 
@@ -62,6 +70,8 @@ async def _run_episode_cb(a):
             await _run_episode_cb(task)
         async def print_results():
             import time
+            line_length = 50
+            line_length += len(self.ranking_sys._rank_headers())
             while True:
                 import sys
                 import time
@@ -71,14 +81,14 @@ async def print_results():
 
                 lines.append(f"==== {self.cfg.name} ====")
                 # lines.append("")
-                lines.append(f"{'Player':36.36}| {'Rating':8.8}| {'Episodes':14.14}")
-                lines.append("-"*62)
+                lines.append(f"{'Player':36.36}| {self.ranking_sys._rank_headers()}| {'Episodes':14.14}")
+                lines.append("-"*line_length)
                 players_sorted = [self.players[p] for p in self.players]
                 players_sorted = sorted(players_sorted, key=lambda p : p.rank.rating, reverse=True)
                 for p in players_sorted:
                     rank = p.rank
-                    lines.append(f"{p.id:36.36}| {str(rank.rating):8.8}| {str(rank.episodes):14.14}")
-                lines.append("-"*62)
+                    lines.append(f"{p.id:36.36}| {self.ranking_sys._rank_info(rank)}| {str(rank.episodes):14.14}")
+                lines.append("-"*line_length)
                 lines.append(f"{len(episodes)} episodes are running")
 
 

setup.py

@@ -9,7 +9,7 @@ def read(fname):
 
 setup(
     name="luxai2022",
-    version="1.1.4",
+    version="1.1.5",
     author="Lux AI Challenge",
     description="The Lux AI Challenge Season 2",
     license="MIT",