Dev (experimental)

scripts/define_routes.py

@@ -10,6 +10,7 @@
     a2c_pong,
     trpo_pong,
     qrdqn_pong,
+    ars_fb,
     ppo_fb,
     trpo_fb
 )
@@ -19,25 +20,30 @@ def define_routes() -> List[Tuple[str, Type, dict]]:
     routes = []
     pong_routes = [
         (r"/ws/pong/pong-bot/", PongBot),
-        (r"/ws/pong/pong-a2c/", AiHandler, dict(
-            agent=PongAgent(a2c_pong, 3)
-        )),
+        # # DO NOT UNCOMMENT
+        # (r"/ws/pong/pong-a2c/", AiHandler, dict(
+        #     agent=PongAgent(a2c_pong, 3)
+        # )),
         (r"/ws/pong/pong-dqn/", AiHandler, dict(
             agent=PongAgent(dqn_pong, 3)
         )),
         (r"/ws/pong/pong-ppo/", AiHandler, dict(
             agent=PongAgent(ppo_pong, 3)
         )),
-        (r"/ws/pong/pong-qrdqn/", AiHandler, dict(
-            agent=PongAgent(qrdqn_pong, 3)
-        )),
+        # # DO NOT UNCOMMENT
+        # (r"/ws/pong/pong-qrdqn/", AiHandler, dict(
+        #     agent=PongAgent(qrdqn_pong, 3)
+        # )),
         (r"/ws/pong/pong-trpo/", AiHandler, dict(
             agent=PongAgent(trpo_pong, 3)
         ))
     ]
 
     flappybird_routes = [
         (r"/ws/flappybird/flappybird-bot/", FlappybirdBot),
+        (r"/ws/flappybird/flappybird-ars/", AiHandler, dict(
+            agent=FlappyBirdAgent(ars_fb, 1)
+        )),
         (r"/ws/flappybird/flappybird-ppo/", AiHandler, dict(
             agent=FlappyBirdAgent(ppo_fb, 3)
         )),

scripts/load_models.py

@@ -1,7 +1,7 @@
 from stable_baselines3 import DQN, PPO, A2C
-from sb3_contrib import TRPO, QRDQN
+from sb3_contrib import ARS, TRPO, QRDQN
 from scripts.paths.pong_paths import dqn_pong_path, ppo_pong_path, a2c_pong_path, trpo_pong_path, qrdqn_pong_path
-from scripts.paths.flappy_bird_paths import ppo_fb_path, trpo_fb_path
+from scripts.paths.flappy_bird_paths import ars_fb_path, ppo_fb_path, trpo_fb_path
 
 # Pong
 dqn_pong = DQN.load(path=dqn_pong_path)
@@ -12,5 +12,6 @@
 
 
 # Flappy Bird
+ars_fb = ARS.load(path=ars_fb_path)
 ppo_fb = PPO.load(path=ppo_fb_path)
 trpo_fb = TRPO.load(path=trpo_fb_path)

scripts/paths/algos_paths.py

@@ -5,3 +5,4 @@
 a2c_path = os.path.join('ready-models', 'a2c')
 trpo_path = os.path.join('ready-models', 'trpo')
 qrdqn_path = os.path.join('ready-models', 'qrdqn')
+ars_path = os.path.join('ready-models', 'ars')

scripts/paths/flappy_bird_paths.py

@@ -1,4 +1,4 @@
-from scripts.paths.algos_paths import ppo_path, trpo_path
+from scripts.paths.algos_paths import ars_path, ppo_path, trpo_path
 
 import os
 
@@ -8,5 +8,6 @@
     'WebsocketFlappyBird-v0_200000_steps.zip'
 )
 
+ars_fb_path = os.path.join(ars_path, env_path)
 ppo_fb_path = os.path.join(ppo_path, env_path)
 trpo_fb_path = os.path.join(trpo_path, env_path)

src/agents/web_env.py

@@ -11,6 +11,7 @@ def __init__(self, model: BaseAlgorithm, history_length: int = 1):
         if history_length < 1:
             raise ValueError("history_length must be an integer greater than or equal to 1")
         self.model = model
+        self.history_length = history_length
         self.states = deque(maxlen=history_length)
 
     def start(self) -> bool:
@@ -22,12 +23,14 @@ def prepare_observation(self, data: dict) -> np.array:
 
     @final
     def state_stack(self, observation: np.array) -> np.array:
-        if len(self.states) == 0:
-            for _ in range(self.states.maxlen):
+        if self.history_length > 1:
+            if len(self.states) == 0:
+                for _ in range(self.states.maxlen):
+                    self.states.append(observation)
+            else:
                 self.states.append(observation)
-        else:
-            self.states.append(observation)
-        return np.array(self.states).flatten()
+            return np.array(self.states).flatten()
+        return observation
 
     @abstractmethod
     def return_prediction(self, data: dict) -> dict:

src/agents/web_pong.py

@@ -7,8 +7,11 @@
 class PongAgent(WebsocketAgent):
     def __init__(self, model: BaseAlgorithm, history_length: int):
         super().__init__(model, history_length)
-        self.min_values = np.array([0, 0, 0, 0, -100, -100], dtype=np.float32)
-        self.max_values = np.array([600, 600, 1000, 600, 100, 100], dtype=np.float32)
+        # # For opponent observation
+        # self.min_values = np.array([0, 0, 0, 0, -100, -100], dtype=np.float32)
+        # self.max_values = np.array([600, 600, 1000, 600, 100, 100], dtype=np.float32)
+        self.min_values = np.array([0, 0, 0, -100, -100], dtype=np.float32)
+        self.max_values = np.array([600, 1000, 600, 100, 100], dtype=np.float32)
         self.action_map = {0: -1, 1: 0, 2: 1}
 
     def prepare_observation(self, data: dict) -> np.array:
@@ -17,7 +20,7 @@ def prepare_observation(self, data: dict) -> np.array:
         if player == 0:
             curr_observation = np.array([
                 state['leftPaddleY'],
-                state['rightPaddleY'],
+                # state['rightPaddleY'], # Opponent
                 state['ballX'],
                 state['ballY'],
                 state['ballSpeedX'],
@@ -26,18 +29,18 @@ def prepare_observation(self, data: dict) -> np.array:
         else:
             curr_observation = np.array([
                 state['rightPaddleY'],
-                state['leftPaddleY'],
+                # state['leftPaddleY'],  # Opponent
                 1000 - state['ballX'],
                 state['ballY'],
                 -state['ballSpeedX'],
                 state['ballSpeedY'],
             ], dtype=np.float32)
 
-        curr_observation[:4] = ((curr_observation[:4] - self.min_values[:4]) /
-                                (self.max_values[:4] - self.min_values[:4]))
+        curr_observation[:3] = ((curr_observation[:3] - self.min_values[:3]) /
+                                (self.max_values[:3] - self.min_values[:3]))
 
-        curr_observation[4:] = 2 * ((curr_observation[4:] - self.min_values[4:]) /
-                                    (self.max_values[4:] - self.min_values[4:])) - 1
+        curr_observation[3:] = 2 * ((curr_observation[3:] - self.min_values[3:]) /
+                                    (self.max_values[3:] - self.min_values[3:])) - 1
 
         return self.state_stack(curr_observation)
 

src/api.py

@@ -76,16 +76,16 @@ def on_close(self):
         self.after_close()
 
     def after_close(self):
-        raise NotImplementedError
+        pass
 
     @final
-    def on_message(self, message):  # do not override
+    async def on_message(self, message):  # do not override
         if self.last_message_time is None or time.time() - self.last_message_time >= 0.045:
             self.last_message_time = time.time()
-            self.send_message(message)
+            await self.send_message(message)
 
     @abstractmethod
-    def send_message(self, message):
+    async def send_message(self, message):
         raise NotImplementedError
 
 

src/bots.py

@@ -4,7 +4,7 @@
 
 
 class PongBot(BaseHandler):
-    def send_message(self, message):
+    async def send_message(self, message):
         data = json.loads(message)
         player = data['playerId']
         state = data['state']
@@ -20,11 +20,11 @@ def send_message(self, message):
             else:
                 move = -1
 
-        self.write_message(json.dumps({'move': move, 'start': 1}))
+        await self.write_message(json.dumps({'move': move, 'start': 1}))
 
 
 class FlappybirdBot(BaseHandler):
-    def send_message(self, message):
+    async def send_message(self, message):
         state = json.loads(message)['state']
         jump = 0
 
@@ -38,11 +38,11 @@ def send_message(self, message):
             if state['birdY'] > nearest_obstacle['centerGapY']:
                 jump = 1
 
-        self.write_message(json.dumps({'jump': jump}))
+        await self.write_message(json.dumps({'jump': jump}))
 
 
 class SkijumpBot(BaseHandler):
-    def send_message(self, message):
+    async def send_message(self, message):
         state = json.loads(message)['state']
 
         space = 0
@@ -58,4 +58,4 @@ def send_message(self, message):
         if state['jumperInclineRad'] < 0.7:
             up = 1
 
-        self.write_message(json.dumps({'space': space, 'up': up, 'down': down}))
+        await self.write_message(json.dumps({'space': space, 'up': up, 'down': down}))

src/handlers.py

@@ -18,7 +18,7 @@ def initialize(
     def after_close(self):
         self.agent.states.clear()
 
-    def send_message(self, message):
+    async def send_message(self, message):
         data = json.loads(message)
         action = self.agent.return_prediction(data)
-        self.write_message(json.dumps(action))
+        await self.write_message(json.dumps(action))