diff --git a/config.yaml b/config.yaml deleted file mode 100644 index 682386b..0000000 --- a/config.yaml +++ /dev/null @@ -1,52 +0,0 @@ -name: new_beginnings -project: ConnectX -entity: filipinogambino -group: debug -use_mixed_precision: false -total_steps: 1e4 -batch_size: 8 -checkpoint_freq: 60.0 -num_actors: 2 -n_actor_envs: 16 -unroll_length: 16 -player_id: 0 -seed: 42 -model_arch: mha_model -embedding_dim: 32 -hidden_dim: 128 -n_heads: 4 -n_blocks: 1 -device: cpu -rescale_value_input: false -obs_space_kwargs: {} -reward_space_kwargs: {} -debug: true -act_space: BasicActionSpace -obs_space: BasicObsSpace -reward_space: LongGameReward -optimizer_class: Adam -optimizer_kwargs: - lr: 0.0001 - eps: 0.0003 -min_lr_mod: 0.01 -entropy_cost: 0.001 -baseline_cost: 1.0 -teacher_kl_cost: 0.0 -lmb: 0.8 -reduction: sum -actor_device: cpu -learner_device: cpu -disable_wandb: false -model_log_freq: 100 -sharing_strategy: file_descriptor -num_buffers: 4 -discounting: 0.999 -clip_grads: 10.0 -num_learner_threads: 1 -use_teacher: false -teacher_baseline_cost: 0.0 -use_index_select: true -load_dir: null -checkpoint_file: null -weights_only: false -n_value_warmup_batches: 0 diff --git a/connectx/connectx_gym/connectx_env.py b/connectx/connectx_gym/connectx_env.py index 7b56d07..e9c4190 100644 --- a/connectx/connectx_gym/connectx_env.py +++ b/connectx/connectx_gym/connectx_env.py @@ -10,6 +10,7 @@ from ..utility_constants import BOARD_SIZE + class ConnectFour(gym.Env): metadata = {'render_modes': ['human']} spec = None diff --git a/connectx/connectx_gym/wrappers.py b/connectx/connectx_gym/wrappers.py index 47556ec..1271196 100644 --- a/connectx/connectx_gym/wrappers.py +++ b/connectx/connectx_gym/wrappers.py @@ -19,11 +19,9 @@ def info(self, info: Dict[str, np.ndarray], rewards: int) -> Dict[str, np.ndarra logs = dict(step=self.env.unwrapped.turn) self.reward_sum.append(rewards) - logs["reward_length"] = [len(self.reward_sum)] logs["mean_cumulative_rewards"] = [np.mean(self.reward_sum)] logs["mean_cumulative_reward_magnitudes"] = [np.mean(np.abs(self.reward_sum))] logs["max_cumulative_rewards"] = [np.max(self.reward_sum)] - logs["rewards"] = self.reward_sum info.update({f"LOGGING_{key}": np.array(val, dtype=np.float32) for key, val in logs.items()}) @@ -47,23 +45,16 @@ def __init__(self, env: gym.Env, reward_space: BaseRewardSpace): self.reward_space = reward_space def _get_rewards_and_done(self) -> Tuple[Tuple[float, float], bool]: - # for key,val in self.unwrapped.env.dict().items(): - # print(key, val) - rewards, done = self.reward_space.compute_rewards(self.unwrapped.env) - if self.unwrapped.env.done and not done: - raise RuntimeError("Reward space did not return done, but the connectx engine is done.") + rewards, done = self.reward_space.compute_rewards(self.unwrapped) return rewards, done def reset(self, **kwargs): - obs, reward, done, info = super(RewardSpaceWrapper, self).reset(**kwargs) - # return obs, *self._get_rewards_and_done(), info - return obs, reward, done, info + obs, _, _, info = super(RewardSpaceWrapper, self).reset(**kwargs) + return obs, *self._get_rewards_and_done(), info def step(self, action): - # obs, _, _, info = super(RewardSpaceWrapper, self).step(action) - # return obs, *self._get_rewards_and_done(), info - obs, reward, done, info = super(RewardSpaceWrapper, self).step(action) - return obs, reward, done, info + obs, _, _, info = super(RewardSpaceWrapper, self).step(action) + return obs, *self._get_rewards_and_done(), info class VecEnv(gym.Env): def __init__(self, envs: List[gym.Env]): diff --git a/outputs/02-01/16-34-37/.hydra/config.yaml b/outputs/02-01/16-34-37/.hydra/config.yaml deleted file mode 100644 index f180e4a..0000000 --- a/outputs/02-01/16-34-37/.hydra/config.yaml +++ /dev/null @@ -1,41 +0,0 @@ -name: new_beginnings -project: ConnectX -entity: filipinogambino -group: debug -use_mixed_precision: false -total_steps: 10000.0 -batch_size: 8 -checkpoint_freq: 60.0 -num_actors: 1 -n_actor_envs: 2 -unroll_length: 42 -player_id: 0 -seed: 42 -model_arch: mha_model -embedding_dim: 32 -hidden_dim: 128 -n_heads: 4 -n_blocks: 1 -device: cpu -rescale_value_input: false -obs_space_kwargs: {} -reward_space_kwargs: {} -debug: true -act_space: BasicActionSpace -obs_space: BasicObsSpace -reward_space: GameResultReward -optimizer_class: Adam -optimizer_kwargs: - lr: 0.0001 - eps: 0.0003 -min_lr_mod: 0.01 -entropy_cost: 0.001 -baseline_cost: 1.0 -teacher_kl_cost: 0.0 -lmb: 0.8 -reduction: sum -actor_device: cpu -learner_device: cpu -disable_wandb: true -model_log_freq: 100 -sharing_strategy: file_descriptor diff --git a/outputs/02-01/16-34-37/.hydra/hydra.yaml b/outputs/02-01/16-34-37/.hydra/hydra.yaml deleted file mode 100644 index 034a138..0000000 --- a/outputs/02-01/16-34-37/.hydra/hydra.yaml +++ /dev/null @@ -1,154 +0,0 @@ -hydra: - run: - dir: ./outputs/${now:%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: [] - job: - name: run_monobeast - chdir: null - override_dirname: '' - id: ??? - num: ??? - config_name: new_beginnings - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: C:\Users\nick.gorichs\PycharmProjects\Connect_Four_2 - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: C:\Users\nick.gorichs\PycharmProjects\Connect_Four_2\conf - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: C:\Users\nick.gorichs\PycharmProjects\Connect_Four_2\outputs\02-01\16-34-37 - choices: - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/02-01/16-34-37/.hydra/overrides.yaml b/outputs/02-01/16-34-37/.hydra/overrides.yaml deleted file mode 100644 index fe51488..0000000 --- a/outputs/02-01/16-34-37/.hydra/overrides.yaml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/outputs/02-02/09-55-53/.hydra/config.yaml b/outputs/02-02/09-55-53/.hydra/config.yaml deleted file mode 100644 index 5403098..0000000 --- a/outputs/02-02/09-55-53/.hydra/config.yaml +++ /dev/null @@ -1,41 +0,0 @@ -name: new_beginnings -project: ConnectX -entity: filipinogambino -group: debug -use_mixed_precision: false -total_steps: 10000.0 -batch_size: 8 -checkpoint_freq: 60.0 -num_actors: 1 -n_actor_envs: 2 -unroll_length: 42 -player_id: 0 -seed: 42 -model_arch: mha_model -embedding_dim: 32 -hidden_dim: 128 -n_heads: 4 -n_blocks: 1 -device: cpu -rescale_value_input: false -obs_space_kwargs: {} -reward_space_kwargs: {} -debug: true -act_space: BasicActionSpace -obs_space: BasicObsSpace -reward_space: GameResultReward -optimizer_class: Adam -optimizer_kwargs: - lr: 0.0001 - eps: 0.0003 -min_lr_mod: 0.01 -entropy_cost: 0.001 -baseline_cost: 1.0 -teacher_kl_cost: 0.0 -lmb: 0.8 -reduction: sum -actor_device: cpu -learner_device: cpu -disable_wandb: false -model_log_freq: 100 -sharing_strategy: file_descriptor diff --git a/outputs/02-02/09-55-53/.hydra/hydra.yaml b/outputs/02-02/09-55-53/.hydra/hydra.yaml deleted file mode 100644 index ed40638..0000000 --- a/outputs/02-02/09-55-53/.hydra/hydra.yaml +++ /dev/null @@ -1,154 +0,0 @@ -hydra: - run: - dir: ./outputs/${now:%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: [] - job: - name: run_monobeast - chdir: null - override_dirname: '' - id: ??? - num: ??? - config_name: new_beginnings - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: C:\Users\nick.gorichs\PycharmProjects\Connect_Four_2 - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: C:\Users\nick.gorichs\PycharmProjects\Connect_Four_2\conf - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: C:\Users\nick.gorichs\PycharmProjects\Connect_Four_2\outputs\02-02\09-55-53 - choices: - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/02-02/09-55-53/.hydra/overrides.yaml b/outputs/02-02/09-55-53/.hydra/overrides.yaml deleted file mode 100644 index fe51488..0000000 --- a/outputs/02-02/09-55-53/.hydra/overrides.yaml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/wandb/run-20240202_095555-eb1n06sq/files/config.yaml b/wandb/run-20240202_095555-eb1n06sq/files/config.yaml deleted file mode 100644 index 646d9e7..0000000 --- a/wandb/run-20240202_095555-eb1n06sq/files/config.yaml +++ /dev/null @@ -1,183 +0,0 @@ -wandb_version: 1 - -name: - desc: null - value: new_beginnings -project: - desc: null - value: ConnectX -entity: - desc: null - value: filipinogambino -group: - desc: null - value: debug -use_mixed_precision: - desc: null - value: false -total_steps: - desc: null - value: 10000.0 -batch_size: - desc: null - value: 8 -checkpoint_freq: - desc: null - value: 60.0 -num_actors: - desc: null - value: 1 -n_actor_envs: - desc: null - value: 2 -unroll_length: - desc: null - value: 42 -player_id: - desc: null - value: 0 -seed: - desc: null - value: 42 -model_arch: - desc: null - value: mha_model -embedding_dim: - desc: null - value: 32 -hidden_dim: - desc: null - value: 128 -n_heads: - desc: null - value: 4 -n_blocks: - desc: null - value: 1 -device: - desc: null - value: cpu -rescale_value_input: - desc: null - value: false -obs_space_kwargs: - desc: null - value: {} -reward_space_kwargs: - desc: null - value: {} -debug: - desc: null - value: true -act_space: - desc: null - value: connectx.connectx_gym.act_spaces.BasicActionSpace -obs_space: - desc: null - value: connectx.connectx_gym.obs_spaces.BasicObsSpace -reward_space: - desc: null - value: connectx.connectx_gym.reward_spaces.GameResultReward -optimizer_class: - desc: null - value: torch.optim.adam.Adam -optimizer_kwargs: - desc: null - value: - lr: 0.0001 - eps: 0.0003 -min_lr_mod: - desc: null - value: 0.01 -entropy_cost: - desc: null - value: 0.001 -baseline_cost: - desc: null - value: 1.0 -teacher_kl_cost: - desc: null - value: 0.0 -lmb: - desc: null - value: 0.8 -reduction: - desc: null - value: sum -actor_device: - desc: null - value: cpu -learner_device: - desc: null - value: cpu -disable_wandb: - desc: null - value: false -model_log_freq: - desc: null - value: 100 -sharing_strategy: - desc: null - value: file_descriptor -num_buffers: - desc: null - value: 4 -discounting: - desc: null - value: 0.999 -clip_grads: - desc: null - value: 10.0 -num_learner_threads: - desc: null - value: 1 -use_teacher: - desc: null - value: false -teacher_baseline_cost: - desc: null - value: 0.0 -use_index_select: - desc: null - value: true -load_dir: - desc: null - value: null -checkpoint_file: - desc: null - value: null -weights_only: - desc: null - value: false -n_value_warmup_batches: - desc: null - value: 0 -_wandb: - desc: null - value: - python_version: 3.9.6 - cli_version: 0.16.2 - framework: huggingface - huggingface_version: 4.35.0 - is_jupyter_run: false - is_kaggle_kernel: true - start_time: 1706889355.617884 - t: - 1: - - 1 - - 11 - - 49 - - 50 - - 55 - 3: - - 13 - - 16 - - 23 - 4: 3.9.6 - 5: 0.16.2 - 6: 4.35.0 - 8: - - 2 - - 3 - - 5 - 13: windows-amd64 diff --git a/wandb/run-20240202_095555-eb1n06sq/files/requirements.txt b/wandb/run-20240202_095555-eb1n06sq/files/requirements.txt deleted file mode 100644 index cd284c9..0000000 --- a/wandb/run-20240202_095555-eb1n06sq/files/requirements.txt +++ /dev/null @@ -1,120 +0,0 @@ -absl-py==2.0.0 -antlr4-python3-runtime==4.9.3 -appdirs==1.4.4 -asttokens==2.4.1 -astunparse==1.6.3 -attrs==23.1.0 -beautifulsoup4==4.12.2 -blinker==1.7.0 -cachetools==5.3.2 -certifi==2023.7.22 -charset-normalizer==3.3.2 -click==8.1.7 -cloudpickle==3.0.0 -colorama==0.4.6 -contourpy==1.2.0 -cycler==0.12.1 -decorator==5.1.1 -docker-pycreds==0.4.0 -exceptiongroup==1.1.3 -executing==2.0.1 -farama-notifications==0.0.4 -filelock==3.13.1 -flask==3.0.0 -flatbuffers==23.5.26 -fonttools==4.44.0 -fsspec==2023.10.0 -gast==0.5.4 -gitdb==4.0.11 -gitpython==3.1.40 -google-auth-oauthlib==1.0.0 -google-auth==2.23.4 -google-pasta==0.2.0 -grpcio==1.59.2 -gym-notices==0.0.8 -gym==0.26.2 -gymnasium==0.29.0 -h5py==3.10.0 -huggingface-hub==0.17.3 -hydra-core==1.3.2 -idna==3.4 -importlib-metadata==6.8.0 -importlib-resources==6.1.0 -ipython==8.17.2 -itsdangerous==2.1.2 -jedi==0.19.1 -jinja2==3.1.2 -jsonschema-specifications==2023.7.1 -jsonschema==4.19.2 -keras==2.14.0 -kiwisolver==1.4.5 -libclang==16.0.6 -markdown==3.5.1 -markupsafe==2.1.3 -matplotlib-inline==0.1.6 -matplotlib==3.8.1 -ml-dtypes==0.2.0 -mpmath==1.3.0 -networkx==3.2.1 -numpy==1.26.3 -oauthlib==3.2.2 -omegaconf==2.3.0 -opt-einsum==3.3.0 -packaging==23.2 -pandas==2.1.2 -parso==0.8.3 -pettingzoo==1.24.0 -pillow==10.1.0 -pip==23.3.2 -prompt-toolkit==3.0.41 -protobuf==4.25.0 -psutil==5.9.7 -pure-eval==0.2.2 -pyasn1-modules==0.3.0 -pyasn1==0.5.0 -pygame==2.5.2 -pygments==2.16.1 -pyparsing==3.1.1 -python-dateutil==2.8.2 -pytz==2023.3.post1 -pyyaml==6.0.1 -referencing==0.30.2 -regex==2023.10.3 -requests-oauthlib==1.3.1 -requests==2.31.0 -rpds-py==0.12.0 -rsa==4.9 -safetensors==0.4.0 -scipy==1.11.3 -sentry-sdk==1.39.1 -setproctitle==1.3.3 -setuptools==68.2.2 -shimmy==1.3.0 -six==1.16.0 -smmap==5.0.1 -soupsieve==2.5 -stable-baselines3==2.1.0 -stack-data==0.6.3 -sympy==1.12 -tensorboard-data-server==0.7.2 -tensorboard==2.14.1 -tensorflow-estimator==2.14.0 -tensorflow-intel==2.14.0 -tensorflow-io-gcs-filesystem==0.31.0 -tensorflow==2.14.0 -termcolor==2.3.0 -tokenizers==0.14.1 -torch==2.1.2 -torchvision==0.16.2 -tqdm==4.66.1 -traitlets==5.13.0 -transformers==4.35.0 -typing-extensions==4.8.0 -tzdata==2023.3 -urllib3==2.0.7 -wandb==0.16.2 -wcwidth==0.2.10 -werkzeug==3.0.1 -wheel==0.37.1 -wrapt==1.14.1 -zipp==3.17.0 \ No newline at end of file diff --git a/wandb/run-20240202_095555-eb1n06sq/files/wandb-metadata.json b/wandb/run-20240202_095555-eb1n06sq/files/wandb-metadata.json deleted file mode 100644 index dd948e8..0000000 --- a/wandb/run-20240202_095555-eb1n06sq/files/wandb-metadata.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "os": "Windows-10-10.0.22631-SP0", - "python": "3.9.6", - "heartbeatAt": "2024-02-02T15:55:56.165904", - "startedAt": "2024-02-02T15:55:55.602843", - "docker": null, - "cuda": null, - "args": [], - "state": "running", - "program": "C:\\Users\\nick.gorichs\\PycharmProjects\\Connect_Four_2\\run_monobeast.py", - "codePathLocal": "run_monobeast.py", - "codePath": "run_monobeast.py", - "git": { - "remote": "https://github.com/FilipinoGambino/Connect_Four_2.git", - "commit": "add8931ea6b76c64496fa2399e74eb2483625422" - }, - "email": null, - "root": "C:/Users/nick.gorichs/PycharmProjects/Connect_Four_2", - "host": "WI-7C7X4S2", - "username": "Nick.Gorichs", - "executable": "C:\\Users\\nick.gorichs\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", - "cpu_count": 4, - "cpu_count_logical": 8, - "cpu_freq": { - "current": 1910.0, - "min": 0.0, - "max": 2112.0 - }, - "cpu_freq_per_core": [ - { - "current": 1910.0, - "min": 0.0, - "max": 2112.0 - } - ], - "disk": { - "/": { - "total": 236.73046493530273, - "used": 221.68227767944336 - } - }, - "memory": { - "total": 7.861743927001953 - } -} diff --git a/wandb/run-20240202_095555-eb1n06sq/run-eb1n06sq.wandb b/wandb/run-20240202_095555-eb1n06sq/run-eb1n06sq.wandb deleted file mode 100644 index 49f9619..0000000 Binary files a/wandb/run-20240202_095555-eb1n06sq/run-eb1n06sq.wandb and /dev/null differ