-
Notifications
You must be signed in to change notification settings - Fork 7
/
Copy pathddpg_move_to_a_point_action_loss_weight.py
158 lines (146 loc) · 6.07 KB
/
ddpg_move_to_a_point_action_loss_weight.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
"""###IMP dummy_seed should always be last in the order in the OrderedDict below!!!
"""
import itertools
from ray import tune
from collections import OrderedDict
num_seeds = 10
var_env_configs = OrderedDict(
{
"state_space_dim": [2], # , 10, 12, 14] # [2**i for i in range(1,6)]
"action_space_dim": [2], # 2, 4, 8, 16] # [2**i for i in range(1,6)]
# "state_space_relevant_indices": [[0, 1]],
# "action_space_relevant_indices": [[0, 1]],
"delay": [0], # + [2**i for i in range(4)],
# 'sequence_length': [1], #, 2, 3, 4],#i for i in range(1,4)]
# 'reward_density': [0.25], # np.linspace(0.0, 1.0, num=5)
"make_denser": [True],
# 'terminal_state_density': [0.25], # np.linspace(0.1, 1.0, num=5)
"transition_noise": [0],
"reward_noise": [0], # , 1, 5, 10, 25] # Std dev. of normal dist.
"target_point": [[0, 0]],
"target_radius": [0.5],
"state_space_max": [10],
"action_space_max": [1],
"action_loss_weight": [0.01, 0.1, 1, 5, 10, 100],
"time_unit": [1.0],
"transition_dynamics_order": [1],
"dummy_seed": [i for i in range(num_seeds)],
}
)
var_configs = OrderedDict({"env": var_env_configs})
env_config = {
"env": "RLToy-v0",
"horizon": 100,
"env_config": {
"seed": 0, # seed
"state_space_type": "continuous",
"action_space_type": "continuous",
"inertia": 1,
"reward_scale": 1.0,
"reward_function": "move_to_a_point",
# 'make_denser': True,
# "log_level": 'INFO',
"log_filename": "/tmp/ddpg_mv_pt.log",
},
}
algorithm = "DDPG"
agent_config = {
# Learning rate for the critic (Q-function) optimizer.
"critic_lr": 1e-3,
# Learning rate for the actor (policy) optimizer.
"actor_lr": 1e-3,
# Update the target by \tau * policy + (1-\tau) * target_policy
"tau": 0.02,
# How many steps of the model to sample before learning starts.
"learning_starts": 2000,
# Postprocess the policy network model output with these hidden layers. If
# use_state_preprocessor is False, then these will be the *only* hidden
# layers in the network.
"actor_hiddens": [32, 32],
# Postprocess the critic network model output with these hidden layers;
# again, if use_state_preprocessor is True, then the state will be
# preprocessed by the model specified with the "model" config option first.
"critic_hiddens": [32, 32],
# Apply a state preprocessor with spec given by the "model" config option
# (like other RL algorithms). This is mostly useful if you have a weird
# observation shape, like an image. Disabled by default.
"use_state_preprocessor": False,
# Hidden layers activation of the postprocessing stage of the policy
# network
"actor_hidden_activation": "relu",
# Hidden layers activation of the postprocessing state of the critic.
"critic_hidden_activation": "relu",
# N-step Q learning
"n_step": 1,
# Update the target network every `target_network_update_freq` steps.
"target_network_update_freq": 0,
# If True, use huber loss instead of squared loss for critic network
# Conventionally, no need to clip gradients if using a huber loss
"use_huber": False,
# Threshold of a huber loss
"huber_threshold": 1.0,
# Weights for L2 regularization
"l2_reg": 1e-6,
# If not None, clip gradients during optimization at this value
"grad_norm_clipping": None,
"buffer_size": 50000,
# If True prioritized replay buffer will be used.
"prioritized_replay": False,
# Alpha parameter for prioritized replay buffer.
"prioritized_replay_alpha": 0.6,
# Beta parameter for sampling from prioritized replay buffer.
"prioritized_replay_beta": 0.4,
# Time steps over which the beta parameter is annealed.
# "prioritized_replay_beta_annealing_timesteps": 20000,
# Final value of beta
"final_prioritized_replay_beta": 0.4,
# Epsilon to add to the TD errors when updating priorities.
"prioritized_replay_eps": 1e-6,
# "schedule_max_timesteps": 20000,
"timesteps_per_iteration": 1000,
# Update the replay buffer with this many samples at once. Note that this
# setting applies per-worker if num_workers > 1.
# "rollout_fragment_length": 1,
"rollout_fragment_length": 1, # Renamed from sample_batch_size in some Ray version
"train_batch_size": 32,
"min_iter_time_s": 0,
}
model_config = {
"model": {
# "fcnet_hiddens": [256, 256],
# "custom_preprocessor": "ohe",
"custom_options": {}, # extra options to pass to your preprocessor
# "no_final_linear": False,
# "vf_share_layers": True,
# "fcnet_activation": "tanh",
"use_lstm": False,
},
}
eval_config = {
"evaluation_interval": 1, # I think this means every x training_iterations
"evaluation_config": {
"explore": False,
"exploration_fraction": 0,
"exploration_final_eps": 0,
"evaluation_num_episodes": 10,
"horizon": 100,
"env_config": {
"dummy_eval": True, # hack Used to check if we are in evaluation mode or training mode inside Ray callback on_episode_end() to be able to write eval stats
"transition_noise": 0
if "state_space_type" in env_config["env_config"]
and env_config["env_config"]["state_space_type"] == "discrete"
else tune.function(lambda a: a.normal(0, 0)),
"reward_noise": tune.function(lambda a: a.normal(0, 0)),
"action_loss_weight": 0.0,
},
},
}
value_tuples = []
for config_type, config_dict in var_configs.items():
for key in config_dict:
assert isinstance(
var_configs[config_type][key], list
), "var_config should be a dict of dicts with lists as the leaf values to allow each configuration option to take multiple possible values"
value_tuples.append(var_configs[config_type][key])
cartesian_product_configs = list(itertools.product(*value_tuples))
print("Total number of configs. to run:", len(cartesian_product_configs))