Add openmanipulator simulation environment agent (#50)

Author: whikwon

Makefile

@@ -1,9 +1,9 @@
 test:
-	env PYTHONPATH=./scripts pytest --flake8  # --cov=algorithms
+	env PYTHONPATH=./scripts pytest --flake8 --ignore=./scripts/envs # --cov=algorithms
 
 format:
 	isort -y
-	python3.6 -m black -t py27 .
+	python3.6 -m black -t py27 . --fast
 
 dev:
 	pip install -r scripts/requirements-dev.txt

README.md

@@ -23,19 +23,19 @@ The [scripts](/scripts) folder contains implementations of a curated list of RL
 
 - Twin Delayed Deep Deterministic Policy Gradient (TD3)
    - TD3 (Fujimoto et al., 2018) is an extension of DDPG (Lillicrap et al., 2015), a deterministic policy gradient algorithm that uses deep neural networks for function approximation. Inspired by Deep Q-Networks (Mnih et al., 2015), DDPG uses experience replay and target network to improve stability. TD3 further improves DDPG by adding clipped double Q-learning (Van Hasselt, 2010) to mitigate overestimation bias (Thrun & Schwartz, 1993) and delaying policy updates to address variance.
-   - [Example Script on LunarLander](/scripts/examples/lunarlander_continuous_v2/td3.py)
+   - [Example Script on LunarLander](/scripts/config/agent/lunarlander_continuous_v2/td3.py)
    - [ArXiv Preprint](https://arxiv.org/abs/1802.09477)
 
 - (Twin) Soft Actor Critic (SAC)
    - SAC (Haarnoja et al., 2018a) incorporates maximum entropy reinforcment learning, where the agent's goal is to maximize expected reward and entropy concurrently. Combined with TD3, SAC achieves state of the art performance in various continuous control tasks. SAC has been extended to allow automatically tuning of the temperature parameter (Haarnoja et al., 2018b), which determines the importance of entropy against the expected reward.
-   - [Example Script on LunarLander](/scripts/examples/lunarlander_continuous_v2/sac.py)
+   - [Example Script on LunarLander](/scripts/config/agent/lunarlander_continuous_v2/sac.py)
    - [ArXiv Preprint](https://arxiv.org/abs/1801.01290) (Original SAC)
    - [ArXiv Preprint](https://arxiv.org/abs/1812.05905) (SAC with autotuned temperature)
 
  - TD3 from Demonstrations, SAC from Demonstrations (TD3fD, SACfD)
    - DDPGfD (Vecerik et al., 2017) is an imitation learning algorithm that infuses demonstration data into experience replay. DDPGfD also improved DDPG by (1) using prioritized experience replay (Schaul et al., 2015), (2) adding n-step returns, (3) learning multiple times per environment step, and (4) adding L2 regularizers to actor and critic losses. We incorporated these improvements to TD3 and SAC and found that it dramatically improves their performance.
-   - [Example Script of TD3fD on LunarLander](/scripts/examples/lunarlander_continuous_v2/td3fd.py)
-   - [Example Script of SACfD on LunarLander](/scripts/examples/lunarlander_continuous_v2/sacfd.py)
+   - [Example Script of TD3fD on LunarLander](/scripts/config/agent/lunarlander_continuous_v2/td3fd.py)
+   - [Example Script of SACfD on LunarLander](/scripts/config/agent/lunarlander_continuous_v2/sacfd.py)
    - [ArXiv Preprint](https://arxiv.org/abs/1707.08817)
 
 ## Installation

docker_train.sh

@@ -6,9 +6,10 @@ KAIR=$CATKIN_WS/src/kair_algorithms_draft
 
 if [ "$1" == "lunarlander" ]; then
 	cd $KAIR/scripts; \
-	   python run_lunarlander_continuous.py --algo $2 --off-render
+		python run_lunarlander_continuous.py --algo $2 --off-render
 elif [ "$1" == "openmanipulator" ]; then
-	echo "Working"
+	cd $KAIR/scripts; \
+		/opt/ros/$ROS_DISTRO/bin/rosrun kair_algorithms run_open_manipulator_reacher_v0.py --algo $2 --off-render
 else
 	echo "Unknown parameter"
 fi

launch/open_manipulator_env.launch

@@ -0,0 +1,50 @@
+<?xml version="1.0"?>
+<launch>
+  <!-- gazebo related args -->
+  <arg name="paused" default="false"/>
+  <arg name="use_sim_time" default="true"/>
+  <arg name="gui" default="true"/>
+  <arg name="headless" default="false"/>
+  <arg name="debug" default="false"/>
+
+  <!-- rviz & tf related args -->
+  <arg name="robot_name"     default="open_manipulator"/>
+  <arg name="open_rviz"      default="false" />
+  <arg name="use_gui"        default="false" />
+
+  <!-- gazebo related -->
+  <rosparam file="$(find open_manipulator_gazebo)/config/gazebo_controller.yaml" command="load" />
+  <include file="$(find gazebo_ros)/launch/empty_world.launch">
+    <arg name="world_name" value="$(find open_manipulator_gazebo)/worlds/empty.world"/>
+    <arg name="debug" value="$(arg debug)" />
+    <arg name="gui" value="$(arg gui)" />
+    <arg name="paused" value="$(arg paused)"/>
+    <arg name="use_sim_time" value="$(arg use_sim_time)"/>
+    <arg name="headless" value="$(arg headless)"/>
+  </include>
+
+  <!-- rviz related -->
+    <!-- Send joint values -->
+    <node pkg="joint_state_publisher" type="joint_state_publisher" name="joint_state_publisher">
+      <param name="/use_gui" value="$(arg use_gui)"/>
+      <rosparam param="source_list" subst_value="true">["$(arg robot_name)/joint_states"]</rosparam>
+    </node>
+    <!-- Combine joint values to TF-->
+    <node name="robot_state_publisher" pkg="robot_state_publisher" type="state_publisher"/>
+
+    <!-- Show in Rviz -->
+    <group if="$(arg open_rviz)">
+      <node name="rviz" pkg="rviz" type="rviz" args="-d $(find open_manipulator_description)/rviz/open_manipulator.rviz"/>
+    </group>
+
+    <!-- Load the URDF into the ROS Parameter Server -->
+    <param name="robot_description"
+    command="$(find xacro)/xacro --inorder '$(find open_manipulator_description)/urdf/open_manipulator.urdf.xacro'"/>
+
+    <!-- Run a python script to the send a service call to gazebo_ros to spawn a URDF robot -->
+    <node name="urdf_spawner" pkg="gazebo_ros" type="spawn_model" respawn="false" output="screen"
+     args="-urdf -model open_manipulator -z 0.0 -param robot_description"/>
+
+    <!-- ros_control robotis manipulator launch file -->
+    <include file="$(find open_manipulator_gazebo)/launch/open_manipulator_controller.launch"/>   
+</launch>

launch/vel_kinematics.launch

@@ -44,7 +44,7 @@ def __init__(self, env, args):
             self.args.max_episode_steps = env._max_episode_steps
 
         # for logging
-        self.env_name = str(self.env.env).split("<")[2].replace(">>", "")
+        self.env_name = str(self.env.env).split("<")[1].replace(">>", "")
         self.sha = (
             subprocess.check_output(["git", "rev-parse", "--short", "HEAD"])[:-1]
             .decode("ascii")

launch/yumi_gazebo_vel.launch

@@ -33,22 +33,28 @@
     "AUTO_ENTROPY_TUNING": True,
     "WEIGHT_DECAY": 0.0,
     "INITIAL_RANDOM_ACTION": 5000,
+    "NETWORK": {
+        "ACTOR_HIDDEN_SIZES": [256, 256],
+        "VF_HIDDEN_SIZES": [256, 256],
+        "QF_HIDDEN_SIZES": [256, 256],
+    },
 }
 
 
-def run(env, args, state_dim, action_dim):
+def get(env, args):
     """Run training or test.
 
     Args:
         env (gym.Env): openAI Gym environment with continuous action space
         args (argparse.Namespace): arguments including training settings
-        state_dim (int): dimension of states
-        action_dim (int): dimension of actions
 
     """
-    hidden_sizes_actor = [256, 256]
-    hidden_sizes_vf = [256, 256]
-    hidden_sizes_qf = [256, 256]
+    state_dim = env.observation_space.shape[0]
+    action_dim = env.action_space.shape[0]
+
+    hidden_sizes_actor = hyper_params["NETWORK"]["ACTOR_HIDDEN_SIZES"]
+    hidden_sizes_vf = hyper_params["NETWORK"]["VF_HIDDEN_SIZES"]
+    hidden_sizes_qf = hyper_params["NETWORK"]["QF_HIDDEN_SIZES"]
 
     # target entropy
     target_entropy = -np.prod((action_dim,)).item()  # heuristic
@@ -102,10 +108,4 @@ def run(env, args, state_dim, action_dim):
     optims = (actor_optim, vf_optim, qf_1_optim, qf_2_optim)
 
     # create an agent
-    agent = Agent(env, args, hyper_params, models, optims, target_entropy)
-
-    # run
-    if args.test:
-        agent.test()
-    else:
-        agent.train()
+    return Agent(env, args, hyper_params, models, optims, target_entropy)

scripts/algorithms/common/abstract/agent.py

@@ -42,22 +42,28 @@
     "PER_EPS": 1e-6,
     "PER_EPS_DEMO": 1.0,
     "INITIAL_RANDOM_ACTION": int(5e3),
+    "NETWORK": {
+        "ACTOR_HIDDEN_SIZES": [256, 256],
+        "VF_HIDDEN_SIZES": [256, 256],
+        "QF_HIDDEN_SIZES": [256, 256],
+    },
 }
 
 
-def run(env, args, state_dim, action_dim):
+def get(env, args):
     """Run training or test.
 
     Args:
         env (gym.Env): openAI Gym environment with continuous action space
         args (argparse.Namespace): arguments including training settings
-        state_dim (int): dimension of states
-        action_dim (int): dimension of actions
 
     """
-    hidden_sizes_actor = [256, 256]
-    hidden_sizes_vf = [256, 256]
-    hidden_sizes_qf = [256, 256]
+    state_dim = env.observation_space.shape[0]
+    action_dim = env.action_space.shape[0]
+
+    hidden_sizes_actor = hyper_params["NETWORK"]["ACTOR_HIDDEN_SIZES"]
+    hidden_sizes_vf = hyper_params["NETWORK"]["VF_HIDDEN_SIZES"]
+    hidden_sizes_qf = hyper_params["NETWORK"]["QF_HIDDEN_SIZES"]
 
     # target entropy
     target_entropy = -np.prod((action_dim,)).item()  # heuristic
@@ -109,10 +115,4 @@ def run(env, args, state_dim, action_dim):
     optims = (actor_optim, vf_optim, qf_1_optim, qf_2_optim)
 
     # create an agent
-    agent = Agent(env, args, hyper_params, models, optims, target_entropy)
-
-    # run
-    if args.test:
-        agent.test()
-    else:
-        agent.train()
+    return Agent(env, args, hyper_params, models, optims, target_entropy)

scripts/examples/__init__.py renamed to scripts/config/__init__.py

@@ -28,21 +28,23 @@
     "TARGET_POLICY_NOISE_CLIP": 0.5,
     "POLICY_UPDATE_FREQ": 2,
     "INITIAL_RANDOM_ACTIONS": 1e4,
+    "NETWORK": {"ACTOR_HIDDEN_SIZES": [400, 300], "CRITIC_HIDDEN_SIZES": [400, 300]},
 }
 
 
-def run(env, args, state_dim, action_dim):
+def get(env, args):
     """Run training or test.
 
     Args:
         env (gym.Env): openAI Gym environment with continuous action space
         args (argparse.Namespace): arguments including training settings
-        state_dim (int): dimension of states
-        action_dim (int): dimension of actions
 
     """
-    hidden_sizes_actor = [400, 300]
-    hidden_sizes_critic = [400, 300]
+    state_dim = env.observation_space.shape[0]
+    action_dim = env.action_space.shape[0]
+
+    hidden_sizes_actor = hyper_params["NETWORK"]["ACTOR_HIDDEN_SIZES"]
+    hidden_sizes_critic = hyper_params["NETWORK"]["CRITIC_HIDDEN_SIZES"]
 
     # create actor
     actor = MLP(
@@ -123,10 +125,4 @@ def run(env, args, state_dim, action_dim):
     noises = (exploration_noise, target_policy_noise)
 
     # create an agent
-    agent = Agent(env, args, hyper_params, models, optims, noises)
-
-    # run
-    if args.test:
-        agent.test()
-    else:
-        agent.train()
+    return Agent(env, args, hyper_params, models, optims, noises)

scripts/examples/lunarlander_continuous_v2/__init__.py renamed to scripts/config/agent/__init__.py

@@ -40,21 +40,23 @@
     "PER_BETA": 1.0,
     "PER_EPS": 1e-6,
     "PER_EPS_DEMO": 1.0,
+    "NETWORK": {"ACTOR_HIDDEN_SIZES": [400, 300], "CRITIC_HIDDEN_SIZES": [400, 300]},
 }
 
 
-def run(env, args, state_dim, action_dim):
+def get(env, args):
     """Run training or test.
 
     Args:
         env (gym.Env): openAI Gym environment with continuous action space
         args (argparse.Namespace): arguments including training settings
-        state_dim (int): dimension of states
-        action_dim (int): dimension of actions
 
     """
-    hidden_sizes_actor = [400, 300]
-    hidden_sizes_critic = [400, 300]
+    state_dim = env.observation_space.shape[0]
+    action_dim = env.action_space.shape[0]
+
+    hidden_sizes_actor = hyper_params["NETWORK"]["ACTOR_HIDDEN_SIZES"]
+    hidden_sizes_critic = hyper_params["NETWORK"]["CRITIC_HIDDEN_SIZES"]
 
     # create actor
     actor = MLP(
@@ -135,10 +137,4 @@ def run(env, args, state_dim, action_dim):
     noises = (exploration_noise, target_policy_noise)
 
     # create an agent
-    agent = Agent(env, args, hyper_params, models, optims, noises)
-
-    # run
-    if args.test:
-        agent.test()
-    else:
-        agent.train()
+    return Agent(env, args, hyper_params, models, optims, noises)

scripts/examples/reacher-v1/__init__.py renamed to scripts/config/agent/lunarlander_continuous_v2/__init__.py

@@ -28,21 +28,23 @@
     "TARGET_POLICY_NOISE_CLIP": 0.5,
     "POLICY_UPDATE_FREQ": 2,
     "INITIAL_RANDOM_ACTIONS": 1e4,
+    "NETWORK": {"ACTOR_HIDDEN_SIZES": [400, 300], "CRITIC_HIDDEN_SIZES": [400, 300]},
 }
 
 
-def run(env, args, state_dim, action_dim):
+def get(env, args):
     """Run training or test.
 
     Args:
         env (gym.Env): openAI Gym environment with continuous action space
         args (argparse.Namespace): arguments including training settings
-        state_dim (int): dimension of states
-        action_dim (int): dimension of actions
 
     """
-    hidden_sizes_actor = [400, 300]
-    hidden_sizes_critic = [400, 300]
+    state_dim = env.observation_space.shape[0]
+    action_dim = env.action_space.shape[0]
+
+    hidden_sizes_actor = hyper_params["NETWORK"]["ACTOR_HIDDEN_SIZES"]
+    hidden_sizes_critic = hyper_params["NETWORK"]["CRITIC_HIDDEN_SIZES"]
 
     # create actor
     actor = MLP(
@@ -123,10 +125,4 @@ def run(env, args, state_dim, action_dim):
     noises = (exploration_noise, target_policy_noise)
 
     # create an agent
-    agent = Agent(env, args, hyper_params, models, optims, noises)
-
-    # run
-    if args.test:
-        agent.test()
-    else:
-        agent.train()
+    return Agent(env, args, hyper_params, models, optims, noises)