Docs

.gitignore

@@ -13,4 +13,15 @@ franka_control_suite/build/
 !robot_toolkit/calibration_board/target_A4.pdf
 !robot_toolkit/calibration_board/target_US_letter.pdf
 !robot_toolkit/tests/data/*.png 
-!robot_toolkit/tests/data/*.pdf
+!robot_toolkit/tests/data/*.pdf
+
+*.mtl
+*.dae
+*.mtl 
+*.obj 
+*.ply
+*.tgz
+
+!robot_toolkit/robot_arm_algos/src/inference/objects/**/*.dae
+!robot_toolkit/robot_arm_algos/src/inference/objects/**/*.mtl
+!robot_toolkit/robot_arm_algos/src/inference/objects/**/*.obj

README.md

@@ -14,7 +14,8 @@ This repository contains scripts and off-the-shelf starter scripts for developpi
 - docker 
 - git 
 - openssh-server
-
+- nvidia-container-toolkit (look at [robot_toolkit/docs/nvidia-container-toolkit.md](robot_toolkit/docs/nvidia-container-toolkit.md) )
+- GPU with necessary Nvidia drivers (if using workstation docker)
 ```
 sudo apt-get install docker.io docker-compose git 
 ```
@@ -59,14 +60,22 @@ make
 ```
 
 ## [Workstation Computer](#workstation)
-Build docker container for the workstation computer that has GPU/nvidia drivers 
+Build, create, and start the docker container for the workstation computer that has GPU/nvidia drivers 
 
 **Note: it is important to pass the workstation IP address as seen by the Realtime computer here**
 ```
 docker-compose -f docker/workstation_computer/docker-compose-gui.yml build \
-                            --build-arg workstation_ip=<workstation_ip address>
+                            --build-arg workstation_ip=<workstation_ip address> \ 
+                            --build-arg use_robotiq=1 \
+                            --build-arg build_dope=1 \
+                            --build-arg build_contactgraspnet=1 \ 
+                            --build-arg build_megapose=1
+
 docker-compose -f docker/workstation_computer/docker-compose-gui.yml create
+docker-compose -f docker/workstation_computer/docker-compose-gui.yml start
+
 ```
+**Note:** The default CUDA version is 11.3 and Ubuntu version is 20.04, if you would like to change these, please aquire an appropriate docker image and use `--build-arg image=<your image>`.
 **Note:** if you want to use [roboiq gripper](https://robotiq.com/products), please set `--build-arg use_robotiq=1` in the previous command for building workstation docker.
 
 Note: While building the docker container, the above command might print warnings in red color, don't be alarmed and let the process run. If it stops building, that's when there is an error. 
@@ -205,8 +214,11 @@ python3 docker_frankapy_test.py
 
 ## Using calibration
 
-Please checkout [robot_toolkit/docs](robot_toolkit/docs) for documentations of robot camera extrinsic calibration, usage of this tool. 
+Please checkout [robot_toolkit/docs](robot_toolkit/docs) for documentations of robot camera extrinsic calibration, usage of this tool. We also have a video showing how to use our tool [here](https://drive.google.com/file/d/1VMregvZZmWaFMxj7J3FgXDaWfS2ae4tQ/view?usp=sharing)
+
+## Using Deeplearning based prediciton models 
 
+Please checkout [robot_toolkit/docs/deep_learning_models.md](robot_toolkit/docs/deep_learning_models.md) and [robot_toolkit/docs/nvidia-container-toolkit.md](robot_toolkit/docs/nvidia-container-toolkit.md) for running Neural network models with GPUs.
 
 ## Test robotiq gripper
 To run the robotiq device, first in a terminal do

docker/scratchpad/.gitignore

@@ -0,0 +1,3 @@
+*
+!.gitignore
+!install_cudnn.sh

docker/workstation_computer/Dockerfile

@@ -1,17 +1,16 @@
-#FROM ubuntu:focal
-
-FROM osrf/ros:noetic-desktop-full
-
+ARG image=nvidia/cudagl:11.1.1-devel-ubuntu20.04
+FROM ${image}
 
+ARG DEBIAN_FRONTEND=noninteractive
+ARG workstation_ip=192.168.0.1
+ARG use_robotiq=1
+ARG build_dope=1
+ARG build_contactgraspnet=1
+ARG build_megapose=1
 ENV WS_DIR="/root/git"
 WORKDIR ${WS_DIR}
 
 SHELL ["/bin/bash", "-c"]
-
-ARG DEBIAN_FRONTEND=noninteractive
-ARG workstation_ip=192.168.0.1
-ARG use_robotiq=0
-
 # Dependencies for latency plot
 RUN apt-get update && apt-get upgrade -y && apt-get install -y \
     curl \
@@ -22,25 +21,7 @@ RUN apt-get update && apt-get upgrade -y && apt-get install -y \
     python3 \
     python3-distutils \
     gnuplot-qt \ 
-    sshpass 
-# Dependencies for Bash unit-tests
-RUN apt-get update && apt-get install -y \
-    bats \
-    dialog \
-    tmux \
-    iputils-ping 	
-
-# Fetch most recent version of rt-tests and mklatencyplot
-# Dependencies for latency plot
-RUN apt-get update && apt-get install -y \
-    curl \
-    grep \
-    build-essential \
-    make \
-    libnuma-dev \
-    python3 \
-    python3-distutils \
-    gnuplot-qt\
+    sshpass \
     iproute2 \
     python3-pip \
     gnome-terminal\
@@ -51,7 +32,28 @@ RUN apt-get update && apt-get install -y \
     bats \
     dialog \
     tmux \
-    iputils-ping 
+    iputils-ping 	
+
+# ROS Install
+RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys C1CF6E31E6BADE8868B172B4F42ED6FBAB17C654
+RUN echo "deb http://packages.ros.org/ros/ubuntu `lsb_release -sc` main" > /etc/apt/sources.list.d/ros-latest.list
+
+RUN apt-get update && apt-get install --no-install-recommends -y \
+    python3-rosdep \
+    python3-rosinstall \
+    python3-vcstools \
+    python3-catkin-tools \
+    ros-noetic-jsk-tools \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN rosdep init && \
+    rosdep update
+
+RUN apt-get update && apt-get install -y \
+    ros-noetic-desktop-full \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV ROS_DISTRO noetic
 
 # Dependencies for franka-interface and protobuf
 RUN apt-get update && apt-get install -y \
@@ -66,6 +68,7 @@ RUN apt-get update && apt-get install -y \
     unzip \
     python3-catkin-tools \ 
     python3-tk
+
 # Build and install Protobuf 
 RUN cd \
     # && mkdir git \
@@ -87,24 +90,36 @@ RUN apt-get install -y \
     ros-noetic-franka-gripper \
     ros-noetic-realsense2-camera
 
+RUN echo "Installing conda"
+ENV PATH /opt/conda/bin:$PATH
+RUN  export CONDA_DIR=/opt/conda \
+    && wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh  \
+    && /bin/bash ~/miniconda.sh -b -p /opt/conda \ 
+    && export PATH=$CONDA_DIR/bin:$PATH \ 
+    && source ~/.bashrc \
+    && conda config --set auto_activate_base false \ 
+    && conda init bash \ 
+    && conda install python=3.8
+    # && conda deactivate
+
 RUN echo "cloning building autolab perception"
 RUN cd ~/git \ 
     && git clone --recurse-submodules https://github.com/BerkeleyAutomation/perception
 RUN cd ~/git/perception \
-    && sudo pip3 install -e .
+    && pip3 install -e .
 
-RUN echo "cloning frankapy"
+RUN pip3 install -U rosdep rosinstall_generator wstool rosinstall six vcstools
 
+RUN echo "cloning frankapy"
 RUN cd ~/git \ 
     && git clone --recurse-submodules https://github.com/Ruthrash/frankapy 
 
-
-
 RUN echo "building frankapy python modules"
 RUN cd ~/git/frankapy \
-    && sudo pip3 install -e .
+    && pip3 install empy==3.3.4 \
+    &&  pip3 install -e .
 
-RUN echo "compiling catkin_ws"
+RUN echo "compiling frankapy catkin_ws"
 RUN cd ~/git/frankapy \ 
     && source /opt/ros/noetic/setup.bash \
     && ./bash_scripts/make_catkin.sh
@@ -116,53 +131,96 @@ RUN cd ~/git/frankapy \
 RUN apt-get install -y \
     ros-noetic-libfranka \
     ros-noetic-franka-ros \
-    ros-noetic-franka-gripper
+    ros-noetic-franka-gripper \
+    ros-noetic-rosbash ros-noetic-ros-comm
 
 RUN echo "source /opt/ros/noetic/setup.bash" >> ~/.bashrc
-RUN echo "source ~/git/frankapy/catkin_ws/devel/setup.bash " >> ~/.bashrc
+RUN echo "source /root/git/frankapy/catkin_ws/devel/setup.bash " >> ~/.bashrc
 
-# RUN echo "export ROS_HOSTNAME=192.168.0.1" >> ~/.bashrc
-# RUN echo "export ROS_IP=192.168.0.1" >> ~/.bashrc
-# RUN echo "export ROS_MASTER_URI=http://192.168.0.1:11311" >> ~/.bashrc
-
-# RUN echo "export ROS_HOSTNAME=${workstation_ip}" >> ~/.bashrc
 RUN echo "export ROS_IP=${workstation_ip}" >> ~/.bashrc
 RUN echo "export ROS_MASTER_URI=http://${workstation_ip}:11311" >> ~/.bashrc
 RUN echo "export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python" >> ~/.bashrc
+
 
+RUN pip3 install --upgrade protobuf
 
-RUN sudo pip3 install --upgrade protobuf
-
-RUN sudo pip3 install numpy --force-reinstall
+RUN pip3 install numpy --force-reinstall
 #reinstalling numba seems to solve an error when we import FrankaArm from frankapy
-RUN pip3 install zmq pyrealsense2 numba scipy open3d 
-
-
-# install librealsense sdk
-# RUN mkdir -p /etc/apt/keyrings \ 
-#     && echo "deb [signed-by=/etc/apt/keyrings/librealsense.pgp] https://librealsense.intel.com/Debian/apt-repo `lsb_release -cs` main" | \
-#     sudo tee /etc/apt/sources.list.d/librealsense.list \
-#     && apt-get update \ 
-#     && apt-get install librealsense2-dkms \ 
-#     && apt-get install librealsense2-utils
-
-# RUN cd /root/git/franka_arm_infra/franka_control_suite \
-#     && mkdir build \
-#     && cd build \ 
-#     && cmake .. \
-#     && make
-ARG DEBIAN_FRONTEND=dialog
-
+RUN pip3 install zmq pyrealsense2 numba scipy open3d  opencv-python  wrapt_timeout_decorator
 RUN if [[ "x$use_robotiq" == "x1" ]] ;\
     then echo "use_robotiq is set to TRUE" \
     && apt-get install ros-noetic-soem \
-    && pip3 install -U pymodbus==2.5.3 \
+    && pip3 install -U pymodbus==2.5.3 empy==3.3.4 \
     && cd ~/git \
     && mkdir -p robotiq_ws/src \
     && cd robotiq_ws/src \
     && git clone https://github.com/pairlab/robotiq.git \
     && cd ../ \ 
-    && source /opt/ros/${ROS_DISTRO}/setup.bash\
+    && source /opt/ros/noetic/setup.bash\
     && catkin build \
-    && echo "source ~/git/robotiq_ws/devel/setup.bash" >> ~/.bashrc ; \
-    fi
+    && echo "source ~/git/robotiq_ws/devel/setup.bash --extend"  >> ~/.bashrc ; \
+    fi
+
+RUN if [[ "x$build_contactgraspnet" == "x1" ]] ;\
+    then echo "build_contactgraspnet is set to TRUE" \
+    && echo "Building conda for contact_graspnet" \
+    && apt-get update && apt-get upgrade -y && apt-get install -y  freeglut3-dev\
+    && cd /root/git \ 
+    && git clone https://github.com/NVlabs/contact_graspnet \
+    && cd contact_graspnet\
+    && conda env create -f contact_graspnet_env.yml \
+    && source activate contact_graspnet_env\
+    && pip3 install open3d pyrealsense2\
+    && cd ~/git/frankapy \
+    # && pip install empy==3.3.4 \
+    && pip3 install -e . \
+    && conda deactivate\    
+    && conda deactivate; \     
+    fi
+
+
+RUN if [[ "x$build_dope" == "x1" ]] ;\
+    then echo "build_dope is set to TRUE" \
+    && apt-get install ros-noetic-vision-msgs \ 
+    && echo "Building conda for DOPE" \
+    && cd /root/git && git clone https://github.com/NVlabs/Deep_Object_Pose.git dope \
+    && export CONDA_DIR=/opt/conda && export PATH=$CONDA_DIR/bin:$PATH \
+    && conda create --name dope python=3.8 \
+    && source activate dope\ 
+    && pip3 install catkin_pkg\  
+    && cd dope && python3 -m pip install -r requirements.txt\
+    && pip3 install -e . \ 
+    && pip3 install torch torchvision --upgrade \ 
+    && pip3 install numpy==1.20.3 \
+    && pip3 install open3d pyrealsense2  \ 
+    && cd ~/git/frankapy \
+    # && pip install empy==3.3.4 \
+    && pip3 install -e . \    
+    && conda deactivate \
+    && conda deactivate;\
+    fi
+
+RUN if [[ "x$build_megapose" == "x1" ]] ;\
+    then echo "build_megapose is set to TRUE" \
+    && cd /root/git && git clone --recurse-submodules https://github.com/megapose6d/megapose6d megapose \
+    && export CONDA_DIR=/opt/conda && export PATH=$CONDA_DIR/bin:$PATH \
+    && cd /root/git/megapose/conda \
+    && sed -i 's/notebook/notebook==6.4.12/g' environment_full.yaml \ 
+    &&  conda env create -f environment_full.yaml \
+    && source activate megapose\    
+    && cd /root/git/megapose && pip3 install -e . \ 
+    && python -m megapose.scripts.download --megapose_models \
+    && pip3 install torch torchvision --upgrade \ 
+    # && pip3 install numpy==1.20.3 \
+    && pip3 install open3d pyrealsense2 \ 
+    && cd ~/git/frankapy \
+    && pip install empy==3.3.4 \
+    && pip3 install -e . \    
+    && conda deactivate\
+    && conda deactivate \ 
+    && pip3 install rospkg  wrapt-timeout-decorator;\
+    fi    
+
+
+
+SHELL ["conda", "run", "-n", "myenv", "/bin/bash", "-c"]

docker/workstation_computer/docker-compose-gui.yml

@@ -1,4 +1,4 @@
-version: "2.2"
+version: "2.4"
 services:
   docker_workstation_computer:
     extends:

docker/workstation_computer/docker-compose.yml

@@ -1,4 +1,4 @@
-version: "2.2"
+version: "2.4"
 services:
   docker_workstation_computer:
     build:
@@ -8,6 +8,8 @@ services:
     tty: true
     cap_add:
       - SYS_NICE
+    environment:
+      - CUDA_VISIBLE_DEVICES=0
     ulimits:
       rtprio: 99
       rttime: -1
@@ -16,6 +18,7 @@ services:
     privileged: true
     devices:
     - /dev:/dev
+    runtime: nvidia
     volumes:
       # - ../../calibration:/root/git/calibration
       - ../../frankapy_control_test_scripts:/root/git/frankapy_control_test_scripts