add grasp_detectoin part

Author: shirohasuki

grasp_detection/README.md

@@ -0,0 +1,66 @@
+# grasp_detection
+
+
+
+
+### Main system requirements:
+- CUDA 11.1.1
+- Python 3.8
+- TensorFlow v2.4.1
+
+
+
+### Quick Start
+1. Clone this repository
+```
+git clone https://github.com/ivalab/grasp_multiObject_multiGrasp.git
+cd grasp_multiObject_multiGrasp
+```
+
+2. Build Cython modules
+```
+cd lib
+make clean
+make
+cd ..
+```
+
+3. Install [Python COCO API](https://github.com/cocodataset/cocoapi)
+```
+cd data
+git clone https://github.com/pdollar/coco.git
+cd coco/PythonAPI
+make
+cd ../../..
+```
+
+4. Put models under `output/res50/train/default/`
+4. Run demo
+
+```
+./tools/demo_graspRGD.py --net res50 --dataset grasp
+```
+6. Run with own images
+
+If you want to run demo with your own images,  just put your image into `./data/demo/` and change file path in `./tools/demo_graspRGD.py`
+
+
+
+### Train
+
+1. Generate data
+   1-1. Download [Cornell Dataset](http://pr.cs.cornell.edu/grasping/rect_data/data.php)
+   1-2. Run `dataPreprocessingTest_fasterrcnn_split.m` (please modify paths according to your structure)
+2. Train
+
+```
+./experiments/scripts/train_faster_rcnn.sh 0 graspRGB res50
+```
+
+
+
+
+
+### Thanks
+
+[GitHub - ivalab/grasp_multiObject_multiGrasp: An implementation of our RA-L work 'Real-world Multi-object, Multi-grasp Detection'](https://github.com/ivalab/grasp_multiObject_multiGrasp)

grasp_detection/data/demo/valid_net0.png

@@ -0,0 +1,110 @@
+%% script to test dataPreprocessing
+%% created by Fu-Jen Chu on 09/15/2016
+
+close all
+clear
+
+%parpool(4)
+addpath('/media/fujenchu/home3/data/grasps/')
+
+
+% generate list for splits
+list = [100:949 1000:1034];
+list_idx = randperm(length(list));
+train_list_idx = list_idx(length(list)/5+1:end);
+test_list_idx = list_idx(1:length(list)/5);
+train_list = list(train_list_idx);
+test_list = list(test_list_idx);
+
+
+for folder = 1:10
+display(['processing folder ' int2str(folder)])
+
+imgDataDir = ['/home/fujenchu/projects/deepLearning/tensorflow-finetune-flickr-style-master/data/grasps/' sprintf('%02d',folder) '_rgd'];
+txtDataDir = ['/home/fujenchu/projects/deepLearning/tensorflow-finetune-flickr-style-master/data/grasps/' sprintf('%02d',folder)];
+
+%imgDataOutDir = ['/home/fujenchu/projects/deepLearning/tensorflow-finetune-flickr-style-master/data/grasps/' sprintf('%02d',folder) '_Cropped320_rgd'];
+imgDataOutDir = '/media/fujenchu/home3/fasterrcnn_grasp/rgd_multibbs_5_5_5_tf/data/Images';
+annotationDataOutDir = '/media/fujenchu/home3/fasterrcnn_grasp/rgd_multibbs_5_5_5_tf/data/Annotations';
+imgSetTrain = '/media/fujenchu/home3/fasterrcnn_grasp/rgd_multibbs_5_5_5_tf/data/ImageSets/train.txt'; 
+imgSetTest = '/media/fujenchu/home3/fasterrcnn_grasp/rgd_multibbs_5_5_5_tf/data/ImageSets/test.txt'; 
+
+imgFiles = dir([imgDataDir '/*.png']);
+txtFiles = dir([txtDataDir '/*pos.txt']);
+
+logfileID = fopen('log.txt','a');
+mainfileID = fopen(['/home/fujenchu/projects/deepLearning/deepGraspExtensiveOffline/data/grasps/scripts/trainttt' sprintf('%02d',folder) '.txt'],'a');
+for idx = 1:length(imgFiles) 
+    %% display progress
+    tic
+    display(['processing folder: ' sprintf('%02d',folder) ', imgFiles: ' int2str(idx)])
+    
+    %% reading data
+    imgName = imgFiles(idx).name;
+    [pathstr,imgname] = fileparts(imgName);
+    
+    filenum = str2num(imgname(4:7));
+    if(any(test_list == filenum))
+        file_writeID = fopen(imgSetTest,'a');
+        fprintf(file_writeID, '%s\n', [imgDataDir(1:end-3) 'Cropped320_rgd/' imgname '_preprocessed_1.png' ] );
+        fclose(file_writeID);
+        continue;
+    end
+    
+    txtName = txtFiles(idx).name;
+    [pathstr,txtname] = fileparts(txtName);
+
+    img = imread([imgDataDir '/' imgname '.png']);
+    fileID = fopen([txtDataDir '/' txtname '.txt'],'r');
+    sizeA = [2 100];
+    bbsIn_all = fscanf(fileID, '%f %f', sizeA);
+    fclose(fileID);
+    
+    %% data pre-processing
+    [imagesOut bbsOut] = dataPreprocessing_fasterrcnn(img, bbsIn_all, 227, 5, 5);
+    
+    % for each augmented image
+    for i = 1:1:size(imagesOut,2)
+        
+        % for each bbs
+        file_writeID = fopen([annotationDataOutDir '/' imgname '_preprocessed_' int2str(i) '.txt'],'w');
+        printCount = 0;
+        for ibbs = 1:1:size(bbsOut{i},2)
+          A = bbsOut{i}{ibbs};  
+          xy_ctr = sum(A,2)/4; x_ctr = xy_ctr(1); y_ctr = xy_ctr(2);
+          width = sqrt(sum((A(:,1) - A(:,2)).^2)); height = sqrt(sum((A(:,2) - A(:,3)).^2));
+          if(A(1,1) > A(1,2))
+              theta = atan((A(2,2)-A(2,1))/(A(1,1)-A(1,2)));
+          else
+              theta = atan((A(2,1)-A(2,2))/(A(1,2)-A(1,1))); % note y is facing down
+          end  
+    
+          % process to fasterrcnn
+          x_min = x_ctr - width/2; x_max = x_ctr + width/2;
+          y_min = y_ctr - height/2; y_max = y_ctr + height/2;
+          %if(x_min < 0 || y_min < 0 || x_max > 227 || y_max > 227) display('yoooooooo'); end
+          if((x_min < 0 && x_max < 0) || (y_min > 227 && y_max > 227) || (x_min > 227 && x_max > 227) || (y_min < 0 && y_max < 0)) display('xxxxxxxxx'); break; end
+          cls = round((theta/pi*180+90)/10) + 1;
+          
+          % write as lefttop rightdown, Xmin Ymin Xmax Ymax, ex: 261 109 511 705  (x水平 y垂直)
+          fprintf(file_writeID, '%d %f %f %f %f\n', cls, x_min, y_min, x_max, y_max );   
+          printCount = printCount+1;
+        end
+        if(printCount == 0) fprintf(logfileID, '%s\n', [imgname '_preprocessed_' int2str(i) ]);end
+        
+        fclose(file_writeID);
+        imwrite(imagesOut{i}, [imgDataOutDir '/' imgname '_preprocessed_' int2str(i) '.png']); 
+        
+        % write filename to imageSet 
+        file_writeID = fopen(imgSetTrain,'a');
+        fprintf(file_writeID, '%s\n', [imgname '_preprocessed_' int2str(i) ] );
+        fclose(file_writeID);
+
+    end
+    
+
+    toc
+end
+fclose(mainfileID);
+
+end

grasp_detection/data/demo/valid_net1.png

@@ -0,0 +1,95 @@
+function [imagesOut bbsOut] = dataPreprocessing( imageIn, bbsIn_all, cropSize, translationShiftNumber, roatateAngleNumber)
+% dataPreprocessing function perfroms
+% 1) croping
+% 2) padding
+% 3) rotatation
+% 4) shifting
+%
+% for a input image with a bbs as 4 points, 
+% dataPreprocessing outputs a set of images with corresponding bbs.
+%
+%
+% Inputs: 
+%   imageIn: input image (480 by 640 by 3)
+%   bbsIn: bounding box (2 by 4)
+%   cropSize: output image size
+%   shift: shifting offset
+%   rotate: rotation angle
+%
+% Outputs:
+%   imagesOut: output images (n images)
+%   bbsOut: output bbs according to shift and rotation
+%
+%% created by Fu-Jen Chu on 09/15/2016
+
+debug_dev = 0;
+debug = 0;
+%% show image and bbs
+if(debug_dev)
+figure(1); imshow(imageIn); hold on;
+x = bbsIn_all(1, [1:3]);
+y = bbsIn_all(2, [1:3]);
+plot(x,y); hold off;
+end
+
+%% crop image and padding image
+% cropping to 321 by 321 from center
+imgCrop = imcrop(imageIn, [145 65 351 351]); 
+
+% padding to 501 by 501
+imgPadding = padarray(imgCrop, [75 75], 'replicate', 'both');
+
+count = 1;
+for i_rotate = 1:roatateAngleNumber*translationShiftNumber*translationShiftNumber
+    % random roatateAngle
+    theta = randi(360)-1;
+    %theta = 0;
+
+    % random translationShift
+    dx = randi(101)-51;
+    %dx = 0;
+
+    %% rotation and shifting
+    % random translationShift
+    dy = randi(101)-51;
+    %dy = 0;
+
+    imgRotate = imrotate(imgPadding, theta);
+    if(debug_dev)figure(2); imshow(imgRotate);end
+    imgCropRotate = imcrop(imgRotate, [size(imgRotate,1)/2-160-dx size(imgRotate,1)/2-160-dy 320 320]);
+    if(debug_dev)figure(3); imshow(imgCropRotate);end 
+    imgResize = imresize(imgCropRotate, [cropSize cropSize]);
+    if(debug)figure(4); imshow(imgResize); hold on;end
+
+    %% modify bbs
+    [m, n] = size(bbsIn_all);
+    bbsNum = n/4;
+    
+    countbbs = 1;
+    for idx = 1:bbsNum
+      bbsIn =  bbsIn_all(:,idx*4-3:idx*4); 
+      if(sum(sum(isnan(bbsIn)))) continue; end
+      
+      bbsInShift = bbsIn - repmat([320; 240], 1, 4);
+      R = [cos(theta/180*pi) -sin(theta/180*pi); sin(theta/180*pi) cos(theta/180*pi)];
+      bbsRotated = (bbsInShift'*R)';
+      bbsInShiftBack = (bbsRotated + repmat([160; 160], 1, 4) + repmat([dx; dy], 1, 4))*cropSize/320;
+      if(debug)
+        figure(4)
+        x = bbsInShiftBack(1, [1:4 1]);
+        y = bbsInShiftBack(2, [1:4 1]);
+        plot(x,y); hold on; pause(0.01);
+      end
+      bbsOut{count}{countbbs} = bbsInShiftBack;
+      countbbs = countbbs + 1;
+    end
+    
+    imagesOut{count} = imgResize;
+    count = count +1;
+
+   
+end
+
+
+
+end

grasp_detection/data/demo/valid_net2.png

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../" && pwd )"
+cd $DIR
+
+NET=res101
+FILE=voc_0712_80k-110k.tgz
+# replace it with gs11655.sp.cs.cmu.edu if ladoga.graphics.cs.cmu.edu does not work
+URL=http://ladoga.graphics.cs.cmu.edu/xinleic/tf-faster-rcnn/$NET/$FILE
+CHECKSUM=cb32e9df553153d311cc5095b2f8c340
+
+if [ -f $FILE ]; then
+  echo "File already exists. Checking md5..."
+  os=`uname -s`
+  if [ "$os" = "Linux" ]; then
+    checksum=`md5sum $FILE | awk '{ print $1 }'`
+  elif [ "$os" = "Darwin" ]; then
+    checksum=`cat $FILE | md5`
+  fi
+  if [ "$checksum" = "$CHECKSUM" ]; then
+    echo "Checksum is correct. No need to download."
+    exit 0
+  else
+    echo "Checksum is incorrect. Need to download again."
+  fi
+fi
+
+echo "Downloading Resnet 101 Faster R-CNN models Pret-trained on VOC 07+12 (340M)..."
+
+wget $URL -O $FILE
+
+echo "Unzipping..."
+
+tar zxvf $FILE
+
+echo "Done. Please run this command again to verify that checksum = $CHECKSUM."

grasp_detection/data/demo/valid_net3.png

@@ -0,0 +1,23 @@
+EXP_DIR: res101-lg
+TRAIN:
+  HAS_RPN: True
+  IMS_PER_BATCH: 1
+  BBOX_NORMALIZE_TARGETS_PRECOMPUTED: True
+  RPN_POSITIVE_OVERLAP: 0.7
+  RPN_BATCHSIZE: 256
+  PROPOSAL_METHOD: gt
+  BG_THRESH_LO: 0.0
+  DISPLAY: 20
+  BATCH_SIZE: 256
+  WEIGHT_DECAY: 0.0001
+  DOUBLE_BIAS: False
+  SNAPSHOT_PREFIX: res101_faster_rcnn
+  SCALES: [800]
+  MAX_SIZE: 1333
+TEST:
+  HAS_RPN: True
+  SCALES: [800]
+  MAX_SIZE: 1333
+  RPN_POST_NMS_TOP_N: 1000
+POOLING_MODE: crop
+ANCHOR_SCALES: [2,4,8,16,32]

grasp_detection/data/demo/valid_net4.png

@@ -0,0 +1,17 @@
+EXP_DIR: res101
+TRAIN:
+  HAS_RPN: True
+  IMS_PER_BATCH: 1
+  BBOX_NORMALIZE_TARGETS_PRECOMPUTED: True
+  RPN_POSITIVE_OVERLAP: 0.7
+  RPN_BATCHSIZE: 256
+  PROPOSAL_METHOD: gt
+  BG_THRESH_LO: 0.0
+  DISPLAY: 20
+  BATCH_SIZE: 256
+  WEIGHT_DECAY: 0.0001
+  DOUBLE_BIAS: False
+  SNAPSHOT_PREFIX: res101_faster_rcnn
+TEST:
+  HAS_RPN: True
+POOLING_MODE: crop

grasp_detection/data/scripts/dataPreprocessingTest_fasterrcnn_split.m

@@ -0,0 +1,17 @@
+EXP_DIR: res50
+TRAIN:
+  HAS_RPN: True
+  IMS_PER_BATCH: 1
+  BBOX_NORMALIZE_TARGETS_PRECOMPUTED: True
+  RPN_POSITIVE_OVERLAP: 0.7
+  RPN_BATCHSIZE: 256
+  PROPOSAL_METHOD: gt
+  BG_THRESH_LO: 0.0
+  DISPLAY: 20
+  BATCH_SIZE: 256
+  WEIGHT_DECAY: 0.0001
+  DOUBLE_BIAS: False
+  SNAPSHOT_PREFIX: res50_faster_rcnn
+TEST:
+  HAS_RPN: True
+POOLING_MODE: crop

grasp_detection/data/scripts/dataPreprocessing_fasterrcnn.m

@@ -0,0 +1,15 @@
+EXP_DIR: vgg16
+TRAIN:
+  HAS_RPN: True
+  IMS_PER_BATCH: 1
+  BBOX_NORMALIZE_TARGETS_PRECOMPUTED: True
+  RPN_POSITIVE_OVERLAP: 0.7
+  RPN_BATCHSIZE: 256
+  PROPOSAL_METHOD: gt
+  BG_THRESH_LO: 0.0
+  DISPLAY: 20
+  BATCH_SIZE: 256
+  SNAPSHOT_PREFIX: vgg16_faster_rcnn
+TEST:
+  HAS_RPN: True
+POOLING_MODE: crop