evaluation

Author: art-programmer

PlaneSetGeneration/train_planenet.py

@@ -32,7 +32,7 @@
 MOVING_AVERAGE_DECAY = 0.99
 
 deepSupervisionLayers=['res4b22_relu']
-deepSupervisionLayers=[]
+#deepSupervisionLayers=[]
 
 def build_graph(img_inp_train, img_inp_val, plane_gt_train, plane_gt_val, validating_inp, is_training=True, numOutputPlanes=20, gpu_id = 0, useCRF= 0, suffix='forward'):
     if suffix == '12_22':
@@ -731,7 +731,7 @@ def test(gpu_id, dumpdir, logdir, testdir, keyname, restore, numOutputPlanes=20,
             ranges = np.array([urange / imageWidth * 640 / focalLength, np.ones(urange.shape), -vrange / imageHeight * 480 / focalLength]).transpose([1, 2, 0])
 
 
-            for index in xrange(100):
+            for index in xrange(10):
                 print(('image', index))
                 t0=time.time()
                 #im, planes, depth, normal, boundary, s_8, p_8, b_8, s_16, p_16, b_16, s_32, p_32, b_32 = sess.run([img_inp, plane_gt, depth_gt, normal_gt, boundary_gt, s_8_gt, p_8_gt, b_8_gt, s_16_gt, p_16_gt, b_16_gt, s_32_gt, p_32_gt, b_32_gt])

code/RecordWriterRGBD.py

@@ -100,10 +100,10 @@ def writeRecordFile(tfrecords_filename, imagePaths):
 if __name__=='__main__':
     imagePaths = glob.glob('/home/chenliu/Projects/Data/NYU_RGBD/raw/train/color_*.png')
     imagePaths = [{'image': imagePath, 'depth': imagePath.replace('color', 'depth')} for imagePath in imagePaths]
-    #print(len(imagePaths))
+    print(len(imagePaths))
     #exit(1)
     random.shuffle(imagePaths)
-    writeRecordFile('../planes_nyu_rgbd_train_temp.tfrecords', imagePaths)
+    writeRecordFile('../planes_nyu_rgbd_train.tfrecords', imagePaths)
     #reader.readRecordFile()
 
 

code/evaluate.py

@@ -51,7 +51,7 @@ def __init__(self, inputs, options, trainable=True, is_training=False):
                                                        shape=[],
                                                        name='use_dropout')
 
-        self.setup(True, options=options)
+        self.setup(is_training, options=options)
 
     def setup(self, is_training):
         '''Construct the network. '''

code/kaffe/__init__.pyc

@@ -945,3 +945,32 @@ def depthToNormalModule(depth):
 
     normal = normal / tf.norm(normal, axis=3, keep_dims=True)
     return normal
+
+def findBoundaryModule(depth, normal, segmentation, plane_mask, max_depth_diff = 0.1, max_normal_diff = np.sqrt(2 * (1 - np.cos(np.deg2rad(20))))):
+    kernel_size = 3
+    padding = (kernel_size - 1) / 2
+    neighbor_kernel_array = gaussian(kernel_size, kernel_size)
+    neighbor_kernel_array[(kernel_size - 1) / 2][(kernel_size - 1) / 2] = 0
+    neighbor_kernel_array /= neighbor_kernel_array.sum()
+    neighbor_kernel_array *= -1
+    neighbor_kernel_array[(kernel_size - 1) / 2][(kernel_size - 1) / 2] = 1
+    neighbor_kernel = tf.constant(neighbor_kernel_array.reshape(-1), shape=neighbor_kernel_array.shape, dtype=tf.float32)
+    neighbor_kernel = tf.reshape(neighbor_kernel, [kernel_size, kernel_size, 1, 1])
+        
+    depth_diff = tf.abs(tf.nn.depthwise_conv2d(depth, neighbor_kernel, strides=[1, 1, 1, 1], padding='VALID'))
+    depth_diff = tf.pad(depth_diff, paddings = [[0, 0], [padding, padding], [padding, padding], [0, 0]])
+    max_depth_diff = 0.1
+    depth_boundary = tf.greater(depth_diff, max_depth_diff)
+
+    normal_diff = tf.norm(tf.nn.depthwise_conv2d(normal, tf.tile(neighbor_kernel, [1, 1, 3, 1]), strides=[1, 1, 1, 1], padding='VALID'), axis=3, keep_dims=True)
+    normal_diff = tf.pad(normal_diff, paddings = [[0, 0], [padding, padding], [padding, padding], [0, 0]])
+    max_normal_diff = np.sqrt(2 * (1 - np.cos(np.deg2rad(20))))
+    normal_boundary = tf.greater(normal_diff, max_normal_diff)
+
+    plane_region = tf.nn.max_pool(plane_mask, ksize=[1, kernel_size, kernel_size, 1], strides=[1, 1, 1, 1], padding='SAME', name='max_pool')
+    segmentation_eroded = 1 - tf.nn.max_pool(1 - segmentation, ksize=[1, 3, 3, 1], strides=[1, 1, 1, 1], padding='SAME', name='max_pool')
+    plane_region -= tf.reduce_max(segmentation_eroded, axis=3, keep_dims=True)
+    boundary = tf.cast(tf.logical_or(depth_boundary, normal_boundary), tf.float32)
+    smooth_boundary = tf.cast(tf.logical_and(normal_boundary, tf.less_equal(depth_diff, max_depth_diff)), tf.float32) * plane_region
+    boundary_gt = tf.concat([smooth_boundary, boundary - smooth_boundary], axis=3)
+    return boundary_gt

code/kaffe/caffe/__init__.pyc

@@ -20,6 +20,7 @@
 from planenet import PlaneNet
 from RecordReader import *
 from RecordReaderRGBD import *
+from RecordReaderScanNet import *
 
 #training_flag: toggle dropout and batch normalization mode
 #it's true for training and false for validation, testing, prediction
@@ -29,7 +30,8 @@
 def build_graph(img_inp_train, img_inp_val, img_inp_rgbd_train, img_inp_rgbd_val, training_flag, options):
     with tf.device('/gpu:%d'%options.gpu_id):
         img_inp_rgbd = tf.cond(tf.equal(training_flag % 2, 0), lambda: img_inp_rgbd_train, lambda: img_inp_rgbd_val)
-        img_inp = tf.cond(tf.less(training_flag, 2), lambda: tf.cond(tf.equal(training_flag % 2, 0), lambda: img_inp_train, lambda: img_inp_val), lambda: img_inp_rgbd)
+        img_inp = tf.cond(tf.equal(training_flag % 2, 0), lambda: img_inp_train, lambda: img_inp_val)
+        img_inp = tf.cond(tf.less(training_flag, 2), lambda: img_inp, lambda: img_inp_rgbd)
 
         net = PlaneNet({'img_inp': img_inp}, is_training=tf.equal(training_flag % 2, 0), options=options)
 
@@ -94,6 +96,10 @@ def build_loss_rgbd(global_pred_dict, local_pred_dict, deep_pred_dicts, global_g
         validDepthMask = tf.cast(tf.greater(global_gt_dict['depth'], 1e-4), tf.float32)
         depth_loss = tf.reduce_mean(tf.reduce_sum(tf.squared_difference(all_depths, global_gt_dict['depth']) * all_segmentations_softmax, axis=3, keep_dims=True) * validDepthMask) * 1000
 
+        if options.predictPixelwise == 1:
+            depth_loss += tf.reduce_mean(tf.squared_difference(global_pred_dict['non_plane_mask'], global_gt_dict['depth']) * validDepthMask) * 1000
+            pass
+        
         #non plane mask loss
         segmentation_loss = tf.reduce_mean(tf.slice(all_segmentations_softmax, [0, 0, 0, options.numOutputPlanes], [options.batchSize, HEIGHT, WIDTH, 1])) * 100
 
@@ -148,7 +154,7 @@ def build_loss(global_pred_dict, local_pred_dict, deep_pred_dicts, global_gt_dic
 
                 plane_gt_shuffled = tf.transpose(tf.matmul(global_gt_dict['plane'], forward_map, transpose_a=True), [0, 2, 1]) / tf.maximum(num_matches, 1e-4)
                 plane_confidence_gt = tf.cast(num_matches > 0.5, tf.float32)
-                plane_loss += tf.reduce_mean(tf.squared_difference(pred_dict['plane'], plane_gt_shuffled) * plane_confidence_gt) * 1000
+                plane_loss += tf.reduce_mean(tf.squared_difference(pred_dict['plane'], plane_gt_shuffled) * plane_confidence_gt) * 10000
 
 
                 #all segmentations is the concatenation of plane segmentations and non plane mask
@@ -200,8 +206,13 @@ def build_loss(global_pred_dict, local_pred_dict, deep_pred_dicts, global_gt_dic
         validDepthMask = tf.cast(tf.greater(global_gt_dict['depth'], 1e-4), tf.float32)
         depth_loss = tf.reduce_mean(tf.reduce_sum(tf.squared_difference(all_depths, global_gt_dict['depth']) * all_segmentations_softmax, axis=3, keep_dims=True) * validDepthMask) * 1000
 
-        #normal loss for non-plane region
-        normal_loss = tf.reduce_mean(tf.squared_difference(global_pred_dict['non_plane_normal'], global_gt_dict['normal']) * (1 - plane_mask)) * 1000
+        if options.predictPixelwise == 1:
+            depth_loss += tf.reduce_mean(tf.squared_difference(global_pred_dict['non_plane_mask'], global_gt_dict['depth']) * validDepthMask) * 1000
+            normal_loss = tf.reduce_mean(tf.squared_difference(global_pred_dict['non_plane_normal'], global_gt_dict['normal']) * validDepthMask) * 1000
+        else:
+            #normal loss for non-plane region
+            normal_loss = tf.reduce_mean(tf.squared_difference(global_pred_dict['non_plane_normal'], global_gt_dict['normal']) * (1 - plane_mask)) * 1000
+            pass
 
 
         #local loss
@@ -293,14 +304,6 @@ def build_loss(global_pred_dict, local_pred_dict, deep_pred_dicts, global_gt_dic
             #we predict boundaries directly for post-processing purpose
             boundary_loss += tf.reduce_mean(tf.losses.sigmoid_cross_entropy(logits=global_pred_dict['boundary'], multi_class_labels=boundary_gt, weights=tf.maximum(global_gt_dict['boundary'] * 3, 1))) * 1000
 
-
-        if options.diverseLoss:
-            plane_diff = tf.reduce_sum(tf.pow(tf.expand_dims(global_pred_dict['plane'], 1) - tf.expand_dims(global_pred_dict['plane'], 2), 2), axis=3)
-            plane_diff = tf.matrix_set_diag(plane_diff, tf.ones((options.batchSize, options.numOutputPlanes)))
-            minPlaneDiff = 0.1
-            diverse_loss += tf.reduce_mean(tf.clip_by_value(1 - plane_diff / minPlaneDiff, 0, 1)) * 10000
-            pass
-
 
         #regularization
         l2_losses = tf.add_n([options.l2Weight * tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'weights' in v.name])
@@ -392,7 +395,7 @@ def main(options):
         sess.run(init_op)
         if options.restore == 0:
             #fine-tune from DeepLab model
-            var_to_restore = [v for v in var_to_restore if 'res5d' not in v.name and 'segmentation' not in v.name and 'plane' not in v.name and 'deep_supervision' not in v.name and 'local' not in v.name and 'boundary' not in v.name and 'degridding' not in v.name]
+            var_to_restore = [v for v in var_to_restore if 'res5d' not in v.name and 'segmentation' not in v.name and 'plane' not in v.name and 'deep_supervision' not in v.name and 'local' not in v.name and 'boundary' not in v.name and 'degridding' not in v.name and 'res2a_branch2a' not in v.name and 'res2a_branch1' not in v.name]
             pretrained_model_loader = tf.train.Saver(var_to_restore)
             pretrained_model_loader.restore(sess,"../pretrained_models/deeplab_resnet.ckpt")
         elif options.restore == 1:
@@ -406,8 +409,11 @@ def main(options):
             loader = tf.train.Saver(var_to_restore)
             loader.restore(sess,"%s/checkpoint.ckpt"%(options.checkpoint_dir))
             sess.run(batchno.assign(1))
-        elif options.restore == 3:
+        elif options.restore == 3:            
             #restore the same model from standard training
+            if options.predictBoundary == 1:
+                var_to_restore = [v for v in var_to_restore if 'boundary' not in v.name]
+                pass            
             if options.predictConfidence == 1:
                 var_to_restore = [v for v in var_to_restore if 'confidence' not in v.name]
                 pass
@@ -454,6 +460,7 @@ def main(options):
                     batchType = 0
                     pass
 
+
                 _, total_loss, losses, losses_rgbd, summary_str = sess.run([train_op, loss, loss_dict, loss_dict_rgbd, summary_op], feed_dict = {training_flag: batchType})
                 writers[batchType].add_summary(summary_str, bno)
                 ema[batchType] = ema[batchType] * MOVING_AVERAGE_DECAY + total_loss
@@ -467,6 +474,14 @@ def main(options):
                     pass
 
                 print bno,'train', ema[0] / ema_acc[0], 'val', ema[1] / ema_acc[1], 'train rgbd', ema[2] / ema_acc[2], 'val rgbd', ema[3] / ema_acc[3], 'loss', total_loss, 'time', time.time()-t0
+
+                if np.random.random() < 0.01:
+                    if batchType < 2:
+                        print(losses)
+                    else:
+                        print(losses_rgbd)
+                        pass
+                    pass                
                 continue
 
         except tf.errors.OutOfRangeError:
@@ -1056,7 +1071,10 @@ def parse_args():
                         default=0, type=int)
     parser.add_argument('--predictConfidence', dest='predictConfidence',
                         help='whether predict plane confidence or not: [0, 1]',
-                        default=0, type=int)        
+                        default=0, type=int)
+    parser.add_argument('--predictPixelwise', dest='predictPixelwise',
+                        help='whether predict pixelwise depth or not: [0, 1]',
+                        default=0, type=int)    
     parser.add_argument('--fineTuningCheckpoint', dest='fineTuningCheckpoint',
                         help='specify the model for fine-tuning',
                         default='../PlaneSetGeneration/dump_planenet_diverse/train_planenet_diverse.ckpt', type=str)
@@ -1105,6 +1123,9 @@ def parse_args():
     if args.predictConfidence == 1:
         args.keyname += '_pc'
         pass    
+    if args.predictPixelwise == 1:
+        args.keyname += '_pp'
+        pass    
     if args.sameMatching == 0:
         args.keyname += '_sm0'
         pass

code/kaffe/caffe/resolver.pyc

@@ -33,8 +33,8 @@ def build_graph(img_inp_train, img_inp_val, img_inp_rgbd_train, img_inp_rgbd_val
         img_inp_rgbd = tf.cond(tf.equal(training_flag % 2, 0), lambda: img_inp_rgbd_train, lambda: img_inp_rgbd_val)
         img_inp = tf.cond(tf.less(training_flag, 2), lambda: tf.cond(tf.equal(training_flag % 2, 0), lambda: img_inp_train, lambda: img_inp_val), lambda: img_inp_rgbd)
 
-        #net = PlaneNet({'img_inp': img_inp}, is_training=tf.equal(training_flag % 2, 0), options=options)
-        net = PlaneNet({'img_inp': img_inp}, is_training=tf.equal(0, 0), options=options)
+        net = PlaneNet({'img_inp': img_inp}, is_training=tf.equal(training_flag % 2, 0), options=options)
+        #net = PlaneNet({'img_inp': img_inp}, is_training=tf.equal(0, 0), options=options)
         #global predictions
         plane_pred = net.layers['plane_pred']
 
@@ -707,10 +707,12 @@ def test(options):
             loader = tf.train.Saver(var_to_restore)
             loader.restore(sess, '../PlaneSetGeneration/dump_planenet_pixelwise/train_planenet_pixelwise.ckpt')
         else:
+            #var_to_restore = [v for v in var_to_restore if 'res4b22_relu_non_plane' not in v.name]
             loader = tf.train.Saver(var_to_restore)
             loader.restore(sess, "%s/checkpoint.ckpt"%(options.checkpoint_dir))
+            #loader.restore(sess, options.fineTuningCheckpoint)
             pass
-        #loader.restore(sess, options.fineTuningCheckpoint)
+        
 
         coord = tf.train.Coordinator()
         threads = tf.train.start_queue_runners(sess=sess, coord=coord)
@@ -732,7 +734,7 @@ def test(options):
             ranges = np.array([urange / imageWidth * 640 / focalLength, np.ones(urange.shape), -vrange / imageHeight * 480 / focalLength]).transpose([1, 2, 0])
 
 
-            for index in xrange(100):
+            for index in xrange(10):
                 print(('image', index))
                 t0=time.time()
 
@@ -996,7 +998,7 @@ def test(options):
             planeMasks = np.array(planeMasks)
             #predMasks = np.array(predMasks)
             evaluateDepths(predDepths, gtDepths, np.ones(planeMasks.shape, dtype=np.bool), planeMasks)
-            print(lossSum / num)
+            #print(lossSum / num)
             exit(1)
 
         except tf.errors.OutOfRangeError:
@@ -1423,7 +1425,8 @@ def parse_args():
     args = parser.parse_args()
     args.keyname = os.path.basename(__file__).rstrip('.py')
     args.keyname = args.keyname.replace('train_', '')
-
+    args.keyname = args.keyname.replace('_confidence', '')
+    
     if args.numOutputPlanes != 20:
         args.keyname += '_np' + str(args.numOutputPlanes)
         pass