Add 3D keypoints to TFJS hand detection (#850)

ahmedsabie · web-flow · commit 250b679bace4 · 2021-10-29T23:31:27.000-04:00
* Add 3D keypoints to TFJS hand detection

* Fix CSS typo

* Add maxNumHands controller
diff --git a/hand-detection/demos/live_video/index.html b/hand-detection/demos/live_video/index.html
@@ -27,10 +27,13 @@
       position: relative;
       margin: 0;
     }
-    #canvas-wrapper,
-    #scatter-gl-container {
+    #canvas-wrapper {
       position: relative;
     }
+    #scatter-gl-container-left, #scatter-gl-container-right {
+      position: relative;
+      float: left;
+    }
   </style>
 </head>
 <body>
@@ -48,7 +51,9 @@
           ">
         </video>
       </div>
-      <div id="scatter-gl-container"></div>
+      <div id="scatter-gl-container-left"></div>
+      </div>
+      <div id="scatter-gl-container-right"></div>
     </div>
   </div>
 </body>
diff --git a/hand-detection/demos/live_video/src/camera.js b/hand-detection/demos/live_video/src/camera.js
@@ -21,7 +21,7 @@ import {isMobile} from './util';
 
 // These anchor points allow the hand pointcloud to resize according to its
 // position in the input.
-const ANCHOR_POINTS = [[0, 0, 0], [0, 1, 0], [-1, 0, 0], [-1, -1, 0]];
+const ANCHOR_POINTS = [[0, 0, 0], [0, 0.1, 0], [-0.1, 0, 0], [-0.1, -0.1, 0]];
 
 const fingerLookupIndices = {
   thumb: [0, 1, 2, 3, 4],
@@ -39,18 +39,27 @@ const connections = [
   [0, 17], [17, 18],[18, 19], [19,20]
 ];
 
+function createScatterGLContext(selectors) {
+  const scatterGLEl = document.querySelector(selectors);
+  return {
+    scatterGLEl,
+    scatterGL: new scatter.ScatterGL(scatterGLEl, {
+      'rotateOnStart': true,
+      'selectEnabled': false,
+      'styles': {polyline: {defaultOpacity: 1, deselectedOpacity: 1}}
+    }),
+    scatterGLHasInitialized: false,
+  };
+}
+
+const scatterGLCtxtLeftHand = createScatterGLContext('#scatter-gl-container-left');
+const scatterGLCtxtRightHand = createScatterGLContext('#scatter-gl-container-right');
+
 export class Camera {
   constructor() {
     this.video = document.getElementById('video');
     this.canvas = document.getElementById('output');
     this.ctx = this.canvas.getContext('2d');
-    this.scatterGLEl = document.querySelector('#scatter-gl-container');
-    this.scatterGL = new scatter.ScatterGL(this.scatterGLEl, {
-      'rotateOnStart': true,
-      'selectEnabled': false,
-      'styles': {polyline: {defaultOpacity: 1, deselectedOpacity: 1}}
-    });
-    this.scatterGLHasInitialized = false;
   }
 
   /**
@@ -108,12 +117,14 @@ export class Camera {
     camera.ctx.translate(camera.video.videoWidth, 0);
     camera.ctx.scale(-1, 1);
 
-    camera.scatterGLEl.style =
-        `width: ${videoWidth}px; height: ${videoHeight}px;`;
-    camera.scatterGL.resize();
+    for (const ctxt of [scatterGLCtxtLeftHand, scatterGLCtxtRightHand]) {
+      ctxt.scatterGLEl.style =
+          `width: ${videoWidth / 2}px; height: ${videoHeight / 2}px;`;
+      ctxt.scatterGL.resize();
 
-    camera.scatterGLEl.style.display =
-        params.STATE.modelConfig.render3D ? 'inline-block' : 'none';
+      ctxt.scatterGLEl.style.display =
+          params.STATE.modelConfig.render3D ? 'inline-block' : 'none';
+    }
 
     return camera;
   }
@@ -132,31 +143,53 @@ export class Camera {
    * @param hands A list of hands to render.
    */
   drawResults(hands) {
-    for (const hand of hands) {
-      this.drawResult(hand);
+    // Sort by right to left hands.
+    hands.sort((hand1, hand2) => {
+      if (hand1.handedness < hand2.handedness) return 1;
+      if (hand1.handedness > hand2.handedness) return -1;
+      return 0;
+    });
+
+    // Pad hands to clear empty scatter GL plots.
+    while (hands.length < 2) hands.push({});
+
+    for (let i = 0; i < hands.length; ++i) {
+      // Third hand and onwards scatterGL context is set to null since we
+      // don't render them.
+      const ctxt = [scatterGLCtxtLeftHand, scatterGLCtxtRightHand][i];
+      this.drawResult(hands[i], ctxt);
     }
   }
 
   /**
    * Draw the keypoints on the video.
    * @param hand A hand with keypoints to render.
+   * @param ctxt Scatter GL context to render 3D keypoints to.
    */
-  drawResult(hand) {
+  drawResult(hand, ctxt) {
     if (hand.keypoints != null) {
-      this.drawKeypoints(hand.keypoints);
+      this.drawKeypoints(hand.keypoints, hand.handedness);
+    }
+    // Don't render 3D hands after first two.
+    if (ctxt == null) {
+      return;
     }
     if (hand.keypoints3D != null && params.STATE.modelConfig.render3D) {
-      this.drawKeypoints3D(hand.keypoints3D);
+      this.drawKeypoints3D(hand.keypoints3D, hand.handedness, ctxt);
+    } else {
+      // Clear scatter plot.
+      this.drawKeypoints3D([], '', ctxt);
     }
   }
 
   /**
    * Draw the keypoints on the video.
    * @param keypoints A list of keypoints.
+   * @param handedness Label of hand (either Left or Right).
    */
-   drawKeypoints(keypoints) {
+   drawKeypoints(keypoints, handedness) {
     const keypointsArray = keypoints;
-    this.ctx.fillStyle = 'Red';
+    this.ctx.fillStyle = handedness === 'Left' ? 'Red' : 'Blue';
     this.ctx.strokeStyle = 'White';
     this.ctx.lineWidth = params.DEFAULT_LINE_WIDTH;
 
@@ -194,29 +227,29 @@ export class Camera {
     this.ctx.fill();
   }
 
-  drawKeypoints3D(keypoints) {
+  drawKeypoints3D(keypoints, handedness, ctxt) {
     const scoreThreshold = params.STATE.modelConfig.scoreThreshold || 0;
     const pointsData =
         keypoints.map(keypoint => ([-keypoint.x, -keypoint.y, -keypoint.z]));
 
     const dataset =
         new scatter.ScatterGL.Dataset([...pointsData, ...ANCHOR_POINTS]);
 
-    this.scatterGL.setPointColorer((i) => {
+    ctxt.scatterGL.setPointColorer((i) => {
       if (keypoints[i] == null || keypoints[i].score < scoreThreshold) {
         // hide anchor points and low-confident points.
         return '#ffffff';
       }
-      return '#ff0000' /* Red */;
+      return handedness === 'Left' ? '#ff0000' : '#0000ff';
     });
 
-    if (!this.scatterGLHasInitialized) {
-      this.scatterGL.render(dataset);
+    if (!ctxt.scatterGLHasInitialized) {
+      ctxt.scatterGL.render(dataset);
     } else {
-      this.scatterGL.updateDataset(dataset);
+      ctxt.scatterGL.updateDataset(dataset);
     }
     const sequences = connections.map(pair => ({indices: pair}));
-    this.scatterGL.setSequences(sequences);
-    this.scatterGLHasInitialized = true;
+    ctxt.scatterGL.setSequences(sequences);
+    ctxt.scatterGLHasInitialized = true;
   }
 }
diff --git a/hand-detection/demos/live_video/src/index.js b/hand-detection/demos/live_video/src/index.js
@@ -45,14 +45,14 @@ async function createDetector() {
         return handdetection.createDetector(STATE.model, {
           runtime,
           modelType: STATE.modelConfig.type,
-          maxHands: STATE.modelConfig.maxHands,
+          maxHands: STATE.modelConfig.maxNumHands,
           solutionPath: `https://cdn.jsdelivr.net/npm/@mediapipe/hands@${mpHands.VERSION}`
         });
       } else if (runtime === 'tfjs') {
         return handdetection.createDetector(STATE.model, {
           runtime,
           modelType: STATE.modelConfig.type,
-          maxHands: STATE.modelConfig.maxHands
+          maxHands: STATE.modelConfig.maxNumHands
         });
       }
   }
diff --git a/hand-detection/demos/live_video/src/option_panel.js b/hand-detection/demos/live_video/src/option_panel.js
@@ -49,6 +49,7 @@ export async function setupDatGui(urlParams) {
 
   const model = urlParams.get('model');
   let type = urlParams.get('type');
+  let maxNumHands = urlParams.get('maxNumHands');
 
   switch (model) {
     case 'mediapipe_hands':
@@ -57,6 +58,10 @@ export async function setupDatGui(urlParams) {
         // Nulify invalid value.
         type = null;
       }
+      if (maxNumHands == null || maxNumHands < 1 ) {
+        // Nulify invalid value.
+        maxNumHands = 2;
+      }
       break;
     default:
       alert(`${urlParams.get('model')}`);
@@ -72,7 +77,7 @@ export async function setupDatGui(urlParams) {
     showBackendConfigs(backendFolder);
   });
 
-  showModelConfigs(modelFolder, type);
+  showModelConfigs(modelFolder, type, maxNumHands);
 
   modelFolder.open();
 
@@ -106,7 +111,7 @@ async function showBackendConfigs(folderController) {
   await showFlagSettings(folderController, params.STATE.backend);
 }
 
-function showModelConfigs(folderController, type) {
+function showModelConfigs(folderController, type, maxNumHands) {
   // Clean up model configs for the previous model.
   // The first constroller under the `folderController` is the model
   // selection.
@@ -119,7 +124,7 @@ function showModelConfigs(folderController, type) {
 
   switch (params.STATE.model) {
     case handdetection.SupportedModels.MediaPipeHands:
-      addMediaPipeHandsControllers(folderController, type);
+      addMediaPipeHandsControllers(folderController, type, maxNumHands);
       break;
     default:
       alert(`Model ${params.STATE.model} is not supported.`);
@@ -128,9 +133,10 @@ function showModelConfigs(folderController, type) {
 
 // The MediaPipeHands model config folder contains options for MediaPipeHands config
 // settings.
-function addMediaPipeHandsControllers(modelConfigFolder, type) {
+function addMediaPipeHandsControllers(modelConfigFolder, type, maxNumHands) {
   params.STATE.modelConfig = {...params.MEDIAPIPE_HANDS_CONFIG};
   params.STATE.modelConfig.type = type != null ? type : 'full';
+  params.STATE.modelConfig.maxNumHands = maxNumHands != null ? maxNumHands : 2;
 
   const typeController = modelConfigFolder.add(
       params.STATE.modelConfig, 'type', ['lite', 'full']);
@@ -140,10 +146,20 @@ function addMediaPipeHandsControllers(modelConfigFolder, type) {
     params.STATE.isModelChanged = true;
   });
 
+  const maxNumHandsController = modelConfigFolder.add(
+    params.STATE.modelConfig, 'maxNumHands', 1, 10).step(1);
+    maxNumHandsController.onChange(_ => {
+    // Set isModelChanged to true, so that we don't render any result during
+    // changing models.
+    params.STATE.isModelChanged = true;
+  });
+
   const render3DController =
       modelConfigFolder.add(params.STATE.modelConfig, 'render3D');
   render3DController.onChange(render3D => {
-    document.querySelector('#scatter-gl-container').style.display =
+    document.querySelector('#scatter-gl-container-left').style.display =
+        render3D ? 'inline-block' : 'none';
+    document.querySelector('#scatter-gl-container-right').style.display =
         render3D ? 'inline-block' : 'none';
   });
 }
diff --git a/hand-detection/demos/live_video/src/params.js b/hand-detection/demos/live_video/src/params.js
@@ -31,7 +31,6 @@ export const STATE = {
   modelConfig: {}
 };
 export const MEDIAPIPE_HANDS_CONFIG = {
-  maxHands: 2,
   type: 'full',
   render3D: true
 };
diff --git a/hand-detection/src/tfjs/constants.ts b/hand-detection/src/tfjs/constants.ts
@@ -112,3 +112,12 @@ export const MPHANDS_TENSORS_TO_LANDMARKS_CONFIG: TensorsToLandmarksConfig = {
   flipHorizontally: false,
   flipVertically: false
 };
+export const MPHANDS_TENSORS_TO_WORLD_LANDMARKS_CONFIG:
+    TensorsToLandmarksConfig = {
+      numLandmarks: 21,
+      inputImageWidth: 1,
+      inputImageHeight: 1,
+      visibilityActivation: 'none',
+      flipHorizontally: false,
+      flipVertically: false
+    };
diff --git a/hand-detection/src/tfjs/detector.ts b/hand-detection/src/tfjs/detector.ts
