Add lite hands model support to TFJS API (#848)

* Add lite hands model support to TFJS API

* Move config model type check to validator

Author: ahmedsabie

hand-detection/src/tfjs/README.md

@@ -62,6 +62,9 @@ Pass in `handDetection.SupportedModels.MediaPipeHands` from the
 
 *   *maxHands*: Defaults to 2. The maximum number of hands that will be detected by the model. The number of returned hands can be less than the maximum (for example when no hands are present in the input).
 
+*   *modelType*: specify which variant to load from `MediaPipeHandsModelType` (i.e.,
+    'lite', 'full'). If unset, the default is 'full'.
+
 *   *detectorModelUrl*: An optional string that specifies custom url of
 the detector model. This is useful for area/countries that don't have access to the model hosted on tf.hub.
 *   *landmarkModelUrl* An optional string that specifies custom url of

hand-detection/src/tfjs/constants.ts

@@ -15,13 +15,15 @@
  * =============================================================================
  */
 
-import { AnchorConfig, ImageToTensorConfig, RectTransformationConfig, TensorsToDetectionsConfig, TensorsToLandmarksConfig } from '../shared/calculators/interfaces/config_interfaces';
-import { MediaPipeHandsTfjsEstimationConfig, MediaPipeHandsTfjsModelConfig } from './types';
+import {AnchorConfig, ImageToTensorConfig, RectTransformationConfig, TensorsToDetectionsConfig, TensorsToLandmarksConfig} from '../shared/calculators/interfaces/config_interfaces';
+import {MediaPipeHandsTfjsEstimationConfig, MediaPipeHandsTfjsModelConfig} from './types';
 
 export const DEFAULT_MPHANDS_DETECTOR_MODEL_URL =
-  'https://storage.googleapis.com/tfjs-testing/hand-detection/handdetector/model.json';
-export const DEFAULT_MPHANDS_LANDMARK_MODEL_URL =
-  'https://storage.googleapis.com/tfjs-testing/hand-detection/handskeleton/model.json';
+    'https://storage.googleapis.com/tfjs-testing/hand-detection/handdetector/model.json';
+export const DEFAULT_MPHANDS_LANDMARK_MODEL_URL_LITE =
+    'https://storage.googleapis.com/tfjs-testing/hand-detection/handskeleton_lite/model.json';
+export const DEFAULT_MPHANDS_LANDMARK_MODEL_URL_FULL =
+    'https://storage.googleapis.com/tfjs-testing/hand-detection/handskeleton_full/model.json';
 export const MPHANDS_DETECTOR_ANCHOR_CONFIGURATION: AnchorConfig = {
   reduceBoxesInLowestLayer: false,
   interpolatedScaleAspectRatio: 1.0,
@@ -40,62 +42,63 @@ export const MPHANDS_DETECTOR_ANCHOR_CONFIGURATION: AnchorConfig = {
 };
 export const DEFAULT_MPHANDS_MODEL_CONFIG: MediaPipeHandsTfjsModelConfig = {
   runtime: 'tfjs',
+  modelType: 'full',
   maxHands: 2,
   detectorModelUrl: DEFAULT_MPHANDS_DETECTOR_MODEL_URL,
-  landmarkModelUrl: DEFAULT_MPHANDS_LANDMARK_MODEL_URL
+  landmarkModelUrl: DEFAULT_MPHANDS_LANDMARK_MODEL_URL_FULL
 };
 export const DEFAULT_MPHANDS_ESTIMATION_CONFIG:
-  MediaPipeHandsTfjsEstimationConfig = {
-  flipHorizontal: false,
-  staticImageMode: false
-};
+    MediaPipeHandsTfjsEstimationConfig = {
+      flipHorizontal: false,
+      staticImageMode: false
+    };
 export const MPHANDS_TENSORS_TO_DETECTION_CONFIGURATION:
-  TensorsToDetectionsConfig = {
-  applyExponentialOnBoxSize: false,
-  flipVertically: false,
-  ignoreClasses: [] as number[],
-  numClasses: 1,
-  numBoxes: 896,
-  numCoords: 18,
-  boxCoordOffset: 0,
-  keypointCoordOffset: 4,
-  numKeypoints: 7,
-  numValuesPerKeypoint: 2,
-  sigmoidScore: true,
-  scoreClippingThresh: 100.0,
-  reverseOutputOrder: true,
-  xScale: 128.0,
-  yScale: 128.0,
-  hScale: 128.0,
-  wScale: 128.0,
-  minScoreThresh: 0.5
-};
+    TensorsToDetectionsConfig = {
+      applyExponentialOnBoxSize: false,
+      flipVertically: false,
+      ignoreClasses: [] as number[],
+      numClasses: 1,
+      numBoxes: 896,
+      numCoords: 18,
+      boxCoordOffset: 0,
+      keypointCoordOffset: 4,
+      numKeypoints: 7,
+      numValuesPerKeypoint: 2,
+      sigmoidScore: true,
+      scoreClippingThresh: 100.0,
+      reverseOutputOrder: true,
+      xScale: 128.0,
+      yScale: 128.0,
+      hScale: 128.0,
+      wScale: 128.0,
+      minScoreThresh: 0.5
+    };
 export const MPHANDS_DETECTOR_NON_MAX_SUPPRESSION_CONFIGURATION = {
   minScoreThreshold: -1.0,
   minSuppressionThreshold: 0.3
 };
 export const MPHANDS_DETECTOR_RECT_TRANSFORMATION_CONFIG:
-  RectTransformationConfig = {
-  shiftX: 0,
-  shiftY: -0.5,
-  scaleX: 2.6,
-  scaleY: 2.6,
-  squareLong: true
-};
+    RectTransformationConfig = {
+      shiftX: 0,
+      shiftY: -0.5,
+      scaleX: 2.6,
+      scaleY: 2.6,
+      squareLong: true
+    };
 export const MPHANDS_LANDMARK_RECT_TRANSFORMATION_CONFIG:
-  RectTransformationConfig = {
-  shiftX: 0,
-  shiftY: -0.1,
-  scaleX: 2.0,
-  scaleY: 2.0,
-  squareLong: true
-};
+    RectTransformationConfig = {
+      shiftX: 0,
+      shiftY: -0.1,
+      scaleX: 2.0,
+      scaleY: 2.0,
+      squareLong: true
+    };
 export const MPHANDS_DETECTOR_IMAGE_TO_TENSOR_CONFIG: ImageToTensorConfig = {
-  inputResolution: { width: 128, height: 128 },
+  inputResolution: {width: 128, height: 128},
   keepAspectRatio: true
 };
 export const MPHANDS_LANDMARK_IMAGE_TO_TENSOR_CONFIG: ImageToTensorConfig = {
-  inputResolution: { width: 224, height: 224 },
+  inputResolution: {width: 224, height: 224},
   keepAspectRatio: true
 };
 export const MPHANDS_HAND_PRESENCE_SCORE = 0.5;

hand-detection/src/tfjs/detector.ts

@@ -301,14 +301,14 @@ class MediaPipeHandsTfjsDetector implements HandDetector {
     // outputs a list of tensors representing, for instance, detection
     // boxes/keypoints and scores.
     // The model returns 3 tensors with the following shape:
-    // conv_landmarks: This tensor (shape: [1, 63]) represents 21 3-d
+    // Identity_2:0: This tensor (shape: [1, 63]) represents 21 3-d
     // keypoints.
     // Identity_1:0: This tensor (shape: [1, 1]) represents the
     // confidence score of the presence of a hand.
     // Identity:0: This tensor (shape: [1, 1]) represents the classication
     // score of handedness
     const landmarkResult = this.landmarkModel.execute(imageValueShifted, [
-      'conv_landmarks', 'Identity_1:0', 'Identity:0'
+      'Identity_2:0', 'Identity_1:0', 'Identity:0'
     ]) as tf.Tensor[];
 
     const landmarkTensor = landmarkResult[0] as tf.Tensor2D,

hand-detection/src/tfjs/detector_utils.ts

@@ -15,7 +15,7 @@
  * =============================================================================
  */
 
-import {DEFAULT_MPHANDS_ESTIMATION_CONFIG, DEFAULT_MPHANDS_MODEL_CONFIG} from './constants';
+import {DEFAULT_MPHANDS_ESTIMATION_CONFIG, DEFAULT_MPHANDS_LANDMARK_MODEL_URL_FULL, DEFAULT_MPHANDS_LANDMARK_MODEL_URL_LITE, DEFAULT_MPHANDS_MODEL_CONFIG} from './constants';
 import {MediaPipeHandsTfjsEstimationConfig, MediaPipeHandsTfjsModelConfig} from './types';
 
 export function validateModelConfig(modelConfig: MediaPipeHandsTfjsModelConfig):
@@ -32,12 +32,29 @@ export function validateModelConfig(modelConfig: MediaPipeHandsTfjsModelConfig):
     config.maxHands = DEFAULT_MPHANDS_MODEL_CONFIG.maxHands;
   }
 
+  if (config.modelType == null) {
+    config.modelType = DEFAULT_MPHANDS_MODEL_CONFIG.modelType;
+  }
+
+  if (config.modelType !== 'lite' && config.modelType !== 'full') {
+    throw new Error(
+        `Model type must be one of lite or full, but got ${config.modelType}`);
+  }
+
   if (config.detectorModelUrl == null) {
     config.detectorModelUrl = DEFAULT_MPHANDS_MODEL_CONFIG.detectorModelUrl;
   }
 
   if (config.landmarkModelUrl == null) {
-    config.landmarkModelUrl = DEFAULT_MPHANDS_MODEL_CONFIG.landmarkModelUrl;
+    switch (config.modelType) {
+      case 'lite':
+        config.landmarkModelUrl = DEFAULT_MPHANDS_LANDMARK_MODEL_URL_LITE;
+        break;
+      case 'full':
+      default:
+        config.landmarkModelUrl = DEFAULT_MPHANDS_LANDMARK_MODEL_URL_FULL;
+        break;
+    }
   }
 
   return config;

hand-detection/src/tfjs/types.ts

@@ -22,6 +22,10 @@ import {MediaPipeHandsEstimationConfig, MediaPipeHandsModelConfig} from '../medi
  *
  * `runtime`: Must set to be 'tfjs'.
  *
+ * `modelType`: Optional. Possible values: 'lite'|'full'. Defaults to
+ * 'full'. Landmark accuracy as well as inference latency generally go up with
+ * the increasing model complexity (lite to full).
+ *
  * `detectorModelUrl`: Optional. An optional string that specifies custom url of
  * the detector model. This is useful for area/countries that don't have access
  * to the model hosted on tf.hub.