Add FaceDetection TFJS implementation (#912)

* Add FaceDetection TFJS implementation

* Add predictIrises default

* Rename classes to clarify use cases

* Add TODO for changing face landmarks config

Author: ahmedsabie

face-landmarks-detection/.npmignore

@@ -0,0 +1,21 @@
+.DS_Store
+.yalc/
+.vscode/
+.rpt2_cache/
+demos/
+scripts/
+src/
+test_data/
+coverage/
+node_modules/
+karma.conf.js
+*.tgz
+.travis.yml
+.npmignore
+tslint.json
+yarn.lock
+yalc.lock
+cloudbuild.yml
+dist/*_test.js
+dist/*_test.js.map
+dist/test_util*

face-landmarks-detection/README.md

@@ -59,8 +59,8 @@ Example output:
       height: 246.87222836072945
     },
     keypoints: [
-      {x: 406.53152857172876, y: 256.8054528661723, name: "lips"},
-      {x: 406.544237446397, y: 230.06933367750395},
+      {x: 406.53152857172876, y: 256.8054528661723, z: 10.2, name: "lips"},
+      {x: 406.544237446397, y: 230.06933367750395, z: 8},
       ...
     ],
   }
@@ -69,7 +69,7 @@ Example output:
 
 The `box` represents the bounding box of the face in the image pixel space, with `xMin`, `xMax` denoting the x-bounds, `yMin`, `yMax` denoting the y-bounds, and `width`, `height` are the dimensions of the bounding box.
 
-For the `keypoints`, x and y represent the actual keypoint position in the image pixel space.
+For the `keypoints`, x and y represent the actual keypoint position in the image pixel space. z represents the depth with the center of the head being the origin, and the smaller the value the closer the keypoint is to the camera. The magnitude of z uses roughly the same scale as x.
 
 The name provides a label for some keypoint, such as 'lips', 'leftEye', etc. Note that not each keypoint will have a label.
 

face-landmarks-detection/src/create_detector.ts

@@ -15,9 +15,11 @@
  * =============================================================================
  */
 
-import {FaceDetector} from './face_detector';
-import {load as loadMediaPipeFaceMeshMediaPipeDetector} from './mediapipe/detector';
+import {FaceLandmarksDetector} from './face_landmarks_detector';
+import {load as loadMediaPipeFaceMeshMediaPipeLandmarksDetector} from './mediapipe/detector';
 import {MediaPipeFaceMeshMediaPipeModelConfig, MediaPipeFaceMeshModelConfig} from './mediapipe/types';
+import {loadMeshModel as loadMediaPipeFaceMeshTfjsLandmarksDetector} from './tfjs/detector';
+import {MediaPipeFaceMeshTfjsModelConfig} from './tfjs/types';
 import {SupportedModels} from './types';
 
 /**
@@ -28,18 +30,19 @@ import {SupportedModels} from './types';
  */
 export async function createDetector(
     model: SupportedModels,
-    modelConfig?: MediaPipeFaceMeshMediaPipeModelConfig):
-    Promise<FaceDetector> {
+    modelConfig?: MediaPipeFaceMeshMediaPipeModelConfig|
+    MediaPipeFaceMeshTfjsModelConfig): Promise<FaceLandmarksDetector> {
   switch (model) {
     case SupportedModels.MediaPipeFaceMesh:
       const config = modelConfig as MediaPipeFaceMeshModelConfig;
       let runtime;
       if (config != null) {
         if (config.runtime === 'tfjs') {
-          throw new Error('TFJS runtime is not yet supported.');
+          return loadMediaPipeFaceMeshTfjsLandmarksDetector(
+              config as MediaPipeFaceMeshTfjsModelConfig);
         }
         if (config.runtime === 'mediapipe') {
-          return loadMediaPipeFaceMeshMediaPipeDetector(
+          return loadMediaPipeFaceMeshMediaPipeLandmarksDetector(
               config as MediaPipeFaceMeshMediaPipeModelConfig);
         }
         runtime = config.runtime;

face-landmarks-detection/src/face_detector.ts renamed to face-landmarks-detection/src/face_landmarks_detector.ts

@@ -15,12 +15,13 @@
  * =============================================================================
  */
 import {MediaPipeFaceMeshMediaPipeEstimationConfig} from './mediapipe/types';
-import {Face, FaceDetectorInput} from './types';
+import {MediaPipeFaceMeshTfjsEstimationConfig} from './tfjs/types';
+import {Face, FaceLandmarksDetectorInput} from './types';
 
 /**
  * User-facing interface for all face pose detectors.
  */
-export interface FaceDetector {
+export interface FaceLandmarksDetector {
   /**
    * Finds faces in the input image.
    *
@@ -29,9 +30,9 @@ export interface FaceDetector {
    * @param estimationConfig common config for `estimateFaces`.
    */
   estimateFaces(
-      input: FaceDetectorInput,
-      estimationConfig?: MediaPipeFaceMeshMediaPipeEstimationConfig):
-      Promise<Face[]>;
+      input: FaceLandmarksDetectorInput,
+      estimationConfig?: MediaPipeFaceMeshMediaPipeEstimationConfig|
+      MediaPipeFaceMeshTfjsEstimationConfig): Promise<Face[]>;
 
   /**
    * Dispose the underlying models from memory.

face-landmarks-detection/src/index.ts

@@ -16,10 +16,11 @@
  */
 
 export {createDetector} from './create_detector';
-// FaceDetector class.
-export {FaceDetector} from './face_detector';
+// FaceLandmarksDetector class.
+export {FaceLandmarksDetector} from './face_landmarks_detector';
 // Entry point to create a new detector instance.
 export {MediaPipeFaceMeshMediaPipeEstimationConfig, MediaPipeFaceMeshMediaPipeModelConfig} from './mediapipe/types';
+export {MediaPipeFaceMeshTfjsEstimationConfig, MediaPipeFaceMeshTfjsModelConfig} from './tfjs/types';
 
 // Supported models enum.
 export * from './types';

face-landmarks-detection/src/mediapipe/README.md

@@ -63,7 +63,7 @@ Pass in `faceLandmarksDetection.SupportedModels.MediaPipeFaceMesh` from the
 
 *   *maxFaces*: Defaults to 1. The maximum number of faces that will be detected by the model. The number of returned faces can be less than the maximum (for example when no faces are present in the input). It is highly recommended to set this value to the expected max number of faces, otherwise the model will continue to search for the missing faces which can slow down the performance.
 
-*   *predictIrises*: If set to true, refines the landmark coordinates around the eyes and lips, and output additional landmarks around the irises.
+*   *refineLandmarks*: Defaults to false. If set to true, refines the landmark coordinates around the eyes and lips, and output additional landmarks around the irises.
 
 *   *solutionPath*: The path to where the wasm binary and model files are located.
 

face-landmarks-detection/src/mediapipe/constants.ts

@@ -20,7 +20,7 @@ export const DEFAULT_FACE_MESH_MODEL_CONFIG:
     MediaPipeFaceMeshMediaPipeModelConfig = {
       runtime: 'mediapipe',
       maxFaces: 1,
-      predictIrises: false
+      refineLandmarks: false
     };
 
 export const DEFAULT_FACE_MESH_ESTIMATION_CONFIG:

face-landmarks-detection/src/mediapipe/detector.ts

@@ -18,18 +18,19 @@ import * as faceMesh from '@mediapipe/face_mesh';
 import * as tf from '@tensorflow/tfjs-core';
 
 import {MEDIAPIPE_KEYPOINTS} from '../constants';
-import {FaceDetector} from '../face_detector';
+import {FaceLandmarksDetector} from '../face_landmarks_detector';
 import {Keypoint} from '../shared/calculators/interfaces/common_interfaces';
 import {landmarksToDetection} from '../shared/calculators/landmarks_to_detection';
-import {Face, FaceDetectorInput} from '../types';
+import {Face, FaceLandmarksDetectorInput} from '../types';
 
 import {validateModelConfig} from './detector_utils';
 import {MediaPipeFaceMeshMediaPipeEstimationConfig, MediaPipeFaceMeshMediaPipeModelConfig} from './types';
 
 /**
  * MediaPipe detector class.
  */
-class MediaPipeFaceMeshMediaPipeDetector implements FaceDetector {
+class MediaPipeFaceMeshMediaPipeLandmarksDetector implements
+    FaceLandmarksDetector {
   private readonly faceMeshSolution: faceMesh.FaceMesh;
 
   // This will be filled out by asynchronous calls to onResults. They will be
@@ -52,7 +53,7 @@ class MediaPipeFaceMeshMediaPipeDetector implements FaceDetector {
       }
     });
     this.faceMeshSolution.setOptions({
-      refineLandmarks: config.predictIrises,
+      refineLandmarks: config.refineLandmarks,
       selfieMode: this.selfieMode,
       maxNumFaces: config.maxFaces,
     });
@@ -80,6 +81,7 @@ class MediaPipeFaceMeshMediaPipeDetector implements FaceDetector {
       const keypoint: Keypoint = {
         x: landmark.x * this.width,
         y: landmark.y * this.height,
+        z: landmark.z * this.width,
       };
 
       const name = MEDIAPIPE_KEYPOINTS.get(i);
@@ -113,7 +115,7 @@ class MediaPipeFaceMeshMediaPipeDetector implements FaceDetector {
    * @return An array of `Face`s.
    */
   async estimateFaces(
-      input: FaceDetectorInput,
+      input: FaceLandmarksDetectorInput,
       estimationConfig?: MediaPipeFaceMeshMediaPipeEstimationConfig):
       Promise<Face[]> {
     if (estimationConfig && estimationConfig.flipHorizontal &&
@@ -158,9 +160,9 @@ class MediaPipeFaceMeshMediaPipeDetector implements FaceDetector {
  * `MediaPipeFaceMeshMediaPipeModelConfig` interface.
  */
 export async function load(modelConfig: MediaPipeFaceMeshMediaPipeModelConfig):
-    Promise<FaceDetector> {
+    Promise<FaceLandmarksDetector> {
   const config = validateModelConfig(modelConfig);
-  const detector = new MediaPipeFaceMeshMediaPipeDetector(config);
+  const detector = new MediaPipeFaceMeshMediaPipeLandmarksDetector(config);
   await detector.initialize();
   return detector;
 }

face-landmarks-detection/src/mediapipe/detector_utils.ts

@@ -33,8 +33,8 @@ export function validateModelConfig(
     config.maxFaces = DEFAULT_FACE_MESH_MODEL_CONFIG.maxFaces;
   }
 
-  if (config.predictIrises == null) {
-    config.predictIrises = DEFAULT_FACE_MESH_MODEL_CONFIG.predictIrises;
+  if (config.refineLandmarks == null) {
+    config.refineLandmarks = DEFAULT_FACE_MESH_MODEL_CONFIG.refineLandmarks;
   }
 
   return config;

face-landmarks-detection/src/mediapipe/mediapipe_test.ts

@@ -93,8 +93,8 @@ const EXPECTED_BOX: BoundingBox = {
 };
 
 export async function expectFaceMesh(
-    detector: faceDetection.FaceDetector, image: HTMLImageElement,
-    staticImageMode: boolean, predictIrises: boolean, numFrames: number,
+    detector: faceDetection.FaceLandmarksDetector, image: HTMLImageElement,
+    staticImageMode: boolean, refineLandmarks: boolean, numFrames: number,
     epsilon: number) {
   for (let i = 0; i < numFrames; ++i) {
     const result = await detector.estimateFaces(image, {staticImageMode});
@@ -112,14 +112,14 @@ export async function expectFaceMesh(
         result[0].keypoints.map(keypoint => [keypoint.x, keypoint.y]);
     expect(keypoints.length)
         .toBe(
-            predictIrises ? MEDIAPIPE_FACEMESH_NUM_KEYPOINTS_WITH_IRISES :
-                            MEDIAPIPE_FACEMESH_NUM_KEYPOINTS);
+            refineLandmarks ? MEDIAPIPE_FACEMESH_NUM_KEYPOINTS_WITH_IRISES :
+                              MEDIAPIPE_FACEMESH_NUM_KEYPOINTS);
 
     for (const [eyeIdx, gtLds] of EYE_INDICES_TO_LANDMARKS) {
       expectArraysClose(keypoints[eyeIdx], gtLds, epsilon);
     }
 
-    if (predictIrises) {
+    if (refineLandmarks) {
       for (const [irisIdx, gtLds] of IRIS_INDICES_TO_LANDMARKS) {
         expectArraysClose(keypoints[irisIdx], gtLds, epsilon);
       }
@@ -142,15 +142,15 @@ describeWithFlags('MediaPipe FaceMesh ', BROWSER_ENVS, () => {
   });
 
   async function expectMediaPipeFaceMesh(
-      image: HTMLImageElement, staticImageMode: boolean, predictIrises: boolean,
-      numFrames: number) {
+      image: HTMLImageElement, staticImageMode: boolean,
+      refineLandmarks: boolean, numFrames: number) {
     // Note: this makes a network request for model assets.
     const model = faceDetection.SupportedModels.MediaPipeFaceMesh;
     const detector = await faceDetection.createDetector(
-        model, {...MEDIAPIPE_MODEL_CONFIG, predictIrises});
+        model, {...MEDIAPIPE_MODEL_CONFIG, refineLandmarks});
 
     await expectFaceMesh(
-        detector, image, staticImageMode, predictIrises, numFrames,
+        detector, image, staticImageMode, refineLandmarks, numFrames,
         EPSILON_IMAGE);
   }