Add FaceDetection TFJS implementation (#912)

* Add FaceDetection TFJS implementation

* Add predictIrises default

* Rename classes to clarify use cases

* Add TODO for changing face landmarks config

Author: ahmedsabie

face-landmarks-detection/.npmignore

@@ -0,0 +1,21 @@
+.DS_Store
+.yalc/
+.vscode/
+.rpt2_cache/
+demos/
+scripts/
+src/
+test_data/
+coverage/
+node_modules/
+karma.conf.js
+*.tgz
+.travis.yml
+.npmignore
+tslint.json
+yarn.lock
+yalc.lock
+cloudbuild.yml
+dist/*_test.js
+dist/*_test.js.map
+dist/test_util*

face-landmarks-detection/README.md

@@ -59,8 +59,8 @@ Example output:
       height: 246.87222836072945
     },
     keypoints: [
-      {x: 406.53152857172876, y: 256.8054528661723, name: "lips"},
-      {x: 406.544237446397, y: 230.06933367750395},
+      {x: 406.53152857172876, y: 256.8054528661723, z: 10.2, name: "lips"},
+      {x: 406.544237446397, y: 230.06933367750395, z: 8},
       ...
     ],
   }
@@ -69,7 +69,7 @@ Example output:
 
 The `box` represents the bounding box of the face in the image pixel space, with `xMin`, `xMax` denoting the x-bounds, `yMin`, `yMax` denoting the y-bounds, and `width`, `height` are the dimensions of the bounding box.
 
-For the `keypoints`, x and y represent the actual keypoint position in the image pixel space.
+For the `keypoints`, x and y represent the actual keypoint position in the image pixel space. z represents the depth with the center of the head being the origin, and the smaller the value the closer the keypoint is to the camera. The magnitude of z uses roughly the same scale as x.
 
 The name provides a label for some keypoint, such as 'lips', 'leftEye', etc. Note that not each keypoint will have a label.
 

face-landmarks-detection/src/create_detector.ts

@@ -15,9 +15,11 @@
  * =============================================================================
  */
 
-import {FaceDetector} from './face_detector';
-import {load as loadMediaPipeFaceMeshMediaPipeDetector} from './mediapipe/detector';
+import {FaceLandmarksDetector} from './face_landmarks_detector';
+import {load as loadMediaPipeFaceMeshMediaPipeLandmarksDetector} from './mediapipe/detector';
 import {MediaPipeFaceMeshMediaPipeModelConfig, MediaPipeFaceMeshModelConfig} from './mediapipe/types';
+import {loadMeshModel as loadMediaPipeFaceMeshTfjsLandmarksDetector} from './tfjs/detector';
+import {MediaPipeFaceMeshTfjsModelConfig} from './tfjs/types';
 import {SupportedModels} from './types';
 
 /**
@@ -28,18 +30,19 @@ import {SupportedModels} from './types';
  */
 export async function createDetector(
     model: SupportedModels,
-    modelConfig?: MediaPipeFaceMeshMediaPipeModelConfig):
-    Promise<FaceDetector> {
+    modelConfig?: MediaPipeFaceMeshMediaPipeModelConfig|
+    MediaPipeFaceMeshTfjsModelConfig): Promise<FaceLandmarksDetector> {
   switch (model) {
     case SupportedModels.MediaPipeFaceMesh:
       const config = modelConfig as MediaPipeFaceMeshModelConfig;
       let runtime;
       if (config != null) {
         if (config.runtime === 'tfjs') {
-          throw new Error('TFJS runtime is not yet supported.');
+          return loadMediaPipeFaceMeshTfjsLandmarksDetector(
+              config as MediaPipeFaceMeshTfjsModelConfig);
         }
         if (config.runtime === 'mediapipe') {
-          return loadMediaPipeFaceMeshMediaPipeDetector(
+          return loadMediaPipeFaceMeshMediaPipeLandmarksDetector(
               config as MediaPipeFaceMeshMediaPipeModelConfig);
         }
         runtime = config.runtime;

face-landmarks-detection/src/face_detector.ts renamed to face-landmarks-detection/src/face_landmarks_detector.ts

@@ -15,12 +15,13 @@
  * =============================================================================
  */
 import {MediaPipeFaceMeshMediaPipeEstimationConfig} from './mediapipe/types';
-import {Face, FaceDetectorInput} from './types';
+import {MediaPipeFaceMeshTfjsEstimationConfig} from './tfjs/types';
+import {Face, FaceLandmarksDetectorInput} from './types';
 
 /**
  * User-facing interface for all face pose detectors.
  */
-export interface FaceDetector {
+export interface FaceLandmarksDetector {
   /**
    * Finds faces in the input image.
    *
@@ -29,9 +30,9 @@ export interface FaceDetector {
    * @param estimationConfig common config for `estimateFaces`.
    */
   estimateFaces(
-      input: FaceDetectorInput,
-      estimationConfig?: MediaPipeFaceMeshMediaPipeEstimationConfig):
-      Promise<Face[]>;
+      input: FaceLandmarksDetectorInput,
+      estimationConfig?: MediaPipeFaceMeshMediaPipeEstimationConfig|
+      MediaPipeFaceMeshTfjsEstimationConfig): Promise<Face[]>;
 
   /**
    * Dispose the underlying models from memory.

face-landmarks-detection/src/index.ts

@@ -16,10 +16,11 @@
  */
 
 export {createDetector} from './create_detector';
-// FaceDetector class.
-export {FaceDetector} from './face_detector';
+// FaceLandmarksDetector class.
+export {FaceLandmarksDetector} from './face_landmarks_detector';
 // Entry point to create a new detector instance.
 export {MediaPipeFaceMeshMediaPipeEstimationConfig, MediaPipeFaceMeshMediaPipeModelConfig} from './mediapipe/types';
+export {MediaPipeFaceMeshTfjsEstimationConfig, MediaPipeFaceMeshTfjsModelConfig} from './tfjs/types';
 
 // Supported models enum.
 export * from './types';

face-landmarks-detection/src/mediapipe/README.md

@@ -63,7 +63,7 @@ Pass in `faceLandmarksDetection.SupportedModels.MediaPipeFaceMesh` from the
 
 *   *maxFaces*: Defaults to 1. The maximum number of faces that will be detected by the model. The number of returned faces can be less than the maximum (for example when no faces are present in the input). It is highly recommended to set this value to the expected max number of faces, otherwise the model will continue to search for the missing faces which can slow down the performance.
 
-*   *predictIrises*: If set to true, refines the landmark coordinates around the eyes and lips, and output additional landmarks around the irises.
+*   *refineLandmarks*: Defaults to false. If set to true, refines the landmark coordinates around the eyes and lips, and output additional landmarks around the irises.
 
 *   *solutionPath*: The path to where the wasm binary and model files are located.
 

face-landmarks-detection/src/mediapipe/constants.ts

@@ -20,7 +20,7 @@ export const DEFAULT_FACE_MESH_MODEL_CONFIG:
     MediaPipeFaceMeshMediaPipeModelConfig = {
       runtime: 'mediapipe',
       maxFaces: 1,
-      predictIrises: false
+      refineLandmarks: false
     };
 
 export const DEFAULT_FACE_MESH_ESTIMATION_CONFIG:

face-landmarks-detection/src/mediapipe/detector.ts

@@ -18,18 +18,19 @@ import * as faceMesh from '@mediapipe/face_mesh';
 import * as tf from '@tensorflow/tfjs-core';
 
 import {MEDIAPIPE_KEYPOINTS} from '../constants';
-import {FaceDetector} from '../face_detector';
+import {FaceLandmarksDetector} from '../face_landmarks_detector';
 import {Keypoint} from '../shared/calculators/interfaces/common_interfaces';
 import {landmarksToDetection} from '../shared/calculators/landmarks_to_detection';
-import {Face, FaceDetectorInput} from '../types';
+import {Face, FaceLandmarksDetectorInput} from '../types';
 
 import {validateModelConfig} from './detector_utils';
 import {MediaPipeFaceMeshMediaPipeEstimationConfig, MediaPipeFaceMeshMediaPipeModelConfig} from './types';
 
 /**
  * MediaPipe detector class.
  */
-class MediaPipeFaceMeshMediaPipeDetector implements FaceDetector {
+class MediaPipeFaceMeshMediaPipeLandmarksDetector implements
+    FaceLandmarksDetector {
   private readonly faceMeshSolution: faceMesh.FaceMesh;
 
   // This will be filled out by asynchronous calls to onResults. They will be
@@ -52,7 +53,7 @@ class MediaPipeFaceMeshMediaPipeDetector implements FaceDetector {
       }
     });
     this.faceMeshSolution.setOptions({
-      refineLandmarks: config.predictIrises,
+      refineLandmarks: config.refineLandmarks,
       selfieMode: this.selfieMode,
       maxNumFaces: config.maxFaces,
     });
@@ -80,6 +81,7 @@ class MediaPipeFaceMeshMediaPipeDetector implements FaceDetector {
       const keypoint: Keypoint = {
         x: landmark.x * this.width,
         y: landmark.y * this.height,
+        z: landmark.z * this.width,
       };
 
       const name = MEDIAPIPE_KEYPOINTS.get(i);
@@ -113,7 +115,7 @@ class MediaPipeFaceMeshMediaPipeDetector implements FaceDetector {
    * @return An array of `Face`s.
    */
   async estimateFaces(
-      input: FaceDetectorInput,
+      input: FaceLandmarksDetectorInput,
       estimationConfig?: MediaPipeFaceMeshMediaPipeEstimationConfig):
       Promise<Face[]> {
     if (estimationConfig && estimationConfig.flipHorizontal &&
@@ -158,9 +160,9 @@ class MediaPipeFaceMeshMediaPipeDetector implements FaceDetector {
  * `MediaPipeFaceMeshMediaPipeModelConfig` interface.
  */
 export async function load(modelConfig: MediaPipeFaceMeshMediaPipeModelConfig):
-    Promise<FaceDetector> {
+    Promise<FaceLandmarksDetector> {
   const config = validateModelConfig(modelConfig);
-  const detector = new MediaPipeFaceMeshMediaPipeDetector(config);
+  const detector = new MediaPipeFaceMeshMediaPipeLandmarksDetector(config);
   await detector.initialize();
   return detector;
 }

face-landmarks-detection/src/mediapipe/detector_utils.ts

@@ -33,8 +33,8 @@ export function validateModelConfig(
     config.maxFaces = DEFAULT_FACE_MESH_MODEL_CONFIG.maxFaces;
   }
 
-  if (config.predictIrises == null) {
-    config.predictIrises = DEFAULT_FACE_MESH_MODEL_CONFIG.predictIrises;
+  if (config.refineLandmarks == null) {
+    config.refineLandmarks = DEFAULT_FACE_MESH_MODEL_CONFIG.refineLandmarks;
   }
 
   return config;

face-landmarks-detection/src/mediapipe/mediapipe_test.ts

@@ -93,8 +93,8 @@ const EXPECTED_BOX: BoundingBox = {
 };
 
 export async function expectFaceMesh(
-    detector: faceDetection.FaceDetector, image: HTMLImageElement,
-    staticImageMode: boolean, predictIrises: boolean, numFrames: number,
+    detector: faceDetection.FaceLandmarksDetector, image: HTMLImageElement,
+    staticImageMode: boolean, refineLandmarks: boolean, numFrames: number,
     epsilon: number) {
   for (let i = 0; i < numFrames; ++i) {
     const result = await detector.estimateFaces(image, {staticImageMode});
@@ -112,14 +112,14 @@ export async function expectFaceMesh(
         result[0].keypoints.map(keypoint => [keypoint.x, keypoint.y]);
     expect(keypoints.length)
         .toBe(
-            predictIrises ? MEDIAPIPE_FACEMESH_NUM_KEYPOINTS_WITH_IRISES :
-                            MEDIAPIPE_FACEMESH_NUM_KEYPOINTS);
+            refineLandmarks ? MEDIAPIPE_FACEMESH_NUM_KEYPOINTS_WITH_IRISES :
+                              MEDIAPIPE_FACEMESH_NUM_KEYPOINTS);
 
     for (const [eyeIdx, gtLds] of EYE_INDICES_TO_LANDMARKS) {
       expectArraysClose(keypoints[eyeIdx], gtLds, epsilon);
     }
 
-    if (predictIrises) {
+    if (refineLandmarks) {
       for (const [irisIdx, gtLds] of IRIS_INDICES_TO_LANDMARKS) {
         expectArraysClose(keypoints[irisIdx], gtLds, epsilon);
       }
@@ -142,15 +142,15 @@ describeWithFlags('MediaPipe FaceMesh ', BROWSER_ENVS, () => {
   });
 
   async function expectMediaPipeFaceMesh(
-      image: HTMLImageElement, staticImageMode: boolean, predictIrises: boolean,
-      numFrames: number) {
+      image: HTMLImageElement, staticImageMode: boolean,
+      refineLandmarks: boolean, numFrames: number) {
     // Note: this makes a network request for model assets.
     const model = faceDetection.SupportedModels.MediaPipeFaceMesh;
     const detector = await faceDetection.createDetector(
-        model, {...MEDIAPIPE_MODEL_CONFIG, predictIrises});
+        model, {...MEDIAPIPE_MODEL_CONFIG, refineLandmarks});
 
     await expectFaceMesh(
-        detector, image, staticImageMode, predictIrises, numFrames,
+        detector, image, staticImageMode, refineLandmarks, numFrames,
         EPSILON_IMAGE);
   }
 

face-landmarks-detection/src/mediapipe/types.ts

@@ -22,7 +22,7 @@ import {EstimationConfig, ModelConfig} from '../types';
  */
 export interface MediaPipeFaceMeshModelConfig extends ModelConfig {
   runtime: 'mediapipe'|'tfjs';
-  predictIrises: boolean;
+  refineLandmarks: boolean;
 }
 
 export interface MediaPipeFaceMeshEstimationConfig extends EstimationConfig {}
@@ -32,7 +32,7 @@ export interface MediaPipeFaceMeshEstimationConfig extends EstimationConfig {}
  *
  * `runtime`: Must set to be 'mediapipe'.
  *
- * `predictIrises`: If set to true, refines the landmark coordinates around
+ * `refineLandmarks`: If set to true, refines the landmark coordinates around
  * the eyes and lips, and output additional landmarks around the irises.
  *
  * `solutionPath`: Optional. The path to where the wasm binary and model files

face-landmarks-detection/src/tfjs/README.md

@@ -0,0 +1,117 @@
+# MediaPipeFaceMesh
+
+MediaPipeHands-TFJS uses TF.js runtime to execute the model, the preprocessing and postprocessing steps.
+
+Please try our our live [demo](https://storage.googleapis.com/tfjs-models/demos/face-landmarks-detection/index.html?model=mediapipe_facemesh).
+In the runtime-backend dropdown, choose 'tfjs-webgl'.
+
+--------------------------------------------------------------------------------
+
+## Table of Contents
+
+1.  [Installation](#installation)
+2.  [Usage](#usage)
+
+## Installation
+
+To use MediaPipeFaceMesh, you need to first select a runtime (TensorFlow.js or MediaPipe).
+This guide is for TensorFlow.js
+runtime. The guide for MediaPipe runtime can be found
+[here](https://github.com/tensorflow/tfjs-models/tree/master/face-landmarks-detection/src/mediapipe).
+
+Via script tags:
+
+```html
+<!-- Require the peer dependencies of face-landmarks-detection. -->
+<script src="https://cdn.jsdelivr.net/npm/@mediapipe/face_mesh"></script>
+<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-core"></script>
+<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-converter"></script>
+
+<!-- You must explicitly require a TF.js backend if you're not using the TF.js union bundle. -->
+<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-webgl"></script>
+
+<script src="https://cdn.jsdelivr.net/npm/@tensorflow-models/face-landmarks-detection"></script>
+```
+
+Via npm:
+
+```sh
+yarn add @tensorflow-models/face-landmarks-detection
+yarn add @tensorflow/tfjs-core, @tensorflow/tfjs-converter
+yarn add @tensorflow/tfjs-backend-webgl
+yarn add @mediapipe/face_mesh
+```
+
+-----------------------------------------------------------------------
+## Usage
+
+If you are using the face-landmarks-detection API via npm, you need to import the libraries first.
+
+### Import the libraries
+
+```javascript
+import * as faceLandmarksDetection from '@tensorflow-models/face-landmarks-detection';
+import '@tensorflow/tfjs-core';
+// Register WebGL backend.
+import '@tensorflow/tfjs-backend-webgl';
+import '@mediapipe/face_mesh';
+```
+### Create a detector
+
+Pass in `handPoseDetection.SupportedModels.MediaPipeFaceMesh` from the
+`faceLandmarksDetection.SupportedModel` enum list along with a `detectorConfig` to the
+`createDetector` method to load and initialize the model.
+
+`detectorConfig` is an object that defines MediaPipeFaceMesh specific configurations for `MediaPipeFaceMeshTfjsModelConfig`:
+
+*   *runtime*: Must set to be 'tfjs'.
+
+*   *maxFaces*: Defaults to 1. The maximum number of faces that will be detected by the model. The number of returned faces can be less than the maximum (for example when no faces are present in the input). It is highly recommended to set this value to the expected max number of faces, otherwise the model will continue to search for the missing faces which can slow down the performance.
+
+*   *refineLandmarks*: Defaults to false. If set to true, refines the landmark coordinates around the eyes and lips, and output additional landmarks around the irises.
+
+*   *detectorModelUrl*: An optional string that specifies custom url of
+the detector model. This is useful for area/countries that don't have access to the model hosted on tf.hub. It also accepts `io.IOHandler` which can be used with
+[tfjs-react-native](https://github.com/tensorflow/tfjs/tree/master/tfjs-react-native)
+to load model from app bundle directory using
+[bundleResourceIO](https://github.com/tensorflow/tfjs/blob/master/tfjs-react-native/src/bundle_resource_io.ts#L169).
+*   *landmarkModelUrl* An optional string that specifies custom url of
+the landmark model. This is useful for area/countries that don't have access to the model hosted on tf.hub. It also accepts `io.IOHandler` which can be used with
+[tfjs-react-native](https://github.com/tensorflow/tfjs/tree/master/tfjs-react-native)
+to load model from app bundle directory using
+[bundleResourceIO](https://github.com/tensorflow/tfjs/blob/master/tfjs-react-native/src/bundle_resource_io.ts#L169).
+
+```javascript
+const model = faceLandmarksDetection.SupportedModels.MediaPipeFaceMesh;
+const detectorConfig = {
+  runtime: 'tfjs',
+};
+detector = await faceLandmarksDetection.createDetector(model, detectorConfig);
+```
+
+### Run inference
+
+Now you can use the detector to detect faces. The `estimateFaces` method
+accepts both image and video in many formats, including: `tf.Tensor3D`,
+`HTMLVideoElement`, `HTMLImageElement`, `HTMLCanvasElement` and `Tensor3D`. If you want more
+options, you can pass in a second `estimationConfig` parameter.
+
+`estimationConfig` is an object that defines MediaPipeFaceMesh specific configurations for `MediaPipeFaceMeshTfjsEstimationConfig`:
+
+*   *flipHorizontal*: Optional. Defaults to false. When image data comes from camera, the result has to flip horizontally.
+
+*   *staticImageMode*: Optional. Defaults to false. If set to true, face detection
+will run on every input image, otherwise if set to false then detection runs
+once and then the model simply tracks those landmarks without invoking
+another detection until it loses track of any of the faces (ideal for videos).
+
+The following code snippet demonstrates how to run the model inference:
+
+```javascript
+const estimationConfig = {flipHorizontal: false};
+const faces = await detector.estimateFaces(image, estimationConfig);
+```
+
+Please refer to the Face API
+[README](https://github.com/tensorflow/tfjs-models/blob/master/face-landmarks-detection/README.md#how-to-run-it)
+about the structure of the returned `faces` array.