+ setMaskSettings({
+ type: thumbnail,
+ index: index,
+ })
+ }
+ >
+ {ThumbnailIcon ? (
+
+
+
+ ) : (
+

+ )}
+
{name}
+
+ );
+}
From dd5b15fc7b31d6feaf5ed1b89933fc40de04c778 Mon Sep 17 00:00:00 2001
From: Eluda <111eluda111@gmail.com>
Date: Sun, 13 Nov 2022 19:55:23 +0100
Subject: [PATCH 06/16] Build face masks video processor.
---
src/processors/face-mask/MaskProcessor.ts | 229 +
src/processors/face-mask/utils/facemesh.ts | 6344 +++++++++++++++++++
src/processors/face-mask/utils/matrix.ts | 1157 ++++
src/processors/face-mask/utils/render-2d.ts | 452 ++
src/processors/face-mask/utils/shaders.ts | 68 +
src/processors/face-mask/utils/textures.ts | 53 +
src/processors/face-mask/utils/webgl.ts | 170 +
7 files changed, 8473 insertions(+)
create mode 100644 src/processors/face-mask/MaskProcessor.ts
create mode 100644 src/processors/face-mask/utils/facemesh.ts
create mode 100644 src/processors/face-mask/utils/matrix.ts
create mode 100644 src/processors/face-mask/utils/render-2d.ts
create mode 100644 src/processors/face-mask/utils/shaders.ts
create mode 100644 src/processors/face-mask/utils/textures.ts
create mode 100644 src/processors/face-mask/utils/webgl.ts
diff --git a/src/processors/face-mask/MaskProcessor.ts b/src/processors/face-mask/MaskProcessor.ts
new file mode 100644
index 000000000..c19a0d56f
--- /dev/null
+++ b/src/processors/face-mask/MaskProcessor.ts
@@ -0,0 +1,229 @@
+import { Processor } from '@twilio/video-processors/es5/processors/Processor';
+import '@tensorflow/tfjs-backend-webgl';
+import * as tfjsWasm from '@tensorflow/tfjs-backend-wasm';
+import {
+ FaceLandmarksDetector,
+ createDetector as loadModel,
+ SupportedModels,
+} from '@tensorflow-models/face-landmarks-detection';
+import { create_image_texture2, create_image_texture_from_file } from './utils/textures';
+import { r2d } from './utils/render-2d';
+import { init_facemesh_render } from './utils/facemesh';
+import { calc_size_to_fit, render_2d_scene } from './utils/webgl';
+tfjsWasm.setWasmPaths(`https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@${tfjsWasm.version_wasm}/dist/`);
+
+/**
+ * Options passed to [[MaskProcessor]] constructor.
+ */
+export interface MaskProcessorOptions {
+ /**
+ * The HTMLImageElement to use for the face mask.
+ * An error will be raised if the image hasn't been fully loaded yet. Additionally, the image must follow
+ * [security guidelines](https://developer.mozilla.org/en-US/docs/Web/HTML/CORS_enabled_image)
+ * when loading the image from a different origin. Failing to do so will result to an empty output frame.
+ */
+ maskImage: HTMLImageElement;
+}
+
+/**
+ * The MaskProcessor, when added to a VideoTrack,
+ * place a mask in each video frame on top of the person's face,
+ * Each instance of MaskProcessor should be added to only one VideoTrack
+ * at a time to prevent overlapping of image data from multiple VideoTracks.
+ *
+ * @example
+ *
+ * ```ts
+ * import { createLocalVideoTrack } from 'twilio-video';
+ *
+ * let faceMask;
+ * const img = new Image();
+ *
+ * img.onload = () => {
+ * faceMask = new MaskProcessor({
+ * assetsPath: 'https://my-server-path/assets',
+ * maskImage: img,
+ * });
+ *
+ * faceMask.loadModel().then(() => {
+ * createLocalVideoTrack({
+ * width: 640,
+ * height: 480,
+ * frameRate: 24
+ * }).then(track => {
+ * track.addProcessor(faceMask);
+ * });
+ * });
+ * };
+ * img.src = '/face-mask.jpg';
+ * ```
+ */
+export class MaskProcessor extends Processor {
+ private _maskImage!: HTMLImageElement;
+ // tslint:disable-next-line no-unused-variable
+ private readonly _name: string = 'MaskProcessor';
+
+ private static _model: FaceLandmarksDetector | null = null;
+ private static async _loadModel(): Promise