Skip to content

Commit ede445b

Browse files
ahmedsabieahmedsabie
authored
Include two FPS counters to track inference as well as end to end FPS (#921)
* Add FPS counter to demo * Add selector for FPS mode
1 parent b8ba15f commit ede445b

File tree

17 files changed

+560
-348
lines changed

17 files changed

+560
-348
lines changed

‎body-segmentation/demos/README.md

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,14 @@ If you want to run any of the demos locally, follow these steps:
3939

4040
4. Install dependencies. `yarn`
4141

42-
5. Run the demo. `yarn watch`
42+
5. To ensure GPU sync for a correct mediapipe FPS, edit
43+
`./node_modules/@mediapipe/selfie_segmentation/selfie_segmentation.js`
44+
as well as `./node_modules/@mediapipe/pose/pose.js`
45+
and after the statement `y=d.l.getContext("webgl2",{});` add the statement
46+
`window.exposedContext=y;`. This will give the demo access to this context. If
47+
you skip the step then the demo will work but the FPS for mediapipe will be
48+
incorrect.
4349

44-
6. The demo runs at `localhost:1234`. (Remember to provide URL model parameter e. g. `localhost:1234/?model=mediapipe_hands`)
50+
6. Run the demo. `yarn watch`
51+
52+
7. The demo runs at `localhost:1234`. (Remember to provide URL model parameter e. g. `localhost:1234/?model=mediapipe_hands`)

‎body-segmentation/demos/live_video/index.html

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,21 @@
3434
#canvas-wrapper {
3535
position: relative;
3636
}
37+
38+
.block-container {
39+
text-align: left;
40+
}
41+
42+
#stats {
43+
display: inline-block;
44+
}
3745
</style>
3846
</head>
3947

4048
<body>
41-
<div id="stats"></div>
49+
<div class="block-container">
50+
<div id="stats"></div>
51+
</div>
4252
<div id="main">
4353
<div class="container">
4454
<div class="canvas-wrapper">

‎body-segmentation/demos/live_video/package.json

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,10 @@
1313
"@mediapipe/selfie_segmentation": "~0.1.0",
1414
"@tensorflow-models/body-segmentation": "file:../../dist",
1515
"@tensorflow-models/pose-detection": "file:../../../pose-detection/dist",
16-
"@tensorflow/tfjs-backend-wasm": "^3.11.0",
17-
"@tensorflow/tfjs-backend-webgl": "^3.11.0",
18-
"@tensorflow/tfjs-converter": "^3.11.0",
19-
"@tensorflow/tfjs-core": "^3.11.0",
16+
"@tensorflow/tfjs-backend-wasm": "^3.13.0",
17+
"@tensorflow/tfjs-backend-webgl": "^3.13.0",
18+
"@tensorflow/tfjs-converter": "^3.13.0",
19+
"@tensorflow/tfjs-core": "^3.13.0",
2020
"scatter-gl": "0.0.8"
2121
},
2222
"scripts": {

‎body-segmentation/demos/live_video/src/index.js

Lines changed: 75 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import '@tensorflow/tfjs-backend-webgl';
2020
import * as mpPose from '@mediapipe/pose';
2121
import * as mpSelfieSegmentation from '@mediapipe/selfie_segmentation';
2222
import * as tfjsWasm from '@tensorflow/tfjs-backend-wasm';
23+
import * as tf from '@tensorflow/tfjs-core';
2324

2425
tfjsWasm.setWasmPaths(
2526
`https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@${
@@ -36,9 +37,18 @@ import {setBackendAndEnvFlags} from './shared/util';
3637

3738
let segmenter, camera, stats;
3839
let cameras;
39-
let startInferenceTime, numInferences = 0;
40-
let inferenceTimeSum = 0, lastPanelUpdate = 0;
40+
let fpsDisplayMode = 'model';
41+
const resetTime = {
42+
startInferenceTime: 0,
43+
numInferences: 0,
44+
inferenceTimeSum: 0,
45+
lastPanelUpdate: 0
46+
};
47+
let modelTime = {...resetTime};
48+
let E2ETime = {...resetTime};
4149
let rafId;
50+
const MODEL_LABEL = '(Model FPS) ';
51+
const E2E_LABEL = '(End2End FPS)';
4252
const canvas = document.createElement('canvas');
4353
const ctx = canvas.getContext('2d');
4454

@@ -128,23 +138,23 @@ async function checkGuiUpdate() {
128138
}
129139
}
130140

131-
function beginEstimateSegmentationStats() {
132-
startInferenceTime = (performance || Date).now();
141+
function beginEstimateSegmentationStats(time) {
142+
time.startInferenceTime = (performance || Date).now();
133143
}
134144

135-
function endEstimateSegmentationStats() {
145+
function endEstimateSegmentationStats(time) {
136146
const endInferenceTime = (performance || Date).now();
137-
inferenceTimeSum += endInferenceTime - startInferenceTime;
138-
++numInferences;
147+
time.inferenceTimeSum += endInferenceTime - time.startInferenceTime;
148+
++time.numInferences;
139149

140150
const panelUpdateMilliseconds = 1000;
141-
if (endInferenceTime - lastPanelUpdate >= panelUpdateMilliseconds) {
142-
const averageInferenceTime = inferenceTimeSum / numInferences;
143-
inferenceTimeSum = 0;
144-
numInferences = 0;
151+
if (endInferenceTime - time.lastPanelUpdate >= panelUpdateMilliseconds) {
152+
const averageInferenceTime = time.inferenceTimeSum / time.numInferences;
153+
time.inferenceTimeSum = 0;
154+
time.numInferences = 0;
145155
stats.customFpsPanel.update(
146156
1000.0 / averageInferenceTime, 120 /* maxValue */);
147-
lastPanelUpdate = endInferenceTime;
157+
time.lastPanelUpdate = endInferenceTime;
148158
}
149159
}
150160

@@ -162,8 +172,24 @@ async function renderResult() {
162172
// Segmenter can be null if initialization failed (for example when loading
163173
// from a URL that does not exist).
164174
if (segmenter != null) {
165-
// FPS only counts the time it takes to finish segmentPeople.
166-
beginEstimateSegmentationStats();
175+
// Change in what FPS should measure.
176+
const newFpsDisplayMode = STATE.fpsDisplay.mode;
177+
if (fpsDisplayMode != newFpsDisplayMode) {
178+
if (newFpsDisplayMode === 'model') {
179+
stats = setupStats(MODEL_LABEL);
180+
modelTime = {...resetTime};
181+
} else {
182+
stats = setupStats(E2E_LABEL);
183+
E2ETime = {...resetTime};
184+
}
185+
fpsDisplayMode = newFpsDisplayMode;
186+
}
187+
// Model FPS only counts the time it takes to finish segmentPeople.
188+
if (fpsDisplayMode === 'model') {
189+
beginEstimateSegmentationStats(modelTime);
190+
} else { // E2E FPS includes rendering time.
191+
beginEstimateSegmentationStats(E2ETime);
192+
}
167193

168194
// Detectors can throw errors, for example when using custom URLs that
169195
// contain a model that doesn't provide the expected output.
@@ -187,7 +213,35 @@ async function renderResult() {
187213
alert(error);
188214
}
189215

190-
endEstimateSegmentationStats();
216+
if (fpsDisplayMode === 'model') {
217+
// Ensure GPU is done for timing purposes.
218+
const [backend] = STATE.backend.split('-');
219+
if (backend === 'tfjs') {
220+
for (const value of segmentation) {
221+
const mask = value.mask;
222+
const tensor = await mask.toTensor();
223+
224+
const res = tensor.dataToGPU();
225+
226+
const webGLBackend = tf.backend();
227+
const buffer =
228+
webGLBackend.gpgpu.createBufferFromTexture(res.texture, 1, 1);
229+
webGLBackend.gpgpu.downloadFloat32MatrixFromBuffer(buffer, 1);
230+
231+
res.tensorRef.dispose();
232+
}
233+
} else if (backend === 'mediapipe') {
234+
// Code in
235+
// node_modules/@mediapipe/selfie_segmentation/selfie_segmentation.js
236+
// must be modified to expose the webgl context it uses.
237+
const gl = window.exposedContext;
238+
if (gl)
239+
gl.readPixels(
240+
0, 0, 1, 1, gl.RGBA, gl.UNSIGNED_BYTE, new Uint8Array(4));
241+
}
242+
243+
endEstimateSegmentationStats(modelTime);
244+
}
191245
}
192246

193247
// The null check makes sure the UI is not in the middle of changing to a
@@ -229,6 +283,10 @@ async function renderResult() {
229283
}
230284
}
231285
camera.drawToCanvas(canvas);
286+
287+
if (fpsDisplayMode === 'e2e') {
288+
endEstimateSegmentationStats(E2ETime);
289+
}
232290
}
233291

234292
async function renderPrediction() {
@@ -239,7 +297,7 @@ async function renderPrediction() {
239297
}
240298

241299
rafId = requestAnimationFrame(renderPrediction);
242-
};
300+
}
243301

244302
async function getVideoInputs() {
245303
if (!navigator.mediaDevices || !navigator.mediaDevices.enumerateDevices) {
@@ -266,7 +324,7 @@ async function app() {
266324

267325
await setupDatGui(urlParams, cameras);
268326

269-
stats = setupStats();
327+
stats = setupStats(MODEL_LABEL);
270328

271329
camera = await Camera.setupCamera(STATE.camera, cameras);
272330
canvas.width = camera.canvas.width;

‎body-segmentation/demos/live_video/src/option_panel.js

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,17 +62,22 @@ export async function setupDatGui(urlParams, cameras) {
6262
});
6363
cameraFolder.open();
6464

65+
// The fps display folder contains options for video settings.
66+
const fpsDisplayFolder = gui.addFolder('FPS Display');
67+
fpsDisplayFolder.add(params.STATE.fpsDisplay, 'mode', ['model', 'e2e']);
68+
fpsDisplayFolder.open();
69+
6570
// The model folder contains options for model selection.
6671
const modelFolder = gui.addFolder('Model');
6772

6873
const model = urlParams.get('model');
6974
let type = urlParams.get('type');
7075

7176
switch (model) {
72-
case 'blaze_pose':
77+
case 'blazepose':
7378
params.STATE.model = poseDetection.SupportedModels.BlazePose;
7479
break;
75-
case 'body_pix':
80+
case 'bodypix':
7681
params.STATE.model = bodySegmentation.SupportedModels.BodyPix;
7782
break;
7883
case 'selfie_segmentation':

‎body-segmentation/demos/live_video/yarn.lock

Lines changed: 22 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -996,43 +996,43 @@
996996
"@tensorflow-models/pose-detection@file:../../../pose-detection/dist":
997997
version "0.0.0"
998998

999-
"@tensorflow/tfjs-backend-cpu@3.12.0":
1000-
version "3.12.0"
1001-
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-3.12.0.tgz#a96279c3116486dfb5deccf8bac78f72d3f77f65"
1002-
integrity sha512-UMf/OHHCzm/+oKXtTDnanlyYFNNKKU5u2hJooKak1JqSOEKChBOPYKmxf9VWmY7Pos4GbbKH/V1pLzfLnLq+Bg==
999+
"@tensorflow/tfjs-backend-cpu@3.13.0":
1000+
version "3.13.0"
1001+
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-3.13.0.tgz#2a141256fe0a5de0670c31a9ba6990465708dee4"
1002+
integrity sha512-POmzUoAP8HooYYTZ72O1ZYkpVZB0f+8PeAkbTxIG0oahcJccj6a0Vovp1A6xWKfljUoPlJb3jWVC++S603ZL8w==
10031003
dependencies:
10041004
"@types/seedrandom" "2.4.27"
10051005
seedrandom "2.4.3"
10061006

1007-
"@tensorflow/tfjs-backend-wasm@^3.11.0":
1008-
version "3.12.0"
1009-
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-wasm/-/tfjs-backend-wasm-3.12.0.tgz#280f42a867d90672dacdc2b92df4152c59b6dce8"
1010-
integrity sha512-H4OmQkebiCpDdWiH5Ob2G+1+jzGNffG2FlkWV5y66RM/wZijYyCKIg4Nm7KCSUrvZPhkhP622QhM2BPvI9U8hA==
1007+
"@tensorflow/tfjs-backend-wasm@^3.13.0":
1008+
version "3.13.0"
1009+
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-wasm/-/tfjs-backend-wasm-3.13.0.tgz#3465c6040542752d2632df7e0c4ac0e5a9ef801a"
1010+
integrity sha512-h5kNS4xvljoySzfcFwqbdFB6QZGR06IA9/Xq/PjBeZt18XEoJGqKHbOCYupmUlr5pxo/gnXzPhAC2h4SfZXPXw==
10111011
dependencies:
1012-
"@tensorflow/tfjs-backend-cpu" "3.12.0"
1012+
"@tensorflow/tfjs-backend-cpu" "3.13.0"
10131013
"@types/emscripten" "~0.0.34"
10141014

1015-
"@tensorflow/tfjs-backend-webgl@^3.11.0":
1016-
version "3.12.0"
1017-
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-webgl/-/tfjs-backend-webgl-3.12.0.tgz#51af13d9790fd07748e95efe02ab72092c52ead1"
1018-
integrity sha512-nFiPS7AiDZ+H4RXpx8dSYPWzy3B1zC/aRKwH9uz9MUp1WiHHbYr90+E0ut9HFrIz0cMb0sR7vGU+zkjSZ1rLsA==
1015+
"@tensorflow/tfjs-backend-webgl@^3.13.0":
1016+
version "3.13.0"
1017+
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-webgl/-/tfjs-backend-webgl-3.13.0.tgz#f99df51253de21e20dae195991d332195ab30b4b"
1018+
integrity sha512-ZuJS11tCoZx2F1Eq7wqiqu8euJpPW/JV0qOKBehlRpV2qQrR+wHMpBT1hhDl4qU4LdgFTtSggKIRg/L8b0ScUQ==
10191019
dependencies:
1020-
"@tensorflow/tfjs-backend-cpu" "3.12.0"
1020+
"@tensorflow/tfjs-backend-cpu" "3.13.0"
10211021
"@types/offscreencanvas" "~2019.3.0"
10221022
"@types/seedrandom" "2.4.27"
10231023
"@types/webgl-ext" "0.0.30"
10241024
"@types/webgl2" "0.0.6"
10251025
seedrandom "2.4.3"
10261026

1027-
"@tensorflow/tfjs-converter@^3.11.0":
1028-
version "3.12.0"
1029-
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-converter/-/tfjs-converter-3.12.0.tgz#2e28c7bf6e92d4614cb6df362a81a55f79432fef"
1030-
integrity sha512-WO6FreEB83CgFwvcyrzyaWG5WshanfyhTI6rOqSbjY86+MlJprTn4fuY9qbnYk/gDLhxEaUpYgubamY4g4L76g==
1027+
"@tensorflow/tfjs-converter@^3.13.0":
1028+
version "3.13.0"
1029+
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-converter/-/tfjs-converter-3.13.0.tgz#3affc86d94c3948b01673a91309a35feb10e5eac"
1030+
integrity sha512-H2VpDTv9Ve0HBt7ttzz46DmnsPaiT0B+yJjVH3NebGZbgY9C8boBgJIsdyqfiqEWBS3WxF8h4rh58Hv5XXMgaQ==
10311031

1032-
"@tensorflow/tfjs-core@^3.11.0":
1033-
version "3.12.0"
1034-
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-core/-/tfjs-core-3.12.0.tgz#af1f93ea924be74e4ecf094608ea22709bf56f17"
1035-
integrity sha512-0rXf+aMjfkFLan7qna+L7DjfSZaaRibKgq9XuorjSy9rZcdZLE2IHFTzYR7B8mdlnq2szArKzm+JXqa9grTl7w==
1032+
"@tensorflow/tfjs-core@^3.13.0":
1033+
version "3.13.0"
1034+
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-core/-/tfjs-core-3.13.0.tgz#0cfd707c668250969564991c5c101fb52e51e1aa"
1035+
integrity sha512-18qBEVIB/4u2OUK9nA5P1XT3e3LyarElD1UKNSNDpnMLxhLTUVZaCR71eHJcpl9wP2Q0cciaTJCTpJdPv1tNDQ==
10361036
dependencies:
10371037
"@types/long" "^4.0.1"
10381038
"@types/offscreencanvas" "~2019.3.0"

‎body-segmentation/demos/shared/params.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ export const VIDEO_SIZE = {
2727
};
2828
export const STATE = {
2929
camera: {targetFPS: 60, sizeOption: '640 X 480', cameraSelector: ''},
30+
fpsDisplay: {mode: 'model'},
3031
backend: '',
3132
flags: {},
3233
modelConfig: {},

‎body-segmentation/demos/shared/stats_panel.js

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,15 @@
1414
* limitations under the License.
1515
* =============================================================================
1616
*/
17-
export function setupStats() {
17+
export function setupStats(label) {
1818
const stats = new Stats();
19-
stats.customFpsPanel = stats.addPanel(new Stats.Panel('FPS', '#0ff', '#002'));
19+
stats.customFpsPanel = stats.addPanel(new Stats.Panel(label, '#0ff', '#002'));
2020
stats.showPanel(stats.domElement.children.length - 1);
2121

2222
const parent = document.getElementById('stats');
23+
if (parent.hasChildNodes()) {
24+
parent.removeChild(parent.lastChild);
25+
}
2326
parent.appendChild(stats.domElement);
2427

2528
const statsPanes = parent.querySelectorAll('canvas');

‎body-segmentation/demos/upload_video/package.json

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,10 @@
1313
"@mediapipe/selfie_segmentation": "~0.1.0",
1414
"@tensorflow-models/body-segmentation": "file:../../dist",
1515
"@tensorflow-models/pose-detection": "file:../../../pose-detection/dist",
16-
"@tensorflow/tfjs-backend-wasm": "^3.11.0",
17-
"@tensorflow/tfjs-backend-webgl": "^3.11.0",
18-
"@tensorflow/tfjs-converter": "^3.11.0",
19-
"@tensorflow/tfjs-core": "^3.11.0"
16+
"@tensorflow/tfjs-backend-wasm": "^3.13.0",
17+
"@tensorflow/tfjs-backend-webgl": "^3.13.0",
18+
"@tensorflow/tfjs-converter": "^3.13.0",
19+
"@tensorflow/tfjs-core": "^3.13.0"
2020
},
2121
"scripts": {
2222
"watch": "cross-env NODE_ENV=development parcel index.html --no-hmr --open",

0 commit comments

Comments
 (0)