Skip to content

Commit fb2677e

Browse files
LinchennLinchenn
authored
Support benchmarking models by CLI and BrowserStack (#3641)
FEATURE
1 parent a0d6250 commit fb2677e

File tree

7 files changed

+1349
-23
lines changed

7 files changed

+1349
-23
lines changed

‎e2e/benchmarks/benchmark_models.js

+79
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
/**
2+
* @license
3+
* Copyright 2020 Google LLC. All Rights Reserved.
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
* =============================================================================
16+
*/
17+
18+
/**
19+
* The purpose of this test file is to benchmark models by a test runner, such
20+
* as karma. To invoke this test, inlude this file to the `files` field of
21+
* `karma.conf.js`.
22+
*
23+
* This file wraps the model benchmarking into a Jasmine test and the benchmark
24+
* results will be logged to the console.
25+
*/
26+
27+
async function getEnvSummary() {
28+
let envSummary = `${tf.getBackend()} backend`;
29+
if (tf.getBackend() === 'webgl') {
30+
envSummary += `, version ${tf.env().get('WEBGL_VERSION')}`;
31+
} else if (tf.getBackend() === 'wasm') {
32+
const hasSIMD = await tf.env().getAsync('WASM_HAS_SIMD_SUPPORT');
33+
envSummary += hasSIMD ? ' with SIMD' : ' without SIMD';
34+
}
35+
return envSummary;
36+
}
37+
38+
async function getBenchmarkSummary(timeInfo, memoryInfo, modelName = 'model') {
39+
if (timeInfo == null) {
40+
throw new Error('Missing the timeInfo parameter.');
41+
} else if (timeInfo.times.length === 0) {
42+
throw new Error('Missing the memoryInfo parameter.');
43+
} else if (memoryInfo == null) {
44+
throw new Error('The length of timeInfo.times is at least 1.');
45+
}
46+
47+
const numRuns = timeInfo.times.length;
48+
const envSummary = await getEnvSummary();
49+
const benchmarkSummary = `
50+
benchmark the ${modelName} on ${envSummary}
51+
1st inference time: ${printTime(timeInfo.times[0])}
52+
Average inference time (${numRuns} runs): ${printTime(timeInfo.averageTime)}
53+
Best inference time: ${printTime(timeInfo.minTime)}
54+
Peak memory: ${printMemory(memoryInfo.peakBytes)}
55+
`;
56+
return benchmarkSummary;
57+
}
58+
59+
describe('benchmark models', () => {
60+
beforeAll(() => {
61+
jasmine.DEFAULT_TIMEOUT_INTERVAL = 1000000;
62+
});
63+
64+
it('mobile net', async () => {
65+
const url =
66+
'https://storage.googleapis.com/learnjs-data/mobilenet_v2_100_fused/model.json';
67+
const model = await tf.loadGraphModel(url);
68+
const input = generateInput(model);
69+
const predict = () => model.predict(input);
70+
71+
const numRuns = 20;
72+
const timeInfo = await profileInferenceTime(predict, numRuns);
73+
const memoryInfo = await profileInferenceMemory(predict);
74+
75+
const benchmarkSummary =
76+
await getBenchmarkSummary(timeInfo, memoryInfo, 'mobilenet_v2');
77+
console.log(benchmarkSummary);
78+
});
79+
});

‎e2e/benchmarks/benchmark_util.js

+36-11
Original file line numberDiff line numberDiff line change
@@ -119,8 +119,12 @@ function getPredictFnForModel(model, input) {
119119
/**
120120
* Executes the predict function for `model` (`model.predict` for tf.LayersModel
121121
* and `model.executeAsync` for tf.GraphModel) and times the inference process
122-
* for `numRuns` rounds. Then returns a promise that resolves with an array of
123-
* inference times for each inference process.
122+
* for `numRuns` rounds. Then returns a promise that resolves with information
123+
* about the model's inference time:
124+
* - `times`: an array of inference time for each inference
125+
* - `averageTime`: the average time of all inferences
126+
* - `minTime`: the minimum time of all inferences
127+
* - `maxTime`: the maximum time of all inferences
124128
*
125129
* The inference time contains the time spent by both `predict()` and `data()`
126130
* called by tensors in the prediction.
@@ -130,10 +134,13 @@ function getPredictFnForModel(model, input) {
130134
* 'https://tfhub.dev/google/imagenet/mobilenet_v2_140_224/classification/2';
131135
* const model = await tf.loadGraphModel(modelUrl, {fromTFHub: true});
132136
* const zeros = tf.zeros([1, 224, 224, 3]);
133-
* const elapsedTimeArray =
137+
* const timeInfo =
134138
* await profileInferenceTimeForModel(model, zeros, 2);
135139
*
136-
* console.log(`Elapsed time array: ${elapsedTimeArray}`);
140+
* console.log(`Elapsed time array: ${timeInfo.times}`);
141+
* console.log(`Average time: ${timeInfo.averageTime}`);
142+
* console.log(`Minimum time: ${timeInfo.minTime}`);
143+
* console.log(`Maximum time: ${timeInfo.maxTime}`);
137144
* ```
138145
*
139146
* @param model An instance of tf.GraphModel or tf.LayersModel for timing the
@@ -148,8 +155,12 @@ async function profileInferenceTimeForModel(model, input, numRuns = 1) {
148155

149156
/**
150157
* Executes `predict()` and times the inference process for `numRuns` rounds.
151-
* Then returns a promise that resolves with an array of inference time for each
152-
* inference process.
158+
* Then returns a promise that resolves with information about the inference
159+
* time:
160+
* - `times`: an array of inference time for each inference
161+
* - `averageTime`: the average time of all inferences
162+
* - `minTime`: the minimum time of all inferences
163+
* - `maxTime`: the maximum time of all inferences
153164
*
154165
* The inference time contains the time spent by both `predict()` and `data()`
155166
* called by tensors in the prediction.
@@ -159,10 +170,13 @@ async function profileInferenceTimeForModel(model, input, numRuns = 1) {
159170
* 'https://tfhub.dev/google/imagenet/mobilenet_v2_140_224/classification/2';
160171
* const model = await tf.loadGraphModel(modelUrl, {fromTFHub: true});
161172
* const zeros = tf.zeros([1, 224, 224, 3]);
162-
* const elapsedTimeArray =
173+
* const timeInfo =
163174
* await profileInferenceTime(() => model.predict(zeros), 2);
164175
*
165-
* console.log(`Elapsed time array: ${elapsedTimeArray}`);
176+
* console.log(`Elapsed time array: ${timeInfo.times}`);
177+
* console.log(`Average time: ${timeInfo.averageTime}`);
178+
* console.log(`Minimum time: ${timeInfo.minTime}`);
179+
* console.log(`Maximum time: ${timeInfo.maxTime}`);
166180
* ```
167181
*
168182
* @param predict The predict function to execute and time.
@@ -175,7 +189,7 @@ async function profileInferenceTime(predict, numRuns = 1) {
175189
`a(n) ${typeof predict} is found.`);
176190
}
177191

178-
const elapsedTimeArray = [];
192+
const times = [];
179193
for (let i = 0; i < numRuns; i++) {
180194
const start = performance.now();
181195
const res = await predict();
@@ -184,9 +198,20 @@ async function profileInferenceTime(predict, numRuns = 1) {
184198
const elapsedTime = performance.now() - start;
185199

186200
tf.dispose(res);
187-
elapsedTimeArray.push(elapsedTime);
201+
times.push(elapsedTime);
188202
}
189-
return elapsedTimeArray;
203+
204+
const averageTime = times.reduce((acc, curr) => acc + curr, 0) / times.length;
205+
const minTime = Math.min(...times);
206+
const maxTime = Math.max(...times);
207+
const timeInfo = {
208+
times,
209+
averageTime,
210+
minTime,
211+
maxTime
212+
213+
};
214+
return timeInfo;
190215
}
191216

192217
/**

‎e2e/benchmarks/benchmark_util_test.js

+22
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,28 @@ describe('benchmark_util', () => {
3434
});
3535
});
3636

37+
describe('profile inference time', () => {
38+
describe('profileInferenceTime', () => {
39+
it('throws when passing in invalid predict', async () => {
40+
const predict = {};
41+
await expectAsync(profileInferenceTime(predict)).toBeRejected();
42+
});
43+
44+
it('does not add new tensors', async () => {
45+
const model = tf.sequential(
46+
{layers: [tf.layers.dense({units: 1, inputShape: [3]})]});
47+
const input = tf.zeros([1, 3]);
48+
49+
const tensorsBefore = tf.memory().numTensors;
50+
await profileInferenceTime(() => model.predict(input));
51+
expect(tf.memory().numTensors).toEqual(tensorsBefore);
52+
53+
model.dispose();
54+
input.dispose();
55+
});
56+
});
57+
});
58+
3759
describe('Profile Memory', () => {
3860
describe('profileInferenceMemory', () => {
3961
it('pass in invalid predict', async () => {

‎e2e/benchmarks/index.html

+10-12
Original file line numberDiff line numberDiff line change
@@ -255,21 +255,21 @@ <h2>TensorFlow.js Model Benchmark</h2>
255255
async function warmUpAndRecordTime() {
256256
await showMsg('Warming up');
257257

258-
let elapsedTimeArray;
258+
let timeInfo;
259259
if (state.benchmark === 'custom') {
260260
const input = generateInput(model);
261261
try {
262-
elapsedTimeArray = await profileInferenceTimeForModel(model, input, 1);
262+
timeInfo = await profileInferenceTimeForModel(model, input, 1);
263263
} finally {
264264
tf.dispose(input);
265265
}
266266
} else {
267-
elapsedTimeArray = await profileInferenceTime(() => predict(model), 1);
267+
timeInfo = await profileInferenceTime(() => predict(model), 1);
268268
}
269269

270270
await showMsg(null);
271271
appendRow(timeTable, 'backend', state.backend);
272-
appendRow(timeTable, '1st inference', printTime(elapsedTimeArray[0]));
272+
appendRow(timeTable, '1st inference', printTime(timeInfo.times[0]));
273273
}
274274

275275
async function showInputs() {
@@ -340,27 +340,25 @@ <h2>TensorFlow.js Model Benchmark</h2>
340340
await showMsg(`Running predict ${state.numRuns} times`);
341341
chartWidth = document.querySelector('#perf-trendline-container').getBoundingClientRect().width;
342342

343-
let times;
343+
let timeInfo;
344344
const numRuns = state.numRuns;
345345
if (state.benchmark === 'custom') {
346346
const input = generateInput(model);
347347
try {
348-
times = await profileInferenceTimeForModel(model, input, numRuns);
348+
timeInfo = await profileInferenceTimeForModel(model, input, numRuns);
349349
} finally {
350350
tf.dispose(input);
351351
}
352352
} else {
353-
times = await profileInferenceTime(() => predict(model), numRuns);
353+
timeInfo = await profileInferenceTime(() => predict(model), numRuns);
354354
}
355355

356356
const forceInferenceTrendYMinToZero = true;
357-
populateTrendline(document.querySelector('#perf-trendline-container'), times, forceInferenceTrendYMinToZero, printTime);
357+
populateTrendline(document.querySelector('#perf-trendline-container'), timeInfo.times, forceInferenceTrendYMinToZero, printTime);
358358

359359
await showMsg(null);
360-
const average = times.reduce((acc, curr) => acc + curr, 0) / times.length;
361-
const min = Math.min(...times);
362-
appendRow(timeTable, `Subsequent average(${state.numRuns} runs)`, printTime(average));
363-
appendRow(timeTable, 'Best time', printTime(min));
360+
appendRow(timeTable, `Subsequent average(${state.numRuns} runs)`, printTime(timeInfo.averageTime));
361+
appendRow(timeTable, 'Best time', printTime(timeInfo.minTime));
364362
}
365363

366364
async function profileMemory() {

‎e2e/benchmarks/karma.conf.js

+98
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
/**
2+
* @license
3+
* Copyright 2020 Google LLC. All Rights Reserved.
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
* =============================================================================
16+
*/
17+
18+
const localRunConfig = {
19+
reporters: ['progress'],
20+
plugins: ['karma-jasmine', 'karma-chrome-launcher'],
21+
browsers: ['Chrome']
22+
};
23+
24+
const browserstackConfig = {
25+
hostname: 'bs-local.com',
26+
plugins: ['karma-jasmine', 'karma-browserstack-launcher'],
27+
reporters: ['progress', 'BrowserStack'],
28+
browserStack: {
29+
username: process.env.BROWSERSTACK_USERNAME,
30+
accessKey: process.env.BROWSERSTACK_ACCESS_KEY,
31+
apiClientEndpoint: 'https://api.browserstack.com'
32+
},
33+
34+
customLaunchers: {
35+
bs_chrome_mac: {
36+
base: 'BrowserStack',
37+
browser: 'chrome',
38+
browser_version: '84.0',
39+
os: 'OS X',
40+
os_version: 'Catalina',
41+
},
42+
bs_firefox_mac: {
43+
base: 'BrowserStack',
44+
browser: 'firefox',
45+
browser_version: '70.0',
46+
os: 'OS X',
47+
os_version: 'Catalina',
48+
},
49+
bs_safari_mac: {
50+
base: 'BrowserStack',
51+
browser: 'Safari',
52+
browser_version: '13.1',
53+
os: 'OS X',
54+
os_version: 'Catalina',
55+
}
56+
},
57+
58+
browsers: ['bs_chrome_mac', 'bs_firefox_mac', 'bs_safari_mac'],
59+
};
60+
61+
module.exports = function(config) {
62+
let extraConfig = null;
63+
if (config.browserstack) {
64+
extraConfig = browserstackConfig;
65+
} else {
66+
extraConfig = localRunConfig;
67+
}
68+
69+
config.set({
70+
...extraConfig,
71+
frameworks: ['jasmine'],
72+
files: [
73+
'https://unpkg.com/@tensorflow/tfjs-core@latest/dist/tf-core.js',
74+
'https://unpkg.com/@tensorflow/tfjs-backend-cpu@latest/dist/tf-backend-cpu.js',
75+
'https://unpkg.com/@tensorflow/tfjs-backend-webgl@latest/dist/tf-backend-webgl.js',
76+
'https://unpkg.com/@tensorflow/tfjs-layers@latest/dist/tf-layers.js',
77+
'https://unpkg.com/@tensorflow/tfjs-converter@latest/dist/tf-converter.js',
78+
'https://unpkg.com/@tensorflow/tfjs-backend-wasm@latest/dist/tf-backend-wasm.js',
79+
'util.js', 'benchmark_util.js', 'benchmark_models.js'
80+
],
81+
preprocessors: {},
82+
singleRun: true,
83+
captureTimeout: 3e5,
84+
reportSlowerThan: 500,
85+
browserNoActivityTimeout: 3e5,
86+
browserDisconnectTimeout: 3e5,
87+
browserDisconnectTolerance: 0,
88+
browserSocketTimeout: 1.2e5,
89+
client: {jasmine: {random: false}},
90+
91+
// The following configurations are generated by karma
92+
port: 9876,
93+
colors: true,
94+
logLevel: config.LOG_INFO,
95+
autoWatch: false,
96+
concurrency: Infinity
97+
})
98+
}

‎e2e/benchmarks/package.json

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
{
2+
"name": "@tensorflow/tfjs-benchmark",
3+
"version": "0.0.1",
4+
"description": "Benchmark models' and ops' performance",
5+
"private": true,
6+
"repository": {
7+
"type": "git",
8+
"url": "https://github.com/tensorflow/tfjs"
9+
},
10+
"devDependencies": {
11+
"karma": "^4.4.1",
12+
"karma-browserstack-launcher": "^1.6.0",
13+
"karma-jasmine": "^3.3.1"
14+
},
15+
"scripts": {
16+
"test": "karma start"
17+
},
18+
"license": "Apache-2.0",
19+
"engines": {
20+
"yarn": ">= 1.0.0"
21+
}
22+
}

0 commit comments

Comments
 (0)