Skip to content

Commit ecb6add

Browse files
HonryPrathik Rao
Honry
authored and
Prathik Rao
committed
Support WebNN EP (#15698)
**Description**: This PR intends to enable WebNN EP in ONNX Runtime Web. It translates the ONNX nodes by [WebNN API](https://webmachinelearning.github.io/webnn/), which is implemented in C++ and uses Emscripten [Embind API](https://emscripten.org/docs/porting/connecting_cpp_and_javascript/embind.html#). Temporarily using preferred layout **NHWC** for WebNN graph partitions since the restriction in WebNN XNNPack backend implementation and the ongoing [discussion](webmachinelearning/webnn#324) in WebNN spec that whether WebNN should support both 'NHWC' and 'NCHW' layouts. No WebNN native EP, only for Web. **Motivation and Context**: Allow ONNXRuntime Web developers to access WebNN API to benefit from hardware acceleration. **WebNN API Implementation Status in Chromium**: - Tracked in Chromium issue: [#1273291](https://bugs.chromium.org/p/chromium/issues/detail?id=1273291) - **CPU device**: based on XNNPack backend, and had been available on Chrome Canary M112 behind "#enable-experimental-web-platform-features" flag for Windows and Linux platforms. Further implementation for more ops is ongoing. - **GPU device**: based on DML, implementation is ongoing. **Open**: - GitHub CI: WebNN currently is only available on Chrome Canary/Dev with XNNPack backend for Linux and Windows. This is an open to reviewers to help identify which GitHub CI should involved the WebNN EP and guide me to enable it. Thanks!
1 parent 97b1bae commit ecb6add

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+3404
-21
lines changed

‎cmake/CMakeLists.txt

+6
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,7 @@ option(onnxruntime_TVM_CUDA_RUNTIME "Build TVM with CUDA support" OFF)
132132
option(onnxruntime_TVM_USE_LLVM "Build TVM with LLVM. Set customized path to llvm-config.exe here if need" OFF)
133133
option(onnxruntime_TVM_USE_HASH "Build ipp-crypto library for support hash algorithm. It is defined for TVM only")
134134
option(onnxruntime_USE_XNNPACK "Build with XNNPACK support. Provides an alternative math library on ARM, WebAssembly and x86." OFF)
135+
option(onnxruntime_USE_WEBNN "Build with WebNN support. Enable hardware acceleration in web browsers." OFF)
135136

136137
# Options related to reducing the binary size produced by the build
137138
# XNNPACK EP requires the internal NHWC contrib ops to be available, so this option must be OFF when onnxruntime_USE_XNNPACK is ON
@@ -735,6 +736,11 @@ if (onnxruntime_USE_XNNPACK)
735736
list(APPEND ORT_PROVIDER_CMAKE_FLAGS -Donnxruntime_USE_XNNPACK=1)
736737
list(APPEND ONNXRUNTIME_PROVIDER_NAMES xnnpack)
737738
endif()
739+
if (onnxruntime_USE_WEBNN)
740+
list(APPEND ORT_PROVIDER_FLAGS -DUSE_WEBNN=1)
741+
list(APPEND ORT_PROVIDER_CMAKE_FLAGS -Donnxruntime_USE_WEBNN=1)
742+
list(APPEND ONNXRUNTIME_PROVIDER_NAMES webnn)
743+
endif()
738744
if (onnxruntime_USE_CANN)
739745
list(APPEND ORT_PROVIDER_FLAGS -DUSE_CANN=1)
740746
list(APPEND ORT_PROVIDER_CMAKE_FLAGS -Donnxruntime_USE_CANN=1)

‎cmake/onnxruntime.cmake

+1
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,7 @@ set(onnxruntime_INTERNAL_LIBRARIES
191191
${PROVIDERS_ROCM}
192192
${PROVIDERS_VITISAI}
193193
${PROVIDERS_XNNPACK}
194+
${PROVIDERS_WEBNN}
194195
${PROVIDERS_AZURE}
195196
${PROVIDERS_INTERNAL_TESTING}
196197
${onnxruntime_winml}

‎cmake/onnxruntime_providers.cmake

+28
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,9 @@ endif()
147147
if (onnxruntime_USE_XNNPACK)
148148
set(PROVIDERS_XNNPACK onnxruntime_providers_xnnpack)
149149
endif()
150+
if(onnxruntime_USE_WEBNN)
151+
set(PROVIDERS_WEBNN onnxruntime_providers_webnn)
152+
endif()
150153
if(onnxruntime_USE_SNPE)
151154
include(onnxruntime_snpe_provider.cmake)
152155
endif()
@@ -1007,6 +1010,31 @@ if (onnxruntime_USE_COREML)
10071010
endif()
10081011
endif()
10091012

1013+
if (onnxruntime_USE_WEBNN)
1014+
if (onnxruntime_MINIMAL_BUILD AND NOT onnxruntime_EXTENDED_MINIMAL_BUILD)
1015+
message(FATAL_ERROR "WebNN EP can not be used in a basic minimal build. Please build with '--minimal_build extended'")
1016+
endif()
1017+
1018+
add_compile_definitions(USE_WEBNN=1)
1019+
if (onnxruntime_ENABLE_WEBASSEMBLY_THREADS)
1020+
add_definitions(-DENABLE_WEBASSEMBLY_THREADS=1)
1021+
endif()
1022+
file(GLOB_RECURSE onnxruntime_providers_webnn_cc_srcs CONFIGURE_DEPENDS
1023+
"${ONNXRUNTIME_ROOT}/core/providers/webnn/*.h"
1024+
"${ONNXRUNTIME_ROOT}/core/providers/webnn/*.cc"
1025+
"${ONNXRUNTIME_ROOT}/core/providers/shared/utils/utils.h"
1026+
"${ONNXRUNTIME_ROOT}/core/providers/shared/utils/utils.cc"
1027+
)
1028+
1029+
source_group(TREE ${REPO_ROOT} FILES ${onnxruntime_providers_webnn_cc_srcs})
1030+
onnxruntime_add_static_library(onnxruntime_providers_webnn ${onnxruntime_providers_webnn_cc_srcs})
1031+
onnxruntime_add_include_to_target(onnxruntime_providers_webnn onnxruntime_common onnx onnx_proto Boost::mp11)
1032+
1033+
add_dependencies(onnxruntime_providers_webnn onnx ${onnxruntime_EXTERNAL_DEPENDENCIES})
1034+
set_target_properties(onnxruntime_providers_webnn PROPERTIES FOLDER "ONNXRuntime")
1035+
set_target_properties(onnxruntime_providers_webnn PROPERTIES LINKER_LANGUAGE CXX)
1036+
endif()
1037+
10101038
if (onnxruntime_USE_NNAPI_BUILTIN)
10111039
if (onnxruntime_MINIMAL_BUILD AND NOT onnxruntime_EXTENDED_MINIMAL_BUILD)
10121040
message(FATAL_ERROR "NNAPI can not be used in a basic minimal build. Please build with '--minimal_build extended'")

‎cmake/onnxruntime_webassembly.cmake

+10
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,7 @@ if (onnxruntime_BUILD_WEBASSEMBLY_STATIC_LIB)
110110
onnxruntime_providers
111111
${PROVIDERS_JS}
112112
${PROVIDERS_XNNPACK}
113+
${PROVIDERS_WEBNN}
113114
onnxruntime_session
114115
onnxruntime_util
115116
re2::re2
@@ -186,6 +187,7 @@ else()
186187
onnxruntime_providers
187188
${PROVIDERS_JS}
188189
${PROVIDERS_XNNPACK}
190+
${PROVIDERS_WEBNN}
189191
onnxruntime_session
190192
onnxruntime_util
191193
re2::re2
@@ -194,6 +196,10 @@ else()
194196
target_link_libraries(onnxruntime_webassembly PRIVATE XNNPACK)
195197
endif()
196198

199+
if(onnxruntime_USE_WEBNN)
200+
target_link_libraries(onnxruntime_webassembly PRIVATE onnxruntime_providers_webnn)
201+
endif()
202+
197203
if (onnxruntime_ENABLE_TRAINING)
198204
target_link_libraries(onnxruntime_webassembly PRIVATE tensorboard)
199205
endif()
@@ -255,6 +261,10 @@ else()
255261
)
256262
endif()
257263

264+
if (onnxruntime_USE_WEBNN)
265+
set_property(TARGET onnxruntime_webassembly APPEND_STRING PROPERTY LINK_FLAGS " --bind")
266+
endif()
267+
258268
# Set link flag to enable exceptions support, this will override default disabling exception throwing behavior when disable exceptions.
259269
target_link_options(onnxruntime_webassembly PRIVATE "SHELL:-s DISABLE_EXCEPTION_THROWING=0")
260270

‎include/onnxruntime/core/graph/constants.h

+1
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ constexpr const char* kJsExecutionProvider = "JsExecutionProvider";
4848
constexpr const char* kSnpeExecutionProvider = "SNPEExecutionProvider";
4949
constexpr const char* kTvmExecutionProvider = "TvmExecutionProvider";
5050
constexpr const char* kXnnpackExecutionProvider = "XnnpackExecutionProvider";
51+
constexpr const char* kWebNNExecutionProvider = "WebNNExecutionProvider";
5152
constexpr const char* kCannExecutionProvider = "CANNExecutionProvider";
5253
constexpr const char* kAzureExecutionProvider = "AzureExecutionProvider";
5354

‎js/common/lib/inference-session.ts

+7-1
Original file line numberDiff line numberDiff line change
@@ -165,14 +165,15 @@ export declare namespace InferenceSession {
165165

166166
// Currently, we have the following backends to support execution providers:
167167
// Backend Node.js binding: supports 'cpu' and 'cuda'.
168-
// Backend WebAssembly: supports 'cpu', 'wasm' and 'xnnpack'.
168+
// Backend WebAssembly: supports 'cpu', 'wasm', 'xnnpack' and 'webnn'.
169169
// Backend ONNX.js: supports 'webgl'.
170170
interface ExecutionProviderOptionMap {
171171
cpu: CpuExecutionProviderOption;
172172
cuda: CudaExecutionProviderOption;
173173
wasm: WebAssemblyExecutionProviderOption;
174174
webgl: WebGLExecutionProviderOption;
175175
xnnpack: XnnpackExecutionProviderOption;
176+
webnn: WebNNExecutionProviderOption;
176177
}
177178

178179
type ExecutionProviderName = keyof ExecutionProviderOptionMap;
@@ -200,6 +201,11 @@ export declare namespace InferenceSession {
200201
export interface XnnpackExecutionProviderOption extends ExecutionProviderOption {
201202
readonly name: 'xnnpack';
202203
}
204+
export interface WebNNExecutionProviderOption extends ExecutionProviderOption {
205+
readonly name: 'webnn';
206+
deviceType?: 'cpu'|'gpu';
207+
powerPreference?: 'default'|'low-power'|'high-performance';
208+
}
203209
// #endregion
204210

205211
// #endregion

‎js/web/karma.conf.js

+50-7
Original file line numberDiff line numberDiff line change
@@ -90,13 +90,56 @@ module.exports = function (config) {
9090
hostname,
9191
listenAddress,
9292
customLaunchers: {
93-
ChromeTest: { base: 'ChromeHeadless', flags: ['--enable-features=SharedArrayBuffer'] },
94-
ChromePerf: { base: 'Chrome', flags: ['--window-size=1,1', '--enable-features=SharedArrayBuffer'] },
95-
ChromeDebug: { debug: true, base: 'Chrome', flags: ['--remote-debugging-port=9333', '--enable-features=SharedArrayBuffer'] },
96-
ChromeCanaryTest: { base: 'ChromeCanary', flags: ['--window-size=1,1', '--enable-features=SharedArrayBuffer', '--enable-unsafe-webgpu'] },
97-
ChromeCanaryProfileTest: { base: 'ChromeCanary', flags: ['--window-size=1,1', '--enable-features=SharedArrayBuffer', '--enable-unsafe-webgpu', '--disable-dawn-features=disallow_unsafe_apis'] },
98-
ChromeCanaryDebug: { debug: true, base: 'ChromeCanary', flags: ['--remote-debugging-port=9333', '--enable-features=SharedArrayBuffer', '--enable-unsafe-webgpu'] },
99-
ChromeCanaryProfileDebug: { debug: true, base: 'ChromeCanary', flags: ['--remote-debugging-port=9333', '--enable-features=SharedArrayBuffer', '--enable-unsafe-webgpu', '--disable-dawn-features=disallow_unsafe_apis'] },
93+
ChromeTest: {
94+
base: 'ChromeHeadless',
95+
flags: ['--enable-features=SharedArrayBuffer']
96+
},
97+
ChromePerf: {
98+
base: 'Chrome',
99+
flags: ['--window-size=1,1', '--enable-features=SharedArrayBuffer']
100+
},
101+
ChromeDebug: {
102+
debug: true,
103+
base: 'Chrome', flags: ['--remote-debugging-port=9333', '--enable-features=SharedArrayBuffer']
104+
},
105+
ChromeCanaryTest: {
106+
base: 'ChromeCanary',
107+
flags: [
108+
'--window-size=1,1',
109+
'--enable-features=SharedArrayBuffer',
110+
'--enable-unsafe-webgpu',
111+
'--enable-experimental-web-platform-features'
112+
]
113+
},
114+
ChromeCanaryProfileTest: {
115+
base: 'ChromeCanary',
116+
flags: [
117+
'--window-size=1,1',
118+
'--enable-features=SharedArrayBuffer',
119+
'--enable-unsafe-webgpu',
120+
'--disable-dawn-features=disallow_unsafe_apis'
121+
]
122+
},
123+
ChromeCanaryDebug: {
124+
debug: true,
125+
base: 'ChromeCanary',
126+
flags: [
127+
'--remote-debugging-port=9333',
128+
'--enable-features=SharedArrayBuffer',
129+
'--enable-unsafe-webgpu',
130+
'--enable-experimental-web-platform-features'
131+
]
132+
},
133+
ChromeCanaryProfileDebug: {
134+
debug: true,
135+
base: 'ChromeCanary',
136+
flags: [
137+
'--remote-debugging-port=9333',
138+
'--enable-features=SharedArrayBuffer',
139+
'--enable-unsafe-webgpu',
140+
'--disable-dawn-features=disallow_unsafe_apis',
141+
]
142+
},
100143
//
101144
// ==== BrowserStack browsers ====
102145
//

‎js/web/lib/index.ts

+1
Original file line numberDiff line numberDiff line change
@@ -22,4 +22,5 @@ if (!BUILD_DEFS.DISABLE_WASM) {
2222
registerBackend('cpu', wasmBackend, 10);
2323
registerBackend('wasm', wasmBackend, 10);
2424
registerBackend('xnnpack', wasmBackend, 9);
25+
registerBackend('webnn', wasmBackend, 9);
2526
}

‎js/web/lib/wasm/session-options.ts

+23
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,29 @@ const setExecutionProviders =
6464
case 'xnnpack':
6565
epName = 'XNNPACK';
6666
break;
67+
case 'webnn':
68+
epName = 'WEBNN';
69+
if (typeof ep !== 'string') {
70+
const webnnOptions = ep as InferenceSession.WebNNExecutionProviderOption;
71+
if (webnnOptions?.deviceType) {
72+
const keyDataOffset = allocWasmString('deviceType', allocs);
73+
const valueDataOffset = allocWasmString(webnnOptions.deviceType, allocs);
74+
if (getInstance()._OrtAddSessionConfigEntry(sessionOptionsHandle, keyDataOffset, valueDataOffset) !==
75+
0) {
76+
throw new Error(`Can't set a session config entry: 'deviceType' - ${webnnOptions.deviceType}`);
77+
}
78+
}
79+
if (webnnOptions?.powerPreference) {
80+
const keyDataOffset = allocWasmString('powerPreference', allocs);
81+
const valueDataOffset = allocWasmString(webnnOptions.powerPreference, allocs);
82+
if (getInstance()._OrtAddSessionConfigEntry(sessionOptionsHandle, keyDataOffset, valueDataOffset) !==
83+
0) {
84+
throw new Error(
85+
`Can't set a session config entry: 'powerPreference' - ${webnnOptions.powerPreference}`);
86+
}
87+
}
88+
}
89+
break;
6790
case 'webgpu':
6891
epName = 'JS';
6992
break;

‎js/web/script/test-runner-cli-args.ts

+11-5
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ Options:
3737
webgpu
3838
wasm
3939
xnnpack
40+
webnn
4041
-e=<...>, --env=<...> Specify the environment to run the test. Should be one of the following:
4142
chrome (default)
4243
edge (Windows only)
@@ -104,7 +105,7 @@ Examples:
104105

105106
export declare namespace TestRunnerCliArgs {
106107
type Mode = 'suite0'|'suite1'|'model'|'unittest'|'op';
107-
type Backend = 'cpu'|'webgl'|'webgpu'|'wasm'|'onnxruntime'|'xnnpack';
108+
type Backend = 'cpu'|'webgl'|'webgpu'|'wasm'|'onnxruntime'|'xnnpack'|'webnn';
108109
type Environment = 'chrome'|'edge'|'firefox'|'electron'|'safari'|'node'|'bs';
109110
type BundleMode = 'prod'|'dev'|'perf';
110111
}
@@ -359,12 +360,12 @@ export function parseTestRunnerCliArgs(cmdlineArgs: string[]): TestRunnerCliArgs
359360
}
360361

361362
// Option: -b=<...>, --backend=<...>
362-
const browserBackends = ['webgl', 'webgpu', 'wasm', 'xnnpack'];
363+
const browserBackends = ['webgl', 'webgpu', 'wasm', 'xnnpack', 'webnn'];
363364

364-
// TODO: remove this when Chrome support WebGPU.
365-
// we need this for now because Chrome does not support webgpu yet,
365+
// TODO: remove this when Chrome support WebGPU or WebNN.
366+
// we need this for now because Chrome does not support webgpu and webnn yet,
366367
// and ChromeCanary is not in CI.
367-
const defaultBrowserBackends = ['webgl', /* 'webgpu', */ 'wasm', 'xnnpack'];
368+
const defaultBrowserBackends = ['webgl', /* 'webgpu', */ 'wasm', 'xnnpack' /*, 'webnn'*/];
368369
const nodejsBackends = ['cpu', 'wasm'];
369370
const backendArgs = args.backend || args.b;
370371
const backend = (typeof backendArgs !== 'string') ? (env === 'node' ? nodejsBackends : defaultBrowserBackends) :
@@ -377,6 +378,11 @@ export function parseTestRunnerCliArgs(cmdlineArgs: string[]): TestRunnerCliArgs
377378

378379
const globalEnvFlags = parseGlobalEnvFlags(args);
379380

381+
if (backend.includes('webnn') && !globalEnvFlags.wasm.proxy) {
382+
// Backend webnn is restricted in the dedicated worker.
383+
globalEnvFlags.wasm.proxy = true;
384+
}
385+
380386
// Options:
381387
// --log-verbose=<...>
382388
// --log-info=<...>

‎js/web/script/test-runner-cli.ts

+14-6
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ async function main() {
5353

5454
// The default backends and opset version lists. Those will be used in suite tests.
5555
const DEFAULT_BACKENDS: readonly TestRunnerCliArgs.Backend[] =
56-
args.env === 'node' ? ['cpu', 'wasm'] : ['wasm', 'webgl', 'webgpu'];
56+
args.env === 'node' ? ['cpu', 'wasm'] : ['wasm', 'webgl', 'webgpu', 'webnn'];
5757
const DEFAULT_OPSET_VERSIONS = fs.readdirSync(TEST_DATA_MODEL_NODE_ROOT, {withFileTypes: true})
5858
.filter(dir => dir.isDirectory() && dir.name.startsWith('opset'))
5959
.map(dir => dir.name.slice(5));
@@ -459,12 +459,13 @@ async function main() {
459459
// STEP 5. use Karma to run test
460460
npmlog.info('TestRunnerCli.Run', '(4/4) Running karma to start test runner...');
461461
const webgpu = args.backends.indexOf('webgpu') > -1;
462+
const webnn = args.backends.indexOf('webnn') > -1;
462463
const browser = getBrowserNameFromEnv(
463464
args.env,
464465
args.bundleMode === 'perf' ? 'perf' :
465466
args.debug ? 'debug' :
466467
'test',
467-
webgpu, config.options.globalEnvFlags?.webgpu?.profilingMode === 'default');
468+
webgpu, webnn, config.options.globalEnvFlags?.webgpu?.profilingMode === 'default');
468469
const karmaArgs = ['karma', 'start', `--browsers ${browser}`];
469470
if (args.debug) {
470471
karmaArgs.push('--log-level info --timeout-mocha 9999999');
@@ -474,7 +475,7 @@ async function main() {
474475
if (args.noSandbox) {
475476
karmaArgs.push('--no-sandbox');
476477
}
477-
if (webgpu) {
478+
if (webgpu || webnn) {
478479
karmaArgs.push('--force-localhost');
479480
}
480481
karmaArgs.push(`--bundle-mode=${args.bundleMode}`);
@@ -569,10 +570,10 @@ async function main() {
569570
}
570571

571572
function getBrowserNameFromEnv(
572-
env: TestRunnerCliArgs['env'], mode: 'debug'|'perf'|'test', webgpu: boolean, profile: boolean) {
573+
env: TestRunnerCliArgs['env'], mode: 'debug'|'perf'|'test', webgpu: boolean, webnn: boolean, profile: boolean) {
573574
switch (env) {
574575
case 'chrome':
575-
return selectChromeBrowser(mode, webgpu, profile);
576+
return selectChromeBrowser(mode, webgpu, webnn, profile);
576577
case 'edge':
577578
return 'Edge';
578579
case 'firefox':
@@ -588,14 +589,21 @@ async function main() {
588589
}
589590
}
590591

591-
function selectChromeBrowser(mode: 'debug'|'perf'|'test', webgpu: boolean, profile: boolean) {
592+
function selectChromeBrowser(mode: 'debug'|'perf'|'test', webgpu: boolean, webnn: boolean, profile: boolean) {
592593
if (webgpu) {
593594
switch (mode) {
594595
case 'debug':
595596
return profile ? 'ChromeCanaryProfileDebug' : 'ChromeCanaryDebug';
596597
default:
597598
return profile ? 'ChromeCanaryProfileTest' : 'ChromeCanaryDebug';
598599
}
600+
} else if (webnn) {
601+
switch (mode) {
602+
case 'debug':
603+
return 'ChromeCanaryDebug';
604+
default:
605+
return 'ChromeCanaryTest';
606+
}
599607
} else {
600608
switch (mode) {
601609
case 'debug':

0 commit comments

Comments
 (0)