Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support WebNN EP #15698

Merged
merged 7 commits into from
May 9, 2023
Merged

Support WebNN EP #15698

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
cbba0dd
Support WebNN EP
Honry Apr 14, 2023
74ee55a
Fixed lint errors
Honry Apr 27, 2023
6db9969
Did npm run format
Honry Apr 27, 2023
ad461ad
Throw error when wasm-enable-proxy is false for webnn ep
Honry Apr 27, 2023
cc8084e
Addressed @fs-eire's comments
Honry Apr 28, 2023
4653246
Addressed further comments
Honry May 4, 2023
58a8bec
Fixed nit
Honry May 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions cmake/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@ option(onnxruntime_TVM_CUDA_RUNTIME "Build TVM with CUDA support" OFF)
option(onnxruntime_TVM_USE_LLVM "Build TVM with LLVM. Set customized path to llvm-config.exe here if need" OFF)
option(onnxruntime_TVM_USE_HASH "Build ipp-crypto library for support hash algorithm. It is defined for TVM only")
option(onnxruntime_USE_XNNPACK "Build with XNNPACK support. Provides an alternative math library on ARM, WebAssembly and x86." OFF)
option(onnxruntime_USE_WEBNN "Build with WebNN support. Enable hardware acceleration in web browsers." OFF)

# Options related to reducing the binary size produced by the build
# XNNPACK EP requires the internal NHWC contrib ops to be available, so this option must be OFF when onnxruntime_USE_XNNPACK is ON
Expand Down Expand Up @@ -722,6 +723,11 @@ if (onnxruntime_USE_XNNPACK)
list(APPEND ORT_PROVIDER_CMAKE_FLAGS -Donnxruntime_USE_XNNPACK=1)
list(APPEND ONNXRUNTIME_PROVIDER_NAMES xnnpack)
endif()
if (onnxruntime_USE_WEBNN)
list(APPEND ORT_PROVIDER_FLAGS -DUSE_WEBNN=1)
list(APPEND ORT_PROVIDER_CMAKE_FLAGS -Donnxruntime_USE_WEBNN=1)
list(APPEND ONNXRUNTIME_PROVIDER_NAMES webnn)
endif()
if (onnxruntime_USE_CANN)
list(APPEND ORT_PROVIDER_FLAGS -DUSE_CANN=1)
list(APPEND ORT_PROVIDER_CMAKE_FLAGS -Donnxruntime_USE_CANN=1)
Expand Down
1 change: 1 addition & 0 deletions cmake/onnxruntime.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,7 @@ set(onnxruntime_INTERNAL_LIBRARIES
${PROVIDERS_ROCM}
${PROVIDERS_VITISAI}
${PROVIDERS_XNNPACK}
${PROVIDERS_WEBNN}
${PROVIDERS_AZURE}
${PROVIDERS_INTERNAL_TESTING}
${onnxruntime_winml}
Expand Down
28 changes: 28 additions & 0 deletions cmake/onnxruntime_providers.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,9 @@ endif()
if (onnxruntime_USE_XNNPACK)
set(PROVIDERS_XNNPACK onnxruntime_providers_xnnpack)
endif()
if(onnxruntime_USE_WEBNN)
set(PROVIDERS_WEBNN onnxruntime_providers_webnn)
endif()
if(onnxruntime_USE_SNPE)
include(onnxruntime_snpe_provider.cmake)
endif()
Expand Down Expand Up @@ -983,6 +986,31 @@ if (onnxruntime_USE_COREML)
endif()
endif()

if (onnxruntime_USE_WEBNN)
if (onnxruntime_MINIMAL_BUILD AND NOT onnxruntime_EXTENDED_MINIMAL_BUILD)
message(FATAL_ERROR "WebNN EP can not be used in a basic minimal build. Please build with '--minimal_build extended'")
endif()

add_compile_definitions(USE_WEBNN=1)
if (onnxruntime_ENABLE_WEBASSEMBLY_THREADS)
add_definitions(-DENABLE_WEBASSEMBLY_THREADS=1)
endif()
file(GLOB_RECURSE onnxruntime_providers_webnn_cc_srcs CONFIGURE_DEPENDS
"${ONNXRUNTIME_ROOT}/core/providers/webnn/*.h"
"${ONNXRUNTIME_ROOT}/core/providers/webnn/*.cc"
"${ONNXRUNTIME_ROOT}/core/providers/shared/utils/utils.h"
"${ONNXRUNTIME_ROOT}/core/providers/shared/utils/utils.cc"
)

source_group(TREE ${REPO_ROOT} FILES ${onnxruntime_providers_webnn_cc_srcs})
onnxruntime_add_static_library(onnxruntime_providers_webnn ${onnxruntime_providers_webnn_cc_srcs})
onnxruntime_add_include_to_target(onnxruntime_providers_webnn onnxruntime_common onnx onnx_proto Boost::mp11)

add_dependencies(onnxruntime_providers_webnn onnx ${onnxruntime_EXTERNAL_DEPENDENCIES})
set_target_properties(onnxruntime_providers_webnn PROPERTIES FOLDER "ONNXRuntime")
set_target_properties(onnxruntime_providers_webnn PROPERTIES LINKER_LANGUAGE CXX)
endif()

if (onnxruntime_USE_NNAPI_BUILTIN)
if (onnxruntime_MINIMAL_BUILD AND NOT onnxruntime_EXTENDED_MINIMAL_BUILD)
message(FATAL_ERROR "NNAPI can not be used in a basic minimal build. Please build with '--minimal_build extended'")
Expand Down
10 changes: 10 additions & 0 deletions cmake/onnxruntime_webassembly.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ if (onnxruntime_BUILD_WEBASSEMBLY_STATIC_LIB)
onnxruntime_providers
${PROVIDERS_JS}
${PROVIDERS_XNNPACK}
${PROVIDERS_WEBNN}
onnxruntime_session
onnxruntime_util
re2::re2
Expand Down Expand Up @@ -186,6 +187,7 @@ else()
onnxruntime_providers
${PROVIDERS_JS}
${PROVIDERS_XNNPACK}
${PROVIDERS_WEBNN}
onnxruntime_session
onnxruntime_util
re2::re2
Expand All @@ -194,6 +196,10 @@ else()
target_link_libraries(onnxruntime_webassembly PRIVATE XNNPACK)
endif()

if(onnxruntime_USE_WEBNN)
target_link_libraries(onnxruntime_webassembly PRIVATE onnxruntime_providers_webnn)
endif()

if (onnxruntime_ENABLE_TRAINING)
target_link_libraries(onnxruntime_webassembly PRIVATE tensorboard)
endif()
Expand Down Expand Up @@ -255,6 +261,10 @@ else()
)
endif()

if (onnxruntime_USE_WEBNN)
set_property(TARGET onnxruntime_webassembly APPEND_STRING PROPERTY LINK_FLAGS " --bind")
endif()

# Set link flag to enable exceptions support, this will override default disabling exception throwing behavior when disable exceptions.
target_link_options(onnxruntime_webassembly PRIVATE "SHELL:-s DISABLE_EXCEPTION_THROWING=0")

Expand Down
1 change: 1 addition & 0 deletions include/onnxruntime/core/graph/constants.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ constexpr const char* kJsExecutionProvider = "JsExecutionProvider";
constexpr const char* kSnpeExecutionProvider = "SNPEExecutionProvider";
constexpr const char* kTvmExecutionProvider = "TvmExecutionProvider";
constexpr const char* kXnnpackExecutionProvider = "XnnpackExecutionProvider";
constexpr const char* kWebNNExecutionProvider = "WebNNExecutionProvider";
constexpr const char* kCannExecutionProvider = "CANNExecutionProvider";
constexpr const char* kAzureExecutionProvider = "AzureExecutionProvider";

Expand Down
8 changes: 7 additions & 1 deletion js/common/lib/inference-session.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,14 +165,15 @@ export declare namespace InferenceSession {

// Currently, we have the following backends to support execution providers:
// Backend Node.js binding: supports 'cpu' and 'cuda'.
// Backend WebAssembly: supports 'cpu', 'wasm' and 'xnnpack'.
// Backend WebAssembly: supports 'cpu', 'wasm', 'xnnpack' and 'webnn'.
// Backend ONNX.js: supports 'webgl'.
interface ExecutionProviderOptionMap {
cpu: CpuExecutionProviderOption;
cuda: CudaExecutionProviderOption;
wasm: WebAssemblyExecutionProviderOption;
webgl: WebGLExecutionProviderOption;
xnnpack: XnnpackExecutionProviderOption;
webnn: WebNNExecutionProviderOption;
}

type ExecutionProviderName = keyof ExecutionProviderOptionMap;
Expand Down Expand Up @@ -200,6 +201,11 @@ export declare namespace InferenceSession {
export interface XnnpackExecutionProviderOption extends ExecutionProviderOption {
readonly name: 'xnnpack';
}
export interface WebNNExecutionProviderOption extends ExecutionProviderOption {
readonly name: 'webnn';
deviceType?: 'cpu'|'gpu';
powerPreference?: 'default'|'low-power'|'high-performance';
}
// #endregion

// #endregion
Expand Down
57 changes: 50 additions & 7 deletions js/web/karma.conf.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,13 +86,56 @@ module.exports = function (config) {
hostname,
listenAddress,
customLaunchers: {
ChromeTest: { base: 'ChromeHeadless', flags: ['--enable-features=SharedArrayBuffer'] },
ChromePerf: { base: 'Chrome', flags: ['--window-size=1,1', '--enable-features=SharedArrayBuffer'] },
ChromeDebug: { debug: true, base: 'Chrome', flags: ['--remote-debugging-port=9333', '--enable-features=SharedArrayBuffer'] },
ChromeCanaryTest: { base: 'ChromeCanary', flags: ['--window-size=1,1', '--enable-features=SharedArrayBuffer', '--enable-unsafe-webgpu'] },
ChromeCanaryProfileTest: { base: 'ChromeCanary', flags: ['--window-size=1,1', '--enable-features=SharedArrayBuffer', '--enable-unsafe-webgpu', '--disable-dawn-features=disallow_unsafe_apis'] },
ChromeCanaryDebug: { debug: true, base: 'ChromeCanary', flags: ['--remote-debugging-port=9333', '--enable-features=SharedArrayBuffer', '--enable-unsafe-webgpu'] },
ChromeCanaryProfileDebug: { debug: true, base: 'ChromeCanary', flags: ['--remote-debugging-port=9333', '--enable-features=SharedArrayBuffer', '--enable-unsafe-webgpu', '--disable-dawn-features=disallow_unsafe_apis'] },
ChromeTest: {
base: 'ChromeHeadless',
flags: ['--enable-features=SharedArrayBuffer']
},
ChromePerf: {
base: 'Chrome',
flags: ['--window-size=1,1', '--enable-features=SharedArrayBuffer']
},
ChromeDebug: {
debug: true,
base: 'Chrome', flags: ['--remote-debugging-port=9333', '--enable-features=SharedArrayBuffer']
},
ChromeCanaryTest: {
base: 'ChromeCanary',
flags: [
'--window-size=1,1',
'--enable-features=SharedArrayBuffer',
'--enable-unsafe-webgpu',
'--enable-experimental-web-platform-features'
]
},
ChromeCanaryProfileTest: {
base: 'ChromeCanary',
flags: [
'--window-size=1,1',
'--enable-features=SharedArrayBuffer',
'--enable-unsafe-webgpu',
'--disable-dawn-features=disallow_unsafe_apis'
]
},
ChromeCanaryDebug: {
debug: true,
base: 'ChromeCanary',
flags: [
'--remote-debugging-port=9333',
'--enable-features=SharedArrayBuffer',
'--enable-unsafe-webgpu',
'--enable-experimental-web-platform-features'
]
},
ChromeCanaryProfileDebug: {
debug: true,
base: 'ChromeCanary',
flags: [
'--remote-debugging-port=9333',
'--enable-features=SharedArrayBuffer',
'--enable-unsafe-webgpu',
'--disable-dawn-features=disallow_unsafe_apis',
]
},
//
// ==== BrowserStack browsers ====
//
Expand Down
1 change: 1 addition & 0 deletions js/web/lib/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,5 @@ if (!BUILD_DEFS.DISABLE_WASM) {
registerBackend('cpu', wasmBackend, 10);
registerBackend('wasm', wasmBackend, 10);
registerBackend('xnnpack', wasmBackend, 9);
registerBackend('webnn', wasmBackend, 9);
}
23 changes: 23 additions & 0 deletions js/web/lib/wasm/session-options.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,29 @@ const setExecutionProviders =
case 'xnnpack':
epName = 'XNNPACK';
break;
case 'webnn':
epName = 'WEBNN';
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if we should throw if proxy is not set; might save devs some time to debug.

Copy link
Contributor

@guschmue guschmue May 1, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

size increase:
ort-wasm-simd.wasm / no webnn: 8870924
ort-wasm-simd.wasm / with webnn: 9535743
~650KB, should be manageable.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if we should throw if proxy is not set; might save devs some time to debug.

Indeed, I will set proxy to true once webnn backend is used.

if (typeof ep !== 'string') {
const webnnOptions = ep as InferenceSession.WebNNExecutionProviderOption;
if (webnnOptions?.deviceType) {
const keyDataOffset = allocWasmString('deviceType', allocs);
const valueDataOffset = allocWasmString(webnnOptions.deviceType, allocs);
if (getInstance()._OrtAddSessionConfigEntry(sessionOptionsHandle, keyDataOffset, valueDataOffset) !==
0) {
throw new Error(`Can't set a session config entry: 'deviceType' - ${webnnOptions.deviceType}`);
}
}
if (webnnOptions?.powerPreference) {
const keyDataOffset = allocWasmString('powerPreference', allocs);
const valueDataOffset = allocWasmString(webnnOptions.powerPreference, allocs);
if (getInstance()._OrtAddSessionConfigEntry(sessionOptionsHandle, keyDataOffset, valueDataOffset) !==
0) {
throw new Error(
`Can't set a session config entry: 'powerPreference' - ${webnnOptions.powerPreference}`);
}
}
}
break;
case 'webgpu':
epName = 'JS';
break;
Expand Down
16 changes: 11 additions & 5 deletions js/web/script/test-runner-cli-args.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ Options:
webgpu
wasm
xnnpack
webnn
-e=<...>, --env=<...> Specify the environment to run the test. Should be one of the following:
chrome (default)
edge (Windows only)
Expand Down Expand Up @@ -104,7 +105,7 @@ Examples:

export declare namespace TestRunnerCliArgs {
type Mode = 'suite0'|'suite1'|'model'|'unittest'|'op';
type Backend = 'cpu'|'webgl'|'webgpu'|'wasm'|'onnxruntime'|'xnnpack';
type Backend = 'cpu'|'webgl'|'webgpu'|'wasm'|'onnxruntime'|'xnnpack'|'webnn';
type Environment = 'chrome'|'edge'|'firefox'|'electron'|'safari'|'node'|'bs';
type BundleMode = 'prod'|'dev'|'perf';
}
Expand Down Expand Up @@ -359,12 +360,12 @@ export function parseTestRunnerCliArgs(cmdlineArgs: string[]): TestRunnerCliArgs
}

// Option: -b=<...>, --backend=<...>
const browserBackends = ['webgl', 'webgpu', 'wasm', 'xnnpack'];
const browserBackends = ['webgl', 'webgpu', 'wasm', 'xnnpack', 'webnn'];

// TODO: remove this when Chrome support WebGPU.
// we need this for now because Chrome does not support webgpu yet,
// TODO: remove this when Chrome support WebGPU or WebNN.
// we need this for now because Chrome does not support webgpu and webnn yet,
// and ChromeCanary is not in CI.
const defaultBrowserBackends = ['webgl', /* 'webgpu', */ 'wasm', 'xnnpack'];
const defaultBrowserBackends = ['webgl', /* 'webgpu', */ 'wasm', 'xnnpack' /*, 'webnn'*/];
const nodejsBackends = ['cpu', 'wasm'];
const backendArgs = args.backend || args.b;
const backend = (typeof backendArgs !== 'string') ? (env === 'node' ? nodejsBackends : defaultBrowserBackends) :
Expand All @@ -377,6 +378,11 @@ export function parseTestRunnerCliArgs(cmdlineArgs: string[]): TestRunnerCliArgs

const globalEnvFlags = parseGlobalEnvFlags(args);

if (backend.includes('webnn') && !globalEnvFlags.wasm.proxy) {
// Backend webnn is restricted in the dedicated worker.
globalEnvFlags.wasm.proxy = true;
}

// Options:
// --log-verbose=<...>
// --log-info=<...>
Expand Down
20 changes: 14 additions & 6 deletions js/web/script/test-runner-cli.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ async function main() {

// The default backends and opset version lists. Those will be used in suite tests.
const DEFAULT_BACKENDS: readonly TestRunnerCliArgs.Backend[] =
args.env === 'node' ? ['cpu', 'wasm'] : ['wasm', 'webgl', 'webgpu'];
args.env === 'node' ? ['cpu', 'wasm'] : ['wasm', 'webgl', 'webgpu', 'webnn'];
const DEFAULT_OPSET_VERSIONS = fs.readdirSync(TEST_DATA_MODEL_NODE_ROOT, {withFileTypes: true})
.filter(dir => dir.isDirectory() && dir.name.startsWith('opset'))
.map(dir => dir.name.slice(5));
Expand Down Expand Up @@ -459,12 +459,13 @@ async function main() {
// STEP 5. use Karma to run test
npmlog.info('TestRunnerCli.Run', '(4/4) Running karma to start test runner...');
const webgpu = args.backends.indexOf('webgpu') > -1;
const webnn = args.backends.indexOf('webnn') > -1;
const browser = getBrowserNameFromEnv(
args.env,
args.bundleMode === 'perf' ? 'perf' :
args.debug ? 'debug' :
'test',
webgpu, config.options.globalEnvFlags?.webgpu?.profilingMode === 'default');
webgpu, webnn, config.options.globalEnvFlags?.webgpu?.profilingMode === 'default');
const karmaArgs = ['karma', 'start', `--browsers ${browser}`];
if (args.debug) {
karmaArgs.push('--log-level info --timeout-mocha 9999999');
Expand All @@ -474,7 +475,7 @@ async function main() {
if (args.noSandbox) {
karmaArgs.push('--no-sandbox');
}
if (webgpu) {
if (webgpu || webnn) {
karmaArgs.push('--force-localhost');
}
karmaArgs.push(`--bundle-mode=${args.bundleMode}`);
Expand Down Expand Up @@ -569,10 +570,10 @@ async function main() {
}

function getBrowserNameFromEnv(
env: TestRunnerCliArgs['env'], mode: 'debug'|'perf'|'test', webgpu: boolean, profile: boolean) {
env: TestRunnerCliArgs['env'], mode: 'debug'|'perf'|'test', webgpu: boolean, webnn: boolean, profile: boolean) {
switch (env) {
case 'chrome':
return selectChromeBrowser(mode, webgpu, profile);
return selectChromeBrowser(mode, webgpu, webnn, profile);
case 'edge':
return 'Edge';
case 'firefox':
Expand All @@ -588,14 +589,21 @@ async function main() {
}
}

function selectChromeBrowser(mode: 'debug'|'perf'|'test', webgpu: boolean, profile: boolean) {
function selectChromeBrowser(mode: 'debug'|'perf'|'test', webgpu: boolean, webnn: boolean, profile: boolean) {
if (webgpu) {
switch (mode) {
case 'debug':
return profile ? 'ChromeCanaryProfileDebug' : 'ChromeCanaryDebug';
default:
return profile ? 'ChromeCanaryProfileTest' : 'ChromeCanaryDebug';
}
} else if (webnn) {
switch (mode) {
case 'debug':
return 'ChromeCanaryDebug';
default:
return 'ChromeCanaryTest';
}
} else {
switch (mode) {
case 'debug':
Expand Down
Loading