From 6bda027a93216eed56ef7faecc843393fc4b4a43 Mon Sep 17 00:00:00 2001 From: Belem Zhang Date: Fri, 28 Mar 2025 12:34:11 +0800 Subject: [PATCH 1/2] Add webnn-gpu and webnn-npu support for MODNet --- video-background-removal/index.html | 3 ++ video-background-removal/main.js | 39 +++++++++++++++++- video-background-removal/package-lock.json | 46 +++++++++++----------- video-background-removal/package.json | 4 +- 4 files changed, 65 insertions(+), 27 deletions(-) diff --git a/video-background-removal/index.html b/video-background-removal/index.html index dc176798..f1d1c6c3 100644 --- a/video-background-removal/index.html +++ b/video-background-removal/index.html @@ -58,6 +58,9 @@

+
+ WebGPU · WebNN GPU · WebNN NPU +
diff --git a/video-background-removal/main.js b/video-background-removal/main.js index e5f9d1b6..749d23df 100644 --- a/video-background-removal/main.js +++ b/video-background-removal/main.js @@ -18,13 +18,44 @@ function setStreamSize(width, height) { status.textContent = "Loading model..."; +function getDeviceConfig(deviceParam) { + const defaultDevice = 'webgpu'; + const webnnDevices = ['webnn-gpu', 'webnn-cpu', 'webnn-npu']; + + const device = (deviceParam || defaultDevice).toLowerCase(); + const dtype = webnnDevices.includes(device) ? 'fp16' : 'fp32'; + + const FREE_DIMENSION_HEIGHT = 256; + const FREE_DIMENSION_WIDTH = 320; + + const sessionOptions = webnnDevices.includes(device) + ? { + freeDimensionOverrides: { + batch_size: 1, + height: FREE_DIMENSION_HEIGHT, + width: FREE_DIMENSION_WIDTH, + }, + } + : {}; + + return { device, dtype, sessionOptions }; +} + +const urlParams = new URLSearchParams(window.location.search); +let { device, dtype, sessionOptions } = getDeviceConfig(urlParams.get('device')); +if (!['webgpu', 'webnn-gpu', 'webnn-cpu', 'webnn-npu'].includes(device)) { + status.textContent = `Unsupported device ${device}. Falling back to WebGPU.`; + device = 'webgpu'; +} + // Load model and processor const model_id = "Xenova/modnet"; let model; try { model = await AutoModel.from_pretrained(model_id, { - device: "webgpu", - dtype: "fp32", // TODO: add fp16 support + device: device, + dtype: dtype, + session_options: sessionOptions }); } catch (err) { status.textContent = err.message; @@ -44,6 +75,10 @@ sizeSlider.addEventListener("input", () => { }); sizeSlider.disabled = false; +if (['webnn-gpu', 'webnn-cpu', 'webnn-npu'].includes(device)) { + sizeSlider.disabled = true; +} + let scale = 0.5; scaleSlider.addEventListener("input", () => { scale = Number(scaleSlider.value); diff --git a/video-background-removal/package-lock.json b/video-background-removal/package-lock.json index 01eb1baa..cbb38aea 100644 --- a/video-background-removal/package-lock.json +++ b/video-background-removal/package-lock.json @@ -8,7 +8,7 @@ "name": "video-background-removal", "version": "0.0.0", "dependencies": { - "@huggingface/transformers": "^3.2.2" + "@huggingface/transformers": "^3.4.1" }, "devDependencies": { "vite": "^6.0.5" @@ -424,21 +424,21 @@ } }, "node_modules/@huggingface/jinja": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.3.2.tgz", - "integrity": "sha512-F2FvuIc+w1blGsaqJI/OErRbWH6bVJDCBI8Rm5D86yZ2wlwrGERsfIaru7XUv9eYC3DMP3ixDRRtF0h6d8AZcQ==", + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.3.3.tgz", + "integrity": "sha512-vQQr2JyWvVFba3Lj9es4q9vCl1sAc74fdgnEMoX8qHrXtswap9ge9uO3ONDzQB0cQ0PUyaKY2N6HaVbTBvSXvw==", "engines": { "node": ">=18" } }, "node_modules/@huggingface/transformers": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/@huggingface/transformers/-/transformers-3.2.2.tgz", - "integrity": "sha512-b5I1NTFtjy6bNSrlbvZaMiMfl1RE1n9Vr8ZOYonC+xDY0Nuw7hsdxHRq2FTsAiw0jhDcX5A58ru8VlfqCY8n5w==", + "version": "3.4.1", + "resolved": "https://registry.npmjs.org/@huggingface/transformers/-/transformers-3.4.1.tgz", + "integrity": "sha512-Inbvq9i/33kmd5XHom9MQU7NAOV5UcGmHBwBk9NFw4IPhdoTnfP7wFJxJmceYhRdS+EL1Hpw4he/Ceimau6ORg==", "dependencies": { - "@huggingface/jinja": "^0.3.2", + "@huggingface/jinja": "^0.3.3", "onnxruntime-node": "1.20.1", - "onnxruntime-web": "1.21.0-dev.20241205-d27fecd3d3", + "onnxruntime-web": "1.22.0-dev.20250306-ccf8fdd9ea", "sharp": "^0.33.5" } }, @@ -1287,9 +1287,9 @@ } }, "node_modules/flatbuffers": { - "version": "1.12.0", - "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-1.12.0.tgz", - "integrity": "sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ==" + "version": "25.2.10", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.2.10.tgz", + "integrity": "sha512-7JlN9ZvLDG1McO3kbX0k4v+SUAg48L1rIwEvN6ZQl/eCtgJz9UylTMzE9wrmYrcorgxm3CX/3T/w5VAub99UUw==" }, "node_modules/foreground-child": { "version": "3.3.0", @@ -1377,9 +1377,9 @@ } }, "node_modules/long": { - "version": "5.2.3", - "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", - "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==" + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.1.tgz", + "integrity": "sha512-ka87Jz3gcx/I7Hal94xaN2tZEOPoUOEVftkQqZx2EeQRN7LGdfLlI3FvZ+7WDplm+vK2Urx9ULrvSowtdCieng==" }, "node_modules/lru-cache": { "version": "10.4.3", @@ -1473,22 +1473,22 @@ } }, "node_modules/onnxruntime-web": { - "version": "1.21.0-dev.20241205-d27fecd3d3", - "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.21.0-dev.20241205-d27fecd3d3.tgz", - "integrity": "sha512-neeC9mv1sFWjUFrTaDl7enufNxbtSSTwR5V2i35ga4yXWS6r1MbpUwWwD1X+VKANujbSG8M5pk/ohRAOm2QhMQ==", + "version": "1.22.0-dev.20250306-ccf8fdd9ea", + "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.22.0-dev.20250306-ccf8fdd9ea.tgz", + "integrity": "sha512-YwqS9Qqx2eKFXIx+HQloqRUG5/STHPUuNk8wn+qVVmwXBIfNdXX0/Lm7wgo5CnC2k+yqZmjDV5V1dZi4PeSPGQ==", "dependencies": { - "flatbuffers": "^1.12.0", + "flatbuffers": "^25.1.24", "guid-typescript": "^1.0.9", "long": "^5.2.3", - "onnxruntime-common": "1.21.0-dev.20241205-6ed77cc374", + "onnxruntime-common": "1.22.0-dev.20250306-aafa8d170a", "platform": "^1.3.6", "protobufjs": "^7.2.4" } }, "node_modules/onnxruntime-web/node_modules/onnxruntime-common": { - "version": "1.21.0-dev.20241205-6ed77cc374", - "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.21.0-dev.20241205-6ed77cc374.tgz", - "integrity": "sha512-U4DGq/dZiboIEK0Zv1KUuWJesJ/txUALpWSXwI8kqOCSxe8GrI65xfRFeMbqYFhPVGAWZPsBpT1zo1s4ksrlrg==" + "version": "1.22.0-dev.20250306-aafa8d170a", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.22.0-dev.20250306-aafa8d170a.tgz", + "integrity": "sha512-NfIQnW4lIk/8LnhnYqknYPeet0U0+AADgKQRlKex36QrNoVSCY+aNaX6wyy2VzQ4CNWxsYh0E203ajRD/zxn0g==" }, "node_modules/package-json-from-dist": { "version": "1.0.1", diff --git a/video-background-removal/package.json b/video-background-removal/package.json index 41965be5..4e28883f 100644 --- a/video-background-removal/package.json +++ b/video-background-removal/package.json @@ -4,7 +4,7 @@ "version": "0.0.0", "type": "module", "scripts": { - "dev": "vite", + "dev": "vite --host", "build": "vite build", "preview": "vite preview" }, @@ -12,6 +12,6 @@ "vite": "^6.0.5" }, "dependencies": { - "@huggingface/transformers": "^3.2.2" + "@huggingface/transformers": "^3.4.1" } } From 0179b41cf3f58c9cbc4f77d20872c4a868931242 Mon Sep 17 00:00:00 2001 From: Belem Zhang Date: Fri, 28 Mar 2025 12:47:04 +0800 Subject: [PATCH 2/2] Remove --host --- video-background-removal/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/video-background-removal/package.json b/video-background-removal/package.json index 4e28883f..d4279f10 100644 --- a/video-background-removal/package.json +++ b/video-background-removal/package.json @@ -4,7 +4,7 @@ "version": "0.0.0", "type": "module", "scripts": { - "dev": "vite --host", + "dev": "vite", "build": "vite build", "preview": "vite preview" },