diff --git a/.gitattributes b/.gitattributes index 56c4ab59..46f81ac5 100644 --- a/.gitattributes +++ b/.gitattributes @@ -5,3 +5,4 @@ *.csv filter=lfs diff=lfs merge=lfs -text *.png filter=lfs diff=lfs merge=lfs -text *.excalidraw filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 98c45c9f..01a29fb2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -46,10 +46,12 @@ jobs: run: npm install @playwright/test@${{ env.PLAYWRIGHT_VERSION }} - name: Install Playwright Browsers run: npx playwright install --with-deps ${{ inputs.browser }} - if: steps.playwright-cache.outputs.cache-hit != 'true' + # disabled for now as we have caching issues: + #_if: steps.playwright-cache.outputs.cache-hit != 'true' - name: Install system dependencies for WebKit # Some WebKit dependencies seem to lay outside the cache and will need to be installed separately - if: ${{ inputs.browser == 'webkit' && steps.playwright-cache.outputs.cache-hit == 'true' }} + # disabled for now as we have caching issues: + #_if: ${{ inputs.browser == 'webkit' && steps.playwright-cache.outputs.cache-hit == 'true' }} run: npx playwright install-deps webkit - name: Run build run: npm run build:ci diff --git a/forge.config.js b/forge.config.js index 79b3d07d..c261015e 100644 --- a/forge.config.js +++ b/forge.config.js @@ -1,24 +1,68 @@ +// for some reason using forge.config.ts doesn't work, +// +// it says: +// Failed to load: /Users/jbilcke/Projects/clapper/forge.config.ts +// An unhandled rejection has occurred inside Forge: +// SyntaxError: Unexpected token 'export' +// +// so we cannot use this: +// import type { ForgeConfig } from '@electron-forge/shared-types'; +// export const config: ForgeConfig = { module.exports = { packagerConfig: { - asar: false, // true, - icon: "./public/icon", - osxSign: {} + name: "Clapper", + asar: true, + icon: "./public/images/logos/CL.png", + osxSign: {}, + + // One or more files to be copied directly into the app's + // Contents/Resources directory for macOS target platforms + // and the resources directory for other target platforms. + // The resources directory can be referenced in the packaged + // app via the process.resourcesPath value. + extraResource: [ + ".next/standalone" + ], + // ignore: ['^\\/public$', '^\\/node_modules$', '^\\/src$', '^\\/[.].+'], + + // Walks the node_modules dependency tree to remove all of + // the packages specified in the devDependencies section of + // package.json from the outputted Electron app. + prune: true, + + ignore: [ + '^\\/.next$', + '^\\/src$', + '^\\/documentation$', + '^\\/test-results$', + '^\\/playwright-report$', + '^\\/.github$', + '^\\/public$', + '^\\/out$', + '^\\/tests$', + '^\\/Dockerfile$', + '^\\/package-lock.json$', + '^\\/.git$', + ], }, rebuildConfig: {}, makers: [ { name: '@electron-forge/maker-squirrel', - config: {}, + config: { + authors: "Clapper contributors" + } }, { name: '@electron-forge/maker-zip', platforms: ['darwin'], + config: {}, }, { name: '@electron-forge/maker-deb', config: { options: { - icon: './public/icon.png' + icon: './public/images/logos/CL.png' } }, }, @@ -26,7 +70,7 @@ module.exports = { name: '@electron-forge/maker-dmg', config: { options: { - icon: './public/icon.icns' + icon: './public/images/logos/CL.icns' } }, }, @@ -45,3 +89,4 @@ module.exports = { */ ], }; + diff --git a/main.js b/main.js index 6888caa1..bb052a6b 100644 --- a/main.js +++ b/main.js @@ -1,6 +1,16 @@ const fs = require('fs') const path = require('path') + +// ----------------------------------------------------------- +// +// attention: if you add dependencies here, you might have to edit +// forge.config.js, , the part about: +// '^\\/node_modules/(?!dotenv)$', +// +// if you have an idea to do that automatically, let me know const dotenv = require('dotenv') +// +// ----------------------------------------------------------- dotenv.config() @@ -20,12 +30,14 @@ try { const { app, BrowserWindow, screen } = require('electron') -const currentDir = process.cwd() - -const mainServerPath = path.join(currentDir, '.next/standalone/server.js') -// now we load the main server -require(mainServerPath) +try { + // used when the app is built with `npm run electron:make` + require(path.join(process.resourcesPath, 'standalone/server.js')) +} catch (err) { + // used when the app is started with `npm run electron:start` + require(path.join(process.cwd(), '.next/standalone/server.js')) +} // TODO: load the proxy server (for AI providers that refuse browser-side clients) // const proxyServerPath = path.join(currentDir, '.next/standalone/proxy-server.js') diff --git a/package-lock.json b/package-lock.json index 27cc454a..b76f0f9d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,19 +1,19 @@ { - "name": "@aitube/clapper", + "name": "clapper", "version": "0.0.5", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "@aitube/clapper", + "name": "clapper", "version": "0.0.5", "license": "GPL-3.0-only", "dependencies": { "@aitube/broadway": "0.0.22", "@aitube/clap": "0.0.30", - "@aitube/clapper-services": "0.0.28", + "@aitube/clapper-services": "0.0.29", "@aitube/engine": "0.0.26", - "@aitube/timeline": "0.0.42", + "@aitube/timeline": "0.0.43", "@fal-ai/serverless-client": "^0.13.0", "@ffmpeg/ffmpeg": "^0.12.10", "@ffmpeg/util": "^0.12.1", @@ -110,6 +110,7 @@ "@electron-forge/maker-squirrel": "^7.4.0", "@electron-forge/maker-zip": "^7.4.0", "@electron-forge/plugin-auto-unpack-natives": "^7.4.0", + "@electron-forge/publisher-github": "^7.4.0", "@playwright/test": "^1.45.1", "@testing-library/react": "^16.0.0", "@types/fluent-ffmpeg": "^2.1.24", @@ -160,12 +161,12 @@ } }, "node_modules/@aitube/clapper-services": { - "version": "0.0.28", - "resolved": "https://registry.npmjs.org/@aitube/clapper-services/-/clapper-services-0.0.28.tgz", - "integrity": "sha512-YmiPGAGtZcgqqnXmgtWvFyd9TL5eCpF343+ndvDa+aorQ43PkIKo7f3df550njOICM5KZhSBRFEIh1c5Dsi7Yg==", + "version": "0.0.29", + "resolved": "https://registry.npmjs.org/@aitube/clapper-services/-/clapper-services-0.0.29.tgz", + "integrity": "sha512-61UH/TQwPcvXArEkPnGNm+IQulaW3zNh73pzihdU2kkqufGzUYCNSd/jHJh9dLqQm3lZtm6QMN2RReFrzGuLNQ==", "peerDependencies": { "@aitube/clap": "0.0.30", - "@aitube/timeline": "0.0.42", + "@aitube/timeline": "0.0.43", "@monaco-editor/react": "4.6.0", "monaco-editor": "0.50.0", "react": "*", @@ -191,9 +192,9 @@ } }, "node_modules/@aitube/timeline": { - "version": "0.0.42", - "resolved": "https://registry.npmjs.org/@aitube/timeline/-/timeline-0.0.42.tgz", - "integrity": "sha512-H9vvHrrOBsTpU7AeM1+PJ7TbGr0Jb63aRGBXXL8CUVawX2L92BIkoEBXhGSVePP6yeS6oUv/u86Z2rHPLW9ChQ==", + "version": "0.0.43", + "resolved": "https://registry.npmjs.org/@aitube/timeline/-/timeline-0.0.43.tgz", + "integrity": "sha512-TnzKrB955YeDKOMWsnniGbQ+qulCmGptMfhNjDLEqA6jRcsnPVUFCR2dQBqWGNn6KfFPXmDvSi0Sihy7Oj98Aw==", "dependencies": { "date-fns": "^3.6.0", "react-virtualized-auto-sizer": "^1.0.24" @@ -2037,6 +2038,43 @@ "node": ">= 16.4.0" } }, + "node_modules/@electron-forge/publisher-github": { + "version": "7.4.0", + "resolved": "https://registry.npmjs.org/@electron-forge/publisher-github/-/publisher-github-7.4.0.tgz", + "integrity": "sha512-hrxKNssJyU8Yuz0qv384y5RKojMG0nWeG7/kidjp8PX/RnqjGRU/JJ0Worl28g8LGiLt5R5JIfNLngLaFMn8tg==", + "dev": true, + "dependencies": { + "@electron-forge/publisher-base": "7.4.0", + "@electron-forge/shared-types": "7.4.0", + "@octokit/core": "^3.2.4", + "@octokit/plugin-retry": "^3.0.9", + "@octokit/request-error": "^2.0.5", + "@octokit/rest": "^18.0.11", + "@octokit/types": "^6.1.2", + "chalk": "^4.0.0", + "debug": "^4.3.1", + "fs-extra": "^10.0.0", + "log-symbols": "^4.0.0", + "mime-types": "^2.1.25" + }, + "engines": { + "node": ">= 16.4.0" + } + }, + "node_modules/@electron-forge/publisher-github/node_modules/fs-extra": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", + "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/@electron-forge/shared-types": { "version": "7.4.0", "resolved": "https://registry.npmjs.org/@electron-forge/shared-types/-/shared-types-7.4.0.tgz", @@ -2513,6 +2551,18 @@ "node": ">=12" } }, + "node_modules/@electron/rebuild/node_modules/fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "dev": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/@electron/rebuild/node_modules/http-proxy-agent": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", @@ -2575,6 +2625,18 @@ "node": ">=10" } }, + "node_modules/@electron/rebuild/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/@electron/rebuild/node_modules/minipass-collect": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/minipass-collect/-/minipass-collect-1.0.2.tgz", @@ -2694,6 +2756,12 @@ "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, + "node_modules/@electron/rebuild/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + }, "node_modules/@electron/universal": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/@electron/universal/-/universal-2.0.1.tgz", @@ -3846,24 +3914,24 @@ } }, "node_modules/@inquirer/confirm": { - "version": "3.1.16", - "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-3.1.16.tgz", - "integrity": "sha512-DXgLZim+YVTk05zRywvFRfJt2Jje7sZ4DO6Ss9RpGtgXEd/T0IiTqubHWst0IazCwdPI9g/06Rtm/nm4IBFJBA==", + "version": "3.1.17", + "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-3.1.17.tgz", + "integrity": "sha512-qCpt/AABzPynz8tr69VDvhcjwmzAryipWXtW8Vi6m651da4H/d0Bdn55LkxXD7Rp2gfgxvxzTdb66AhIA8gzBA==", "dependencies": { - "@inquirer/core": "^9.0.4", - "@inquirer/type": "^1.5.0" + "@inquirer/core": "^9.0.5", + "@inquirer/type": "^1.5.1" }, "engines": { "node": ">=18" } }, "node_modules/@inquirer/core": { - "version": "9.0.4", - "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-9.0.4.tgz", - "integrity": "sha512-46LaWACIctSfVKTu71ziFlqO8SVLhWGSxvaHpf0frfDTphSSpIfeNo5ZH/kJPHYJw4VgPGf/9c3zJN/FnCdaIQ==", + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-9.0.5.tgz", + "integrity": "sha512-QWG41I7vn62O9stYKg/juKXt1PEbr/4ZZCPb4KgXDQGwgA9M5NBTQ7FnOvT1ridbxkm/wTxLCNraUs7y47pIRQ==", "dependencies": { - "@inquirer/figures": "^1.0.4", - "@inquirer/type": "^1.5.0", + "@inquirer/figures": "^1.0.5", + "@inquirer/type": "^1.5.1", "@types/mute-stream": "^0.0.4", "@types/node": "^20.14.11", "@types/wrap-ansi": "^3.0.0", @@ -3981,17 +4049,17 @@ } }, "node_modules/@inquirer/figures": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.4.tgz", - "integrity": "sha512-R7Gsg6elpuqdn55fBH2y9oYzrU/yKrSmIsDX4ROT51vohrECFzTf2zw9BfUbOW8xjfmM2QbVoVYdTwhrtEKWSQ==", + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.5.tgz", + "integrity": "sha512-79hP/VWdZ2UVc9bFGJnoQ/lQMpL74mGgzSYX1xUqCVk7/v73vJCMw1VuyWN1jGkZ9B3z7THAbySqGbCNefcjfA==", "engines": { "node": ">=18" } }, "node_modules/@inquirer/type": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-1.5.0.tgz", - "integrity": "sha512-L/UdayX9Z1lLN+itoTKqJ/X4DX5DaWu2Sruwt4XgZzMNv32x4qllbzMX4MbJlz0yxAQtU19UvABGOjmdq1u3qA==", + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-1.5.1.tgz", + "integrity": "sha512-m3YgGQlKNS0BM+8AFiJkCsTqHEFCWn6s/Rqye3mYwvqY6LdfUv12eSwbsgNzrYyrLXiy7IrrjDLPysaSBwEfhw==", "dependencies": { "mute-stream": "^1.0.0" }, @@ -4052,15 +4120,6 @@ "node": ">=18.0.0" } }, - "node_modules/@isaacs/fs-minipass/node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "optional": true, - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.5", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz", @@ -4449,15 +4508,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/@next/eslint-plugin-next/node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "dev": true, - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, "node_modules/@next/swc-darwin-arm64": { "version": "14.2.5", "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.5.tgz", @@ -4698,6 +4748,148 @@ "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, + "node_modules/@octokit/auth-token": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-2.5.0.tgz", + "integrity": "sha512-r5FVUJCOLl19AxiuZD2VRZ/ORjp/4IN98Of6YJoJOkY75CIBuYfmiNHGrDwXr+aLGG55igl9QrxX3hbiXlLb+g==", + "dev": true, + "dependencies": { + "@octokit/types": "^6.0.3" + } + }, + "node_modules/@octokit/core": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-3.6.0.tgz", + "integrity": "sha512-7RKRKuA4xTjMhY+eG3jthb3hlZCsOwg3rztWh75Xc+ShDWOfDDATWbeZpAHBNRpm4Tv9WgBMOy1zEJYXG6NJ7Q==", + "dev": true, + "dependencies": { + "@octokit/auth-token": "^2.4.4", + "@octokit/graphql": "^4.5.8", + "@octokit/request": "^5.6.3", + "@octokit/request-error": "^2.0.5", + "@octokit/types": "^6.0.3", + "before-after-hook": "^2.2.0", + "universal-user-agent": "^6.0.0" + } + }, + "node_modules/@octokit/endpoint": { + "version": "6.0.12", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-6.0.12.tgz", + "integrity": "sha512-lF3puPwkQWGfkMClXb4k/eUT/nZKQfxinRWJrdZaJO85Dqwo/G0yOC434Jr2ojwafWJMYqFGFa5ms4jJUgujdA==", + "dev": true, + "dependencies": { + "@octokit/types": "^6.0.3", + "is-plain-object": "^5.0.0", + "universal-user-agent": "^6.0.0" + } + }, + "node_modules/@octokit/graphql": { + "version": "4.8.0", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-4.8.0.tgz", + "integrity": "sha512-0gv+qLSBLKF0z8TKaSKTsS39scVKF9dbMxJpj3U0vC7wjNWFuIpL/z76Qe2fiuCbDRcJSavkXsVtMS6/dtQQsg==", + "dev": true, + "dependencies": { + "@octokit/request": "^5.6.0", + "@octokit/types": "^6.0.3", + "universal-user-agent": "^6.0.0" + } + }, + "node_modules/@octokit/openapi-types": { + "version": "12.11.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-12.11.0.tgz", + "integrity": "sha512-VsXyi8peyRq9PqIz/tpqiL2w3w80OgVMwBHltTml3LmVvXiphgeqmY9mvBw9Wu7e0QWk/fqD37ux8yP5uVekyQ==", + "dev": true + }, + "node_modules/@octokit/plugin-paginate-rest": { + "version": "2.21.3", + "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-2.21.3.tgz", + "integrity": "sha512-aCZTEf0y2h3OLbrgKkrfFdjRL6eSOo8komneVQJnYecAxIej7Bafor2xhuDJOIFau4pk0i/P28/XgtbyPF0ZHw==", + "dev": true, + "dependencies": { + "@octokit/types": "^6.40.0" + }, + "peerDependencies": { + "@octokit/core": ">=2" + } + }, + "node_modules/@octokit/plugin-request-log": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@octokit/plugin-request-log/-/plugin-request-log-1.0.4.tgz", + "integrity": "sha512-mLUsMkgP7K/cnFEw07kWqXGF5LKrOkD+lhCrKvPHXWDywAwuDUeDwWBpc69XK3pNX0uKiVt8g5z96PJ6z9xCFA==", + "dev": true, + "peerDependencies": { + "@octokit/core": ">=3" + } + }, + "node_modules/@octokit/plugin-rest-endpoint-methods": { + "version": "5.16.2", + "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-5.16.2.tgz", + "integrity": "sha512-8QFz29Fg5jDuTPXVtey05BLm7OB+M8fnvE64RNegzX7U+5NUXcOcnpTIK0YfSHBg8gYd0oxIq3IZTe9SfPZiRw==", + "dev": true, + "dependencies": { + "@octokit/types": "^6.39.0", + "deprecation": "^2.3.1" + }, + "peerDependencies": { + "@octokit/core": ">=3" + } + }, + "node_modules/@octokit/plugin-retry": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-3.0.9.tgz", + "integrity": "sha512-r+fArdP5+TG6l1Rv/C9hVoty6tldw6cE2pRHNGmFPdyfrc696R6JjrQ3d7HdVqGwuzfyrcaLAKD7K8TX8aehUQ==", + "dev": true, + "dependencies": { + "@octokit/types": "^6.0.3", + "bottleneck": "^2.15.3" + } + }, + "node_modules/@octokit/request": { + "version": "5.6.3", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-5.6.3.tgz", + "integrity": "sha512-bFJl0I1KVc9jYTe9tdGGpAMPy32dLBXXo1dS/YwSCTL/2nd9XeHsY616RE3HPXDVk+a+dBuzyz5YdlXwcDTr2A==", + "dev": true, + "dependencies": { + "@octokit/endpoint": "^6.0.1", + "@octokit/request-error": "^2.1.0", + "@octokit/types": "^6.16.1", + "is-plain-object": "^5.0.0", + "node-fetch": "^2.6.7", + "universal-user-agent": "^6.0.0" + } + }, + "node_modules/@octokit/request-error": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-2.1.0.tgz", + "integrity": "sha512-1VIvgXxs9WHSjicsRwq8PlR2LR2x6DwsJAaFgzdi0JfJoGSO8mYI/cHJQ+9FbN21aa+DrgNLnwObmyeSC8Rmpg==", + "dev": true, + "dependencies": { + "@octokit/types": "^6.0.3", + "deprecation": "^2.0.0", + "once": "^1.4.0" + } + }, + "node_modules/@octokit/rest": { + "version": "18.12.0", + "resolved": "https://registry.npmjs.org/@octokit/rest/-/rest-18.12.0.tgz", + "integrity": "sha512-gDPiOHlyGavxr72y0guQEhLsemgVjwRePayJ+FcKc2SJqKUbxbkvf5kAZEWA/MKvsfYlQAMVzNJE3ezQcxMJ2Q==", + "dev": true, + "dependencies": { + "@octokit/core": "^3.5.1", + "@octokit/plugin-paginate-rest": "^2.16.8", + "@octokit/plugin-request-log": "^1.0.4", + "@octokit/plugin-rest-endpoint-methods": "^5.12.0" + } + }, + "node_modules/@octokit/types": { + "version": "6.41.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-6.41.0.tgz", + "integrity": "sha512-eJ2jbzjdijiL3B4PrSQaSjuF2sPEQPVCPzBvTHJD9Nz+9dw2SGH4K4xeQJ77YfTq5bRQ+bD8wT11JbeDPmxmGg==", + "dev": true, + "dependencies": { + "@octokit/openapi-types": "^12.11.0" + } + }, "node_modules/@open-draft/deferred-promise": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/@open-draft/deferred-promise/-/deferred-promise-2.2.0.tgz", @@ -5882,25 +6074,25 @@ "integrity": "sha512-/RVXdLvJxLg4QKvMoM5WlwNR9ViO9z8B/qPcc+C0Sa/teJY7QG7kJ441DwzOjMYEY7GmU4dj5EcGHIkKZiQZCA==" }, "node_modules/@react-spring/animated": { - "version": "9.7.3", - "resolved": "https://registry.npmjs.org/@react-spring/animated/-/animated-9.7.3.tgz", - "integrity": "sha512-5CWeNJt9pNgyvuSzQH+uy2pvTg8Y4/OisoscZIR8/ZNLIOI+CatFBhGZpDGTF/OzdNFsAoGk3wiUYTwoJ0YIvw==", + "version": "9.7.4", + "resolved": "https://registry.npmjs.org/@react-spring/animated/-/animated-9.7.4.tgz", + "integrity": "sha512-7As+8Pty2QlemJ9O5ecsuPKjmO0NKvmVkRR1n6mEotFgWar8FKuQt2xgxz3RTgxcccghpx1YdS1FCdElQNexmQ==", "dependencies": { - "@react-spring/shared": "~9.7.3", - "@react-spring/types": "~9.7.3" + "@react-spring/shared": "~9.7.4", + "@react-spring/types": "~9.7.4" }, "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0" } }, "node_modules/@react-spring/core": { - "version": "9.7.3", - "resolved": "https://registry.npmjs.org/@react-spring/core/-/core-9.7.3.tgz", - "integrity": "sha512-IqFdPVf3ZOC1Cx7+M0cXf4odNLxDC+n7IN3MDcVCTIOSBfqEcBebSv+vlY5AhM0zw05PDbjKrNmBpzv/AqpjnQ==", + "version": "9.7.4", + "resolved": "https://registry.npmjs.org/@react-spring/core/-/core-9.7.4.tgz", + "integrity": "sha512-GzjA44niEJBFUe9jN3zubRDDDP2E4tBlhNlSIkTChiNf9p4ZQlgXBg50qbXfSXHQPHak/ExYxwhipKVsQ/sUTw==", "dependencies": { - "@react-spring/animated": "~9.7.3", - "@react-spring/shared": "~9.7.3", - "@react-spring/types": "~9.7.3" + "@react-spring/animated": "~9.7.4", + "@react-spring/shared": "~9.7.4", + "@react-spring/types": "~9.7.4" }, "funding": { "type": "opencollective", @@ -5911,30 +6103,31 @@ } }, "node_modules/@react-spring/rafz": { - "version": "9.6.1", - "resolved": "https://registry.npmjs.org/@react-spring/rafz/-/rafz-9.6.1.tgz", - "integrity": "sha512-v6qbgNRpztJFFfSE3e2W1Uz+g8KnIBs6SmzCzcVVF61GdGfGOuBrbjIcp+nUz301awVmREKi4eMQb2Ab2gGgyQ==" + "version": "9.7.4", + "resolved": "https://registry.npmjs.org/@react-spring/rafz/-/rafz-9.7.4.tgz", + "integrity": "sha512-mqDI6rW0Ca8IdryOMiXRhMtVGiEGLIO89vIOyFQXRIwwIMX30HLya24g9z4olDvFyeDW3+kibiKwtZnA4xhldA==" }, "node_modules/@react-spring/shared": { - "version": "9.7.3", - "resolved": "https://registry.npmjs.org/@react-spring/shared/-/shared-9.7.3.tgz", - "integrity": "sha512-NEopD+9S5xYyQ0pGtioacLhL2luflh6HACSSDUZOwLHoxA5eku1UPuqcJqjwSD6luKjjLfiLOspxo43FUHKKSA==", + "version": "9.7.4", + "resolved": "https://registry.npmjs.org/@react-spring/shared/-/shared-9.7.4.tgz", + "integrity": "sha512-bEPI7cQp94dOtCFSEYpxvLxj0+xQfB5r9Ru1h8OMycsIq7zFZon1G0sHrBLaLQIWeMCllc4tVDYRTLIRv70C8w==", "dependencies": { - "@react-spring/types": "~9.7.3" + "@react-spring/rafz": "~9.7.4", + "@react-spring/types": "~9.7.4" }, "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0" } }, "node_modules/@react-spring/three": { - "version": "9.7.3", - "resolved": "https://registry.npmjs.org/@react-spring/three/-/three-9.7.3.tgz", - "integrity": "sha512-Q1p512CqUlmMK8UMBF/Rj79qndhOWq4XUTayxMP9S892jiXzWQuj+xC3Xvm59DP/D4JXusXpxxqfgoH+hmOktA==", + "version": "9.7.4", + "resolved": "https://registry.npmjs.org/@react-spring/three/-/three-9.7.4.tgz", + "integrity": "sha512-HKUhrrvWW7F/MAroObOloqcYyFqsUHp1ANIDvPVxk9cSh7veW7gQbJm2Sc7Ka+L4gVJEwSkS+MRfr8kk+sRZBw==", "dependencies": { - "@react-spring/animated": "~9.7.3", - "@react-spring/core": "~9.7.3", - "@react-spring/shared": "~9.7.3", - "@react-spring/types": "~9.7.3" + "@react-spring/animated": "~9.7.4", + "@react-spring/core": "~9.7.4", + "@react-spring/shared": "~9.7.4", + "@react-spring/types": "~9.7.4" }, "peerDependencies": { "@react-three/fiber": ">=6.0", @@ -5943,9 +6136,9 @@ } }, "node_modules/@react-spring/types": { - "version": "9.7.3", - "resolved": "https://registry.npmjs.org/@react-spring/types/-/types-9.7.3.tgz", - "integrity": "sha512-Kpx/fQ/ZFX31OtlqVEFfgaD1ACzul4NksrvIgYfIFq9JpDHFwQkMVZ10tbo0FU/grje4rcL4EIrjekl3kYwgWw==" + "version": "9.7.4", + "resolved": "https://registry.npmjs.org/@react-spring/types/-/types-9.7.4.tgz", + "integrity": "sha512-iQVztO09ZVfsletMiY+DpT/JRiBntdsdJ4uqk3UJFhrhS8mIC9ZOZbmfGSRs/kdbNPQkVyzucceDicQ/3Mlj9g==" }, "node_modules/@react-three/drei": { "version": "9.108.4", @@ -6018,6 +6211,11 @@ "react": "^16.8.0 || ^17.0.0 || ^18.0.0" } }, + "node_modules/@react-three/drei/node_modules/@react-spring/rafz": { + "version": "9.6.1", + "resolved": "https://registry.npmjs.org/@react-spring/rafz/-/rafz-9.6.1.tgz", + "integrity": "sha512-v6qbgNRpztJFFfSE3e2W1Uz+g8KnIBs6SmzCzcVVF61GdGfGOuBrbjIcp+nUz301awVmREKi4eMQb2Ab2gGgyQ==" + }, "node_modules/@react-three/drei/node_modules/@react-spring/shared": { "version": "9.6.1", "resolved": "https://registry.npmjs.org/@react-spring/shared/-/shared-9.6.1.tgz", @@ -8375,6 +8573,12 @@ } ] }, + "node_modules/before-after-hook": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.3.tgz", + "integrity": "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==", + "dev": true + }, "node_modules/bidi-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/bidi-js/-/bidi-js-1.0.3.tgz", @@ -8474,6 +8678,12 @@ "dev": true, "optional": true }, + "node_modules/bottleneck": { + "version": "2.19.5", + "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz", + "integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==", + "dev": true + }, "node_modules/bowser": { "version": "2.11.0", "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz", @@ -8648,18 +8858,6 @@ "balanced-match": "^1.0.0" } }, - "node_modules/cacache/node_modules/fs-minipass": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-3.0.3.tgz", - "integrity": "sha512-XUBA9XClHbnJWSfBzjkm6RvPsyg3sryZt06BEQoXcF7EK/xpGaQYJgQKDJSUH5SGZ76Y7pFx1QBnXz09rU5Fbw==", - "dev": true, - "dependencies": { - "minipass": "^7.0.3" - }, - "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" - } - }, "node_modules/cacache/node_modules/glob": { "version": "10.4.5", "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", @@ -8716,15 +8914,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/cacache/node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "dev": true, - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, "node_modules/cacheable-lookup": { "version": "5.0.4", "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz", @@ -8830,9 +9019,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001642", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001642.tgz", - "integrity": "sha512-3XQ0DoRgLijXJErLSl+bLnJ+Et4KqV1PY6JJBGAFlsNsz31zeAIncyeZfLCabHK/jtSh+671RM9YMldxjUPZtA==", + "version": "1.0.30001643", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001643.tgz", + "integrity": "sha512-ERgWGNleEilSrHM6iUz/zJNSQTP8Mr21wDWpdgvRwcTXGAq6jMtOUPP4dqFPTdKqZ2wKTdtB+uucZ3MRpAUSmg==", "funding": [ { "type": "opencollective", @@ -9996,6 +10185,12 @@ "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==", "dev": true }, + "node_modules/deprecation": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz", + "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==", + "dev": true + }, "node_modules/dequal": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", @@ -12179,15 +12374,15 @@ } }, "node_modules/fs-minipass": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", - "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-3.0.3.tgz", + "integrity": "sha512-XUBA9XClHbnJWSfBzjkm6RvPsyg3sryZt06BEQoXcF7EK/xpGaQYJgQKDJSUH5SGZ76Y7pFx1QBnXz09rU5Fbw==", "dev": true, "dependencies": { - "minipass": "^3.0.0" + "minipass": "^7.0.3" }, "engines": { - "node": ">= 8" + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, "node_modules/fs-temp": { @@ -13585,6 +13780,15 @@ "node": ">=8" } }, + "node_modules/is-plain-object": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz", + "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-potential-custom-element-name": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz", @@ -14431,15 +14635,6 @@ "node": "^16.14.0 || >=18.0.0" } }, - "node_modules/make-fetch-happen/node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "dev": true, - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, "node_modules/map-age-cleaner": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/map-age-cleaner/-/map-age-cleaner-0.1.3.tgz", @@ -14600,15 +14795,11 @@ } }, "node_modules/minipass": { - "version": "3.3.6", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", - "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", - "dev": true, - "dependencies": { - "yallist": "^4.0.0" - }, + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", "engines": { - "node": ">=8" + "node": ">=16 || 14 >=14.17" } }, "node_modules/minipass-collect": { @@ -14623,15 +14814,6 @@ "node": ">=16 || 14 >=14.17" } }, - "node_modules/minipass-collect/node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "dev": true, - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, "node_modules/minipass-fetch": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/minipass-fetch/-/minipass-fetch-3.0.5.tgz", @@ -14649,15 +14831,6 @@ "encoding": "^0.1.13" } }, - "node_modules/minipass-fetch/node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "dev": true, - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, "node_modules/minipass-flush": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/minipass-flush/-/minipass-flush-1.0.5.tgz", @@ -14670,6 +14843,24 @@ "node": ">= 8" } }, + "node_modules/minipass-flush/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-flush/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + }, "node_modules/minipass-pipeline": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/minipass-pipeline/-/minipass-pipeline-1.2.4.tgz", @@ -14682,6 +14873,24 @@ "node": ">=8" } }, + "node_modules/minipass-pipeline/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-pipeline/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + }, "node_modules/minipass-sized": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/minipass-sized/-/minipass-sized-1.0.3.tgz", @@ -14694,7 +14903,19 @@ "node": ">=8" } }, - "node_modules/minipass/node_modules/yallist": { + "node_modules/minipass-sized/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-sized/node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", @@ -14713,6 +14934,18 @@ "node": ">= 8" } }, + "node_modules/minizlib/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/minizlib/node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", @@ -15194,15 +15427,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/node-gyp/node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "dev": true, - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, "node_modules/node-gyp/node_modules/which": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/which/-/which-4.0.0.tgz", @@ -15606,15 +15830,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/onnxruntime-node/node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "optional": true, - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, "node_modules/onnxruntime-node/node_modules/minizlib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.1.tgz", @@ -16051,14 +16266,6 @@ "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==" }, - "node_modules/path-scurry/node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, "node_modules/path-to-regexp": { "version": "6.2.2", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.2.2.tgz", @@ -18060,15 +18267,6 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, - "node_modules/ssri/node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "dev": true, - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, "node_modules/stackback": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", @@ -18583,14 +18781,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/sucrase/node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, "node_modules/sudo-prompt": { "version": "9.2.1", "resolved": "https://registry.npmjs.org/sudo-prompt/-/sudo-prompt-9.2.1.tgz", @@ -18736,6 +18926,30 @@ "node": ">=10" } }, + "node_modules/tar/node_modules/fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "dev": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/tar/node_modules/fs-minipass/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/tar/node_modules/minipass": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", @@ -19370,6 +19584,12 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, + "node_modules/universal-user-agent": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.1.tgz", + "integrity": "sha512-yCzhz6FN2wU1NiiQRogkTQszlQSlpWaw8SvVegAc+bDxbzHgh1vX8uIe8OYyMH6DwH+sdTJsgMl36+mSMdRJIQ==", + "dev": true + }, "node_modules/universalify": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", diff --git a/package.json b/package.json index 2e35ae18..3db6581f 100644 --- a/package.json +++ b/package.json @@ -5,11 +5,19 @@ "description": "🎬 Clapper", "license": "GPL-3.0-only", "main": "main.js", + "files": [ + "./main.js" + ], + "directories": { + "src:": "./src", + "public:": "./public" + }, "scripts": { "dev": "npm i && npm run checks && next dev", - "build": "npm i && npm run checks && next build && npm run build:copyassets", + "build": "npm i && npm run prepare && npm run checks && rm -Rf out && next build && npm run build:copyassets", + "build:ci": "rm -Rf out && npm run prepare && next build && npm run build:copyassets", "build:copyassets": "cp -R public .next/standalone/public && cp -R .next/static .next/standalone/.next/static", - "build:ci": "next build && npm run build:copyassets", + "prepare": "cp -R node_modules/mediainfo.js/dist/MediaInfoModule.wasm public/wasm/", "start": "next start", "start:prod": "node .next/standalone/server.js", "checks": "npm run format:fix && npm run lint", @@ -23,15 +31,15 @@ "test:e2e": "npx playwright test", "electron": "npm run build && electron .", "electron:start": "npm run build && electron-forge start", - "electron:package": "electron-forge package", - "electron:make": "electron-forge make" + "electron:package": "npm run build && electron-forge package", + "electron:make": "npm run build && electron-forge make" }, "dependencies": { "@aitube/broadway": "0.0.22", "@aitube/clap": "0.0.30", - "@aitube/clapper-services": "0.0.28", + "@aitube/clapper-services": "0.0.29", "@aitube/engine": "0.0.26", - "@aitube/timeline": "0.0.42", + "@aitube/timeline": "0.0.43", "@fal-ai/serverless-client": "^0.13.0", "@ffmpeg/ffmpeg": "^0.12.10", "@ffmpeg/util": "^0.12.1", @@ -128,6 +136,7 @@ "@electron-forge/maker-squirrel": "^7.4.0", "@electron-forge/maker-zip": "^7.4.0", "@electron-forge/plugin-auto-unpack-natives": "^7.4.0", + "@electron-forge/publisher-github": "^7.4.0", "@playwright/test": "^1.45.1", "@testing-library/react": "^16.0.0", "@types/fluent-ffmpeg": "^2.1.24", diff --git a/src/app/CL.icns b/public/images/logos/CL.icns similarity index 100% rename from src/app/CL.icns rename to public/images/logos/CL.icns diff --git a/src/app/CL.iconset/icon_128x128.png b/public/images/logos/CL.iconset/icon_128x128.png similarity index 100% rename from src/app/CL.iconset/icon_128x128.png rename to public/images/logos/CL.iconset/icon_128x128.png diff --git a/src/app/CL.iconset/icon_128x128@2x.png b/public/images/logos/CL.iconset/icon_128x128@2x.png similarity index 100% rename from src/app/CL.iconset/icon_128x128@2x.png rename to public/images/logos/CL.iconset/icon_128x128@2x.png diff --git a/src/app/CL.iconset/icon_16x16.png b/public/images/logos/CL.iconset/icon_16x16.png similarity index 100% rename from src/app/CL.iconset/icon_16x16.png rename to public/images/logos/CL.iconset/icon_16x16.png diff --git a/src/app/CL.iconset/icon_16x16@2x.png b/public/images/logos/CL.iconset/icon_16x16@2x.png similarity index 100% rename from src/app/CL.iconset/icon_16x16@2x.png rename to public/images/logos/CL.iconset/icon_16x16@2x.png diff --git a/src/app/CL.iconset/icon_256x256.png b/public/images/logos/CL.iconset/icon_256x256.png similarity index 100% rename from src/app/CL.iconset/icon_256x256.png rename to public/images/logos/CL.iconset/icon_256x256.png diff --git a/src/app/CL.iconset/icon_256x256@2x.png b/public/images/logos/CL.iconset/icon_256x256@2x.png similarity index 100% rename from src/app/CL.iconset/icon_256x256@2x.png rename to public/images/logos/CL.iconset/icon_256x256@2x.png diff --git a/src/app/CL.iconset/icon_32x32.png b/public/images/logos/CL.iconset/icon_32x32.png similarity index 100% rename from src/app/CL.iconset/icon_32x32.png rename to public/images/logos/CL.iconset/icon_32x32.png diff --git a/src/app/CL.iconset/icon_32x32@2x.png b/public/images/logos/CL.iconset/icon_32x32@2x.png similarity index 100% rename from src/app/CL.iconset/icon_32x32@2x.png rename to public/images/logos/CL.iconset/icon_32x32@2x.png diff --git a/src/app/CL.iconset/icon_512x512.png b/public/images/logos/CL.iconset/icon_512x512.png similarity index 100% rename from src/app/CL.iconset/icon_512x512.png rename to public/images/logos/CL.iconset/icon_512x512.png diff --git a/src/app/CL.iconset/icon_512x512@2x.png b/public/images/logos/CL.iconset/icon_512x512@2x.png similarity index 100% rename from src/app/CL.iconset/icon_512x512@2x.png rename to public/images/logos/CL.iconset/icon_512x512@2x.png diff --git a/public/images/logos/CL.png b/public/images/logos/CL.png new file mode 100644 index 00000000..0f20e455 --- /dev/null +++ b/public/images/logos/CL.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ad55d8b1b92c5d31c1dc66452af1907790dfa5d9d778386f3f12842434e7f6f +size 60404 diff --git a/public/wasm/MediaInfoModule.wasm b/public/wasm/MediaInfoModule.wasm new file mode 100644 index 00000000..45462bb2 --- /dev/null +++ b/public/wasm/MediaInfoModule.wasm @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0a209b7ad5152420f6c739f5bf10e9c02df79798f8f75fbe413c6d1be422bbf +size 2355621 diff --git a/src/lib/core/constants.ts b/src/lib/core/constants.ts index 08169274..8942edfc 100644 --- a/src/lib/core/constants.ts +++ b/src/lib/core/constants.ts @@ -3,7 +3,7 @@ export const HARD_LIMIT_NB_MAX_ASSETS_TO_GENERATE_IN_PARALLEL = 32 export const APP_NAME = 'Clapper.app' -export const APP_REVISION = 'r20240721-0835' +export const APP_REVISION = 'r20240722-0205' export const APP_DOMAIN = 'Clapper.app' export const APP_LINK = 'https://clapper.app' diff --git a/src/lib/utils/base64DataUriToFile.ts b/src/lib/utils/base64DataUriToFile.ts new file mode 100644 index 00000000..184b9aca --- /dev/null +++ b/src/lib/utils/base64DataUriToFile.ts @@ -0,0 +1,12 @@ +export function base64DataUriToFile(dataUrl: string, fileName: string) { + var arr = dataUrl.split(',') + const st = `${arr[0] || ''}` + const mime = `${st.match(/:(.*?);/)?.[1] || ''}` + const bstr = atob(arr[arr.length - 1]) + let n = bstr.length + const u8arr = new Uint8Array(n) + while (n--) { + u8arr[n] = bstr.charCodeAt(n) + } + return new File([u8arr], fileName, { type: mime }) +} diff --git a/src/services/io/createFullVideo.ts b/src/services/io/createFullVideo.ts index b6afdf93..01647f71 100644 --- a/src/services/io/createFullVideo.ts +++ b/src/services/io/createFullVideo.ts @@ -1,481 +1,15 @@ import { UUID } from '@aitube/clap' -import { FFmpeg } from '@ffmpeg/ffmpeg' -import { toBlobURL } from '@ffmpeg/util' - -const TAG = 'io/createFullVideo' - -export type FFMPegVideoInput = { - data: Uint8Array | null - startTimeInMs: number - endTimeInMs: number - durationInSecs: number -} - -export type FFMPegAudioInput = FFMPegVideoInput - -/** - * Download and load single and multi-threading FFMPeg. - * MT for video - * ST for audio (as MT has issues with it) - * toBlobURL is used to bypass CORS issues, urls with the same domain can be used directly. - */ -async function initializeFFmpeg() { - const [ffmpegSt, ffmpegMt] = [new FFmpeg(), new FFmpeg()] - const baseStURL = 'https://unpkg.com/@ffmpeg/core@0.12.6/dist/umd' - const baseMtURL = 'https://unpkg.com/@ffmpeg/core-mt@0.12.6/dist/umd' - - ffmpegSt.on('log', ({ message }) => { - console.log(TAG, 'FFmpeg Single-Thread:', message) - }) - - ffmpegMt.on('log', ({ message }) => { - console.log(TAG, 'FFmpeg Multi-Thread:', message) - }) - - await ffmpegSt.load({ - coreURL: await toBlobURL(`${baseStURL}/ffmpeg-core.js`, 'text/javascript'), - wasmURL: await toBlobURL( - `${baseStURL}/ffmpeg-core.wasm`, - 'application/wasm' - ), - }) - - await ffmpegMt.load({ - coreURL: await toBlobURL(`${baseMtURL}/ffmpeg-core.js`, 'text/javascript'), - wasmURL: await toBlobURL( - `${baseMtURL}/ffmpeg-core.wasm`, - 'application/wasm' - ), - workerURL: await toBlobURL( - `${baseMtURL}/ffmpeg-core.worker.js`, - 'text/javascript' - ), - }) - - return [ffmpegSt, ffmpegMt] as [FFmpeg, FFmpeg] -} - -/** - * Get loaded FFmpeg. - */ -let ffmpegInstance: [FFmpeg, FFmpeg] -export async function loadFFmpegSt() { - if (!ffmpegInstance) ffmpegInstance = await initializeFFmpeg() - return ffmpegInstance[0] -} - -export async function loadFFmpegMt() { - if (!ffmpegInstance) ffmpegInstance = await initializeFFmpeg() - return ffmpegInstance[1] -} - -/** - * Creates an exclusive logger for the FFmpeg calls inside the provided method, - * it calculates the progress based on raw FFmpeg logs and the provided `totalTimeInMs`. - * - * @param totalTimeInMs - * @param method - * @param callback - * @param {number} callback.progress - The progress of the FFmpeg process from 0 to 100. - * @returns - */ -async function captureFFmpegProgress( - ffmpeg: FFmpeg, - totalTimeInMs: number, - method: () => any, - callback: (progress: number) => void -): Promise { - const extractProgressTimeMsFromLogs = (log: string): number | null => { - // `frame` for videos, `size` for audios - if (!log.startsWith('frame') && !log.startsWith('size')) return null - const timeRegex = /time=(\d{2}):(\d{2}):(\d{2})\.(\d{2})/ - const match = log.match(timeRegex) - if (match) { - const hours = parseInt(match[1]) - const minutes = parseInt(match[2]) - const seconds = parseInt(match[3]) - const centiseconds = parseInt(match[4]) - const totalMilliseconds = - hours * 3600000 + minutes * 60000 + seconds * 1000 + centiseconds * 10 - return totalMilliseconds - } - return null - } - let ffmpegLog = true - ffmpeg.on('log', ({ message }) => { - if (!ffmpegLog) return - const timeInMs = extractProgressTimeMsFromLogs(message) - if (timeInMs) callback((timeInMs / totalTimeInMs) * 100) - }) - const result = await method() - ffmpegLog = false - return result -} - -/** - * It will calculate a proportional progress between a targetProgress and a startProgress - * - * @param startProgress e.g. 50 - * @param progress e.g. 50 - * @param targetProgress e.g. 70 - * @returns e.g. 60, because 50% of progress between 70% and 50%, would result on 60% - */ -function calculateProgress( - startProgress: number, - progress: number, - targetProgress: number -): number { - return startProgress + (progress * (targetProgress - startProgress)) / 100 -} - -/** - * Creates an empty black video and appends it to the - * provided `fileListContentArray`. - * - * @param duration time in milliseconds - * @param width - * @param height - * @param filename - * @param fileListContentArray fileList.txt where to append the file name - * @param onProgress callback to capture the progress of this method - */ -export async function addEmptyVideo( - durationInSecs: number, - width: number, - height: number, - filename: string, - fileListContentArray: string[], - onProgress?: (progress: number, message?: string) => void -) { - const ffmpeg = await loadFFmpegMt() - let targetPartialProgress = 0 - - // For some reason, creating empty video with silent audio - // in one exec doesn't work, we need to split it. - - console.log( - TAG, - 'Creating empty video', - filename, - width, - height, - durationInSecs - ) - let currentProgress = 0 - targetPartialProgress = 50 - - await captureFFmpegProgress( - ffmpeg, - durationInSecs * 1000, - async () => { - await ffmpeg.exec([ - '-f', - 'lavfi', - '-i', - `color=c=black:s=${width}x${height}:d=${durationInSecs}`, - '-c:v', - 'libx264', - '-t', - `${durationInSecs}`, - '-loglevel', - 'verbose', - `base_${filename}`, - ]) - }, - (progress) => { - onProgress?.((progress / 100) * targetPartialProgress) - } - ) - - console.log( - TAG, - 'Adding silent audio to empty video', - filename, - width, - height, - durationInSecs - ) - currentProgress = 50 - targetPartialProgress = 100 - - const exitCode = await ffmpeg.exec([ - '-i', - `base_${filename}`, - '-f', - 'lavfi', - '-i', - 'anullsrc', - '-c:v', - 'copy', - '-c:a', - 'aac', - '-t', - `${durationInSecs}`, - '-loglevel', - 'verbose', - filename, - ]) - - if (exitCode) { - throw new Error(`${TAG}: Unexpect error while creating empty video`) - } - - console.log(TAG, 'Empty video created', filename) - fileListContentArray.push(`file ${filename}`) -} - -/** - * Creates the full mixed audio including silence - * segments and loads it into ffmpeg with the given `filename`. - * @param onProgress callback to capture the progress of this method - * @throws Error if ffmpeg returns exit code 1 - */ -export async function createFullAudio( - audios: FFMPegAudioInput[], - filename: string, - totalVideoDurationInMs: number, - onProgress?: (progress: number, message: string) => void -): Promise { - console.log(TAG, 'Creating full audio', filename) - - const ffmpeg = await loadFFmpegSt() - const filterComplexParts = [] - const baseFilename = `base_${filename}` - let currentProgress = 0 - let targetProgress = 25 - - // To mix audios at given times, we need a first empty base audio track - - await captureFFmpegProgress( - ffmpeg, - totalVideoDurationInMs, - async () => { - await ffmpeg.exec([ - '-f', - 'lavfi', - '-i', - 'anullsrc', - '-t', - `${totalVideoDurationInMs / 1000}`, - '-loglevel', - 'verbose', - !audios.length ? filename : baseFilename, - ]) - }, - (progress) => { - onProgress?.( - calculateProgress(currentProgress, progress, targetProgress), - 'Creating base audio...' - ) - } - ) - - // If there is no audios, the base audio is the final one - if (!audios.length) return onProgress?.(100, 'Prepared audios...') - - currentProgress = targetProgress - targetProgress = 50 - - // Mix audios based on their start times - - const audioInputFiles = ['-i', baseFilename] - for (let index = 0; index < audios.length; index++) { - onProgress?.(currentProgress, 'Creating base audio...') - console.log(TAG, `Processing audio #${index}`) - const audio = audios[index] - const expectedProgressForItem = ((1 / audios.length) * targetProgress) / 100 - if (!audio.data) continue - const audioFilename = `audio_${UUID()}.mp3` - await ffmpeg.writeFile(audioFilename, audio.data) - audioInputFiles.push('-i', audioFilename) - const delay = audio.startTimeInMs - const durationInSecs = audio.endTimeInMs - audio.startTimeInMs / 1000 - filterComplexParts.push( - `[${index + 1}:a]atrim=0:${durationInSecs},adelay=${delay}|${delay}[delayed${index}]` - ) - currentProgress += expectedProgressForItem * 100 - } - - const amixInputs = `[0:a]${audios.map((_, index) => `[delayed${index}]`).join('')}amix=inputs=${audios.length + 1}:duration=longest` - filterComplexParts.push(`${amixInputs}[a]`) - const filterComplex = filterComplexParts.join('; ') - - currentProgress = targetProgress - targetProgress = 100 - - const createFullAudioExitCode = await captureFFmpegProgress( - ffmpeg, - totalVideoDurationInMs, - async () => { - await ffmpeg.exec([ - ...audioInputFiles, - '-filter_complex', - filterComplex, - '-map', - '[a]', - '-t', - `${totalVideoDurationInMs / 1000}`, - '-loglevel', - 'verbose', - filename, - ]) - }, - (progress) => { - onProgress?.( - calculateProgress(currentProgress, progress, targetProgress), - 'Mixing audios...' - ) - } - ) - - if (createFullAudioExitCode) { - throw new Error(`${TAG}: Error while creating full audio!`) - } - onProgress?.(targetProgress, 'Prepared audios...') -} - -/** - * Creates the full silent video including empty black - * segments and loads it into ffmpeg with the given `filename`. - * @param onProgress callback to capture the progress of this method - * @throws Error if ffmpeg returns exit code 1 - */ -export async function createFullSilentVideo( - videos: FFMPegVideoInput[], - filename: string, - totalVideoDurationInMs: number, - width: number, - height: number, - excludeEmptyContent = false, - onProgress?: (progress: number, message: string) => void -) { - const ffmpeg = await loadFFmpegMt() - const fileList = 'fileList.txt' - const fileListContentArray = [] - - // Complete array of videos including concatenated empty segments - // This is helpful for cleaner progress log - let lastStartTimeVideoInMs = 0 - let videosWithGaps: FFMPegVideoInput[] - - if (!videos.length) { - videosWithGaps = [ - { - startTimeInMs: 0, - endTimeInMs: totalVideoDurationInMs, - data: null, - durationInSecs: totalVideoDurationInMs / 1000, - }, - ] - } else { - videosWithGaps = videos.reduce((arr: FFMPegVideoInput[], video, index) => { - const emptyVideoDurationInMs = - video.startTimeInMs - lastStartTimeVideoInMs - if (emptyVideoDurationInMs) { - arr.push({ - startTimeInMs: lastStartTimeVideoInMs, - endTimeInMs: lastStartTimeVideoInMs + emptyVideoDurationInMs, - data: null, - durationInSecs: emptyVideoDurationInMs / 1000, - }) - } - arr.push(video) - lastStartTimeVideoInMs = video.endTimeInMs - if ( - index == videos.length - 1 && - lastStartTimeVideoInMs < totalVideoDurationInMs - ) { - arr.push({ - startTimeInMs: lastStartTimeVideoInMs, - endTimeInMs: totalVideoDurationInMs, - data: null, - durationInSecs: - (totalVideoDurationInMs - lastStartTimeVideoInMs) / 1000, - }) - } - return arr - }, []) - } - - onProgress?.(0, 'Preparing videos...') - - // Arbitrary percentage, as `concat` is fast, - // then estimate the generation of gap videos - // as the 70% of the work - let currentProgress = 0 - let targetProgress = 70 - - for (const video of videosWithGaps) { - const expectedProgressForItem = - (((video.durationInSecs * 1000) / totalVideoDurationInMs) * - targetProgress) / - 100 - if (!video.data) { - if (excludeEmptyContent) continue - let collectedProgress = 0 - await addEmptyVideo( - video.durationInSecs, - width, - height, - `empty_video_${UUID()}.mp4`, - fileListContentArray, - (progress) => { - const subProgress = progress / 100 - currentProgress += - (expectedProgressForItem * subProgress - collectedProgress) * 100 - console.log(TAG, 'Current progress', currentProgress) - onProgress?.(currentProgress, 'Preparing videos...') - collectedProgress = expectedProgressForItem * subProgress - } - ) - } else { - const videoFilename = `video_${UUID()}.mp4` - await ffmpeg.writeFile(videoFilename, video.data) - fileListContentArray.push(`file ${videoFilename}`) - currentProgress += expectedProgressForItem * 100 - console.log(TAG, 'Current progress', currentProgress) - onProgress?.(currentProgress, 'Preparing videos...') - } - } - - onProgress?.(targetProgress, 'Concatenating videos...') - currentProgress = 70 - targetProgress = 100 - - const fileListContent = fileListContentArray.join('\n') - await ffmpeg.writeFile(fileList, fileListContent) - - const creatBaseFullVideoExitCode = await captureFFmpegProgress( - ffmpeg, - totalVideoDurationInMs, - async () => { - await ffmpeg.exec([ - '-f', - 'concat', - '-safe', - '0', - '-i', - fileList, - '-loglevel', - 'verbose', - '-c', - 'copy', - filename, - ]) - }, - (progress: number) => { - onProgress?.( - calculateProgress(currentProgress, progress, targetProgress), - 'Merging audio and video...' - ) - } - ) - - if (creatBaseFullVideoExitCode) { - throw new Error(`${TAG}: Error while creating base full video!`) - } - onProgress?.(targetProgress, 'Concatenating videos...') -} +import { + calculateProgress, + captureFFmpegProgress, + createFullAudio, + createFullSilentVideo, + FFMPegAudioInput, + FFMPegVideoInput, + loadFFmpegMt, + loadFFmpegSt, + TAG, +} from './ffmpegUtils' /** * Creates full video with audio using `@ffmpeg/ffmpeg` multi-core, diff --git a/src/services/io/extractCaptionFromFrame.ts b/src/services/io/extractCaptionFromFrameMoondream.ts similarity index 92% rename from src/services/io/extractCaptionFromFrame.ts rename to src/services/io/extractCaptionFromFrameMoondream.ts index 014124d0..d25eb23a 100644 --- a/src/services/io/extractCaptionFromFrame.ts +++ b/src/services/io/extractCaptionFromFrameMoondream.ts @@ -5,7 +5,7 @@ import { RawImage, } from '@xenova/transformers' -export async function extractCaptionFromFrame( +export async function extractCaptionFromFrameMoondream( imageInBase64DataUri: string ): Promise { if (!(navigator as any).gpu) { @@ -16,7 +16,8 @@ export async function extractCaptionFromFrame( 2. You need to enable WebGPU (depends on your browser, see below) 2.1 For Chrome: Perform the following operations in the Chrome / Microsoft Edge address bar -The chrome://flags/#enable-unsafe-webgpu flag must be enabled (not enable-webgpu-developer-features). Linux experimental support also requires launching the browser with --enable-features=Vulkan. +The chrome://flags/#enable-unsafe-webgpu flag must be enabled (not enable-webgpu-developer-features). +Linux experimental support also requires launching the browser with --enable-features=Vulkan. 2.2 For Safari 18 (macOS 15): WebGPU is enabled by default diff --git a/src/services/io/extractCaptionsFromFrames.ts b/src/services/io/extractCaptionsFromFrames.ts new file mode 100644 index 00000000..2017b1ac --- /dev/null +++ b/src/services/io/extractCaptionsFromFrames.ts @@ -0,0 +1,95 @@ +import { + AutoProcessor, + AutoTokenizer, + Florence2ForConditionalGeneration, + RawImage, +} from '@xenova/transformers' + +export async function extractCaptionsFromFrames( + images: string[] = [], + onProgress: ( + progress: number, + storyboardIndex: number, + nbStoryboards: number + ) => void +): Promise { + if (!(navigator as any).gpu) { + throw new Error(`Please enable WebGPU to analyze video frames: + +1. You need a modern browser such as Google Chrome 113+, Microsoft Edge 113+, Safari 18 (macOS 15), Firefox Nightly + +2. You need to enable WebGPU (depends on your browser, see below) + +2.1 For Chrome: Perform the following operations in the Chrome / Microsoft Edge address bar +The chrome://flags/#enable-unsafe-webgpu flag must be enabled (not enable-webgpu-developer-features). +Linux experimental support also requires launching the browser with --enable-features=Vulkan. + +2.2 For Safari 18 (macOS 15): WebGPU is enabled by default + +2.3 For Firefox Nightly: Type about:config in the address bar and set 'dom.webgpu.enabled" to true +`) + } + + let progress = 0 + onProgress(progress, 0, images.length) + // for code example, see: + // https://github.com/xenova/transformers.js/pull/545#issuecomment-2183625876 + + // Load model, processor, and tokenizer + const model_id = 'onnx-community/Florence-2-base-ft' + const model = await Florence2ForConditionalGeneration.from_pretrained( + model_id, + { + dtype: 'fp32', + } + ) + + onProgress((progress = 5), 0, images.length) + + const processor = await AutoProcessor.from_pretrained(model_id) + + onProgress((progress = 10), 0, images.length) + + const tokenizer = await AutoTokenizer.from_pretrained(model_id) + + onProgress((progress = 15), 0, images.length) + + // not all prompts will work properly, see the official examples: + // https://huggingface.co/microsoft/Florence-2-base-ft/blob/e7a5acc73559546de6e12ec0319cd7cc1fa2437c/processing_florence2.py#L115-L117 + + // Prepare text inputs + const prompts = 'Describe with a paragraph what is shown in the image.' + const text_inputs = tokenizer(prompts) + + let i = 1 + const captions: string[] = [] + for (const imageInBase64DataUri of images) { + console.log('analyzing image:', imageInBase64DataUri.slice(0, 64)) + // Prepare vision inputs + const image = await RawImage.fromURL(imageInBase64DataUri) + const vision_inputs = await processor(image) + + console.log(' - generating caption..') + // Generate text + const generated_ids = await model.generate({ + ...text_inputs, + ...vision_inputs, + max_new_tokens: 100, + }) + + // Decode generated text + const generated_text = tokenizer.batch_decode(generated_ids, { + skip_special_tokens: true, + }) + + const caption = `${generated_text[0] || ''}` + console.log(' - caption:', caption) + + const relativeProgress = i / images.length + + progress += relativeProgress * 75 + onProgress(progress, i, images.length) + captions.push(caption) + } + return captions +} diff --git a/src/services/io/extractFramesFromVideo.ts b/src/services/io/extractFramesFromVideo.ts index 78f22875..528ae7db 100644 --- a/src/services/io/extractFramesFromVideo.ts +++ b/src/services/io/extractFramesFromVideo.ts @@ -1,3 +1,5 @@ +'use client' + import { FFmpeg } from '@ffmpeg/ffmpeg' import { toBlobURL } from '@ffmpeg/util' import mediaInfoFactory, { @@ -17,14 +19,20 @@ interface FrameExtractorOptions { maxHeight: number sceneSamplingRate: number // Percentage of additional frames between scene changes (0-100) onProgress?: (progress: number) => void // Callback function for progress updates + debug?: boolean } -async function extractFramesFromVideo( +export async function extractFramesFromVideo( videoBlob: Blob, options: FrameExtractorOptions ): Promise { // Initialize MediaInfo - const mediaInfo = await mediaInfoFactory({ format: 'object' }) + const mediaInfo = await mediaInfoFactory({ + format: 'object', + locateFile: () => { + return '/wasm/MediaInfoModule.wasm' + }, + }) // Get video duration using MediaInfo const getSize = () => videoBlob.size @@ -42,19 +50,33 @@ async function extractFramesFromVideo( reader.readAsArrayBuffer(videoBlob.slice(offset, offset + chunkSize)) }) + if (options.debug) { + console.log('calling await mediaInfo.analyzeData(getSize, readChunk)') + } + const result = await mediaInfo.analyzeData(getSize, readChunk) + if (options.debug) { + console.log('result = ', result) + } let duration: number = 0 for (const track of result.media?.track || []) { - /// '@type': "General" | "Video" | "Audio" | "Text" | "Image" | "Menu" | "Other" + if (options.debug) { + console.log('track = ', track) + } + let maybeDuration: number = 0 if (track['@type'] === 'Audio') { const audioTrack = track as AudioTrack - maybeDuration = audioTrack.Duration || 0 + maybeDuration = audioTrack.Duration + ? parseFloat(`${audioTrack.Duration || 0}`) + : 0 } else if (track['@type'] === 'Video') { const videoTrack = track as VideoTrack - maybeDuration = videoTrack.Duration || 0 + maybeDuration = videoTrack.Duration + ? parseFloat(`${videoTrack.Duration || 0}`) + : 0 } if ( typeof maybeDuration === 'number' && @@ -69,6 +91,10 @@ async function extractFramesFromVideo( throw new Error('Could not determine video duration (or it is length 0)') } + if (options.debug) { + console.log('duration in seconds:', duration) + } + // Initialize FFmpeg const ffmpeg = new FFmpeg() const baseURL = 'https://unpkg.com/@ffmpeg/core@0.12.6/dist/umd' @@ -78,17 +104,26 @@ async function extractFramesFromVideo( wasmURL: await toBlobURL(`${baseURL}/ffmpeg-core.wasm`, 'application/wasm'), }) + if (options.debug) { + console.log('FFmpeg loaded!') + } + // Write video file to FFmpeg's file system const videoUint8Array = new Uint8Array(await videoBlob.arrayBuffer()) await ffmpeg.writeFile('input.mp4', videoUint8Array) - + if (options.debug) { + console.log('input.mp4 written!') + } // Prepare FFmpeg command const sceneFilter = `select='gt(scene,0.4)'` const additionalFramesFilter = `select='not(mod(n,${Math.floor(100 / options.sceneSamplingRate)}))'` - const scaleFilter = `scale=iw*min(${options.maxWidth}/iw\,${options.maxHeight}/ih):ih*min(${options.maxWidth}/iw\,${options.maxHeight}/ih)` + const scaleFilter = `scale='min(${options.maxWidth},iw)':min'(${options.maxHeight},ih)':force_original_aspect_ratio=decrease` let lastProgress = 0 ffmpeg.on('log', ({ message }) => { + if (options.debug) { + console.log('FFmpeg log:', message) + } const timeMatch = message.match(/time=(\d{2}):(\d{2}):(\d{2}\.\d{2})/) if (timeMatch) { const [, hours, minutes, seconds] = timeMatch @@ -102,40 +137,82 @@ async function extractFramesFromVideo( } }) - await ffmpeg.exec([ + const ffmpegCommand = [ '-i', 'input.mp4', + '-loglevel', + 'verbose', '-vf', `${sceneFilter},${additionalFramesFilter},${scaleFilter}`, '-vsync', - '0', + '2', '-q:v', '2', + '-f', + 'image2', + '-frames:v', + '1000', // Limit the number of frames to extract `frames_%03d.${options.format}`, - ]) + ] + + if (options.debug) { + console.log('Executing FFmpeg command:', ffmpegCommand.join(' ')) + } + + try { + await ffmpeg.exec(ffmpegCommand) + } catch (error) { + console.error('FFmpeg execution error:', error) + throw error + } // Read generated frames const files = await ffmpeg.listDir('/') + if (options.debug) { + console.log('All files in FFmpeg filesystem:', files) + } const frameFiles = files.filter( (file) => file.name.startsWith('frames_') && file.name.endsWith(`.${options.format}`) ) + if (options.debug) { + console.log('Frame files found:', frameFiles.length) + } const frames: string[] = [] + const encoder = new TextEncoder() + for (let i = 0; i < frameFiles.length; i++) { const file = frameFiles[i] - const frameData = await ffmpeg.readFile(file.name) - const base64Frame = btoa( - String.fromCharCode.apply(null, frameData as unknown as number[]) - ) - frames.push(`data:image/${options.format};base64,${base64Frame}`) - - // Update progress for frame processing (from 90% to 100%) - options.onProgress?.(90 + Math.round(((i + 1) / frameFiles.length) * 10)) + if (options.debug) { + console.log(`Processing frame file: ${file.name}`) + } + try { + const frameData = await ffmpeg.readFile(file.name) + + // Convert Uint8Array to Base64 string without using btoa + let binary = '' + const bytes = new Uint8Array(frameData as any) + const len = bytes.byteLength + for (let i = 0; i < len; i++) { + binary += String.fromCharCode(bytes[i]) + } + const base64Frame = window.btoa(binary) + + frames.push(`data:image/${options.format};base64,${base64Frame}`) + + // Update progress for frame processing (from 90% to 100%) + options.onProgress?.(90 + Math.round(((i + 1) / frameFiles.length) * 10)) + } catch (error) { + console.error(`Error processing frame ${file.name}:`, error) + // You can choose to either skip this frame or throw an error + // throw error; // Uncomment this line if you want to stop processing on any error + } } + if (options.debug) { + console.log(`Total frames processed: ${frames.length}`) + } return frames } - -export default extractFramesFromVideo diff --git a/src/services/io/ffmpegUtils.ts b/src/services/io/ffmpegUtils.ts new file mode 100644 index 00000000..40b85972 --- /dev/null +++ b/src/services/io/ffmpegUtils.ts @@ -0,0 +1,478 @@ +import { UUID } from '@aitube/clap' +import { FFmpeg } from '@ffmpeg/ffmpeg' +import { toBlobURL } from '@ffmpeg/util' + +export const TAG = 'io/createFullVideo' + +export type FFMPegVideoInput = { + data: Uint8Array | null + startTimeInMs: number + endTimeInMs: number + durationInSecs: number +} + +export type FFMPegAudioInput = FFMPegVideoInput + +/** + * Download and load single and multi-threading FFMPeg. + * MT for video + * ST for audio (as MT has issues with it) + * toBlobURL is used to bypass CORS issues, urls with the same domain can be used directly. + */ +export async function initializeFFmpeg() { + const [ffmpegSt, ffmpegMt] = [new FFmpeg(), new FFmpeg()] + const baseStURL = 'https://unpkg.com/@ffmpeg/core@0.12.6/dist/umd' + const baseMtURL = 'https://unpkg.com/@ffmpeg/core-mt@0.12.6/dist/umd' + + ffmpegSt.on('log', ({ message }) => { + console.log(TAG, 'FFmpeg Single-Thread:', message) + }) + + ffmpegMt.on('log', ({ message }) => { + console.log(TAG, 'FFmpeg Multi-Thread:', message) + }) + + await ffmpegSt.load({ + coreURL: await toBlobURL(`${baseStURL}/ffmpeg-core.js`, 'text/javascript'), + wasmURL: await toBlobURL( + `${baseStURL}/ffmpeg-core.wasm`, + 'application/wasm' + ), + }) + + await ffmpegMt.load({ + coreURL: await toBlobURL(`${baseMtURL}/ffmpeg-core.js`, 'text/javascript'), + wasmURL: await toBlobURL( + `${baseMtURL}/ffmpeg-core.wasm`, + 'application/wasm' + ), + workerURL: await toBlobURL( + `${baseMtURL}/ffmpeg-core.worker.js`, + 'text/javascript' + ), + }) + + return [ffmpegSt, ffmpegMt] as [FFmpeg, FFmpeg] +} + +/** + * Get loaded FFmpeg. + */ +let ffmpegInstance: [FFmpeg, FFmpeg] +export async function loadFFmpegSt() { + if (!ffmpegInstance) ffmpegInstance = await initializeFFmpeg() + return ffmpegInstance[0] +} + +export async function loadFFmpegMt() { + if (!ffmpegInstance) ffmpegInstance = await initializeFFmpeg() + return ffmpegInstance[1] +} + +/** + * Creates an exclusive logger for the FFmpeg calls inside the provided method, + * it calculates the progress based on raw FFmpeg logs and the provided `totalTimeInMs`. + * + * @param totalTimeInMs + * @param method + * @param callback + * @param {number} callback.progress - The progress of the FFmpeg process from 0 to 100. + * @returns + */ +export async function captureFFmpegProgress( + ffmpeg: FFmpeg, + totalTimeInMs: number, + method: () => any, + callback: (progress: number) => void +): Promise { + const extractProgressTimeMsFromLogs = (log: string): number | null => { + // `frame` for videos, `size` for audios + if (!log.startsWith('frame') && !log.startsWith('size')) return null + const timeRegex = /time=(\d{2}):(\d{2}):(\d{2})\.(\d{2})/ + const match = log.match(timeRegex) + if (match) { + const hours = parseInt(match[1]) + const minutes = parseInt(match[2]) + const seconds = parseInt(match[3]) + const centiseconds = parseInt(match[4]) + const totalMilliseconds = + hours * 3600000 + minutes * 60000 + seconds * 1000 + centiseconds * 10 + return totalMilliseconds + } + return null + } + let ffmpegLog = true + ffmpeg.on('log', ({ message }) => { + if (!ffmpegLog) return + const timeInMs = extractProgressTimeMsFromLogs(message) + if (timeInMs) callback((timeInMs / totalTimeInMs) * 100) + }) + const result = await method() + ffmpegLog = false + return result +} + +/** + * It will calculate a proportional progress between a targetProgress and a startProgress + * + * @param startProgress e.g. 50 + * @param progress e.g. 50 + * @param targetProgress e.g. 70 + * @returns e.g. 60, because 50% of progress between 70% and 50%, would result on 60% + */ +export function calculateProgress( + startProgress: number, + progress: number, + targetProgress: number +): number { + return startProgress + (progress * (targetProgress - startProgress)) / 100 +} + +/** + * Creates an empty black video and appends it to the + * provided `fileListContentArray`. + * + * @param duration time in milliseconds + * @param width + * @param height + * @param filename + * @param fileListContentArray fileList.txt where to append the file name + * @param onProgress callback to capture the progress of this method + */ +export async function addEmptyVideo( + durationInSecs: number, + width: number, + height: number, + filename: string, + fileListContentArray: string[], + onProgress?: (progress: number, message?: string) => void +) { + const ffmpeg = await loadFFmpegMt() + let targetPartialProgress = 0 + + // For some reason, creating empty video with silent audio + // in one exec doesn't work, we need to split it. + + console.log( + TAG, + 'Creating empty video', + filename, + width, + height, + durationInSecs + ) + let currentProgress = 0 + targetPartialProgress = 50 + + await captureFFmpegProgress( + ffmpeg, + durationInSecs * 1000, + async () => { + await ffmpeg.exec([ + '-f', + 'lavfi', + '-i', + `color=c=black:s=${width}x${height}:d=${durationInSecs}`, + '-c:v', + 'libx264', + '-t', + `${durationInSecs}`, + '-loglevel', + 'verbose', + `base_${filename}`, + ]) + }, + (progress) => { + onProgress?.((progress / 100) * targetPartialProgress) + } + ) + + console.log( + TAG, + 'Adding silent audio to empty video', + filename, + width, + height, + durationInSecs + ) + currentProgress = 50 + targetPartialProgress = 100 + + const exitCode = await ffmpeg.exec([ + '-i', + `base_${filename}`, + '-f', + 'lavfi', + '-i', + 'anullsrc', + '-c:v', + 'copy', + '-c:a', + 'aac', + '-t', + `${durationInSecs}`, + '-loglevel', + 'verbose', + filename, + ]) + + if (exitCode) { + throw new Error(`${TAG}: Unexpect error while creating empty video`) + } + + console.log(TAG, 'Empty video created', filename) + fileListContentArray.push(`file ${filename}`) +} + +/** + * Creates the full mixed audio including silence + * segments and loads it into ffmpeg with the given `filename`. + * @param onProgress callback to capture the progress of this method + * @throws Error if ffmpeg returns exit code 1 + */ +export async function createFullAudio( + audios: FFMPegAudioInput[], + filename: string, + totalVideoDurationInMs: number, + onProgress?: (progress: number, message: string) => void +): Promise { + console.log(TAG, 'Creating full audio', filename) + + const ffmpeg = await loadFFmpegSt() + const filterComplexParts = [] + const baseFilename = `base_${filename}` + let currentProgress = 0 + let targetProgress = 25 + + // To mix audios at given times, we need a first empty base audio track + + await captureFFmpegProgress( + ffmpeg, + totalVideoDurationInMs, + async () => { + await ffmpeg.exec([ + '-f', + 'lavfi', + '-i', + 'anullsrc', + '-t', + `${totalVideoDurationInMs / 1000}`, + '-loglevel', + 'verbose', + !audios.length ? filename : baseFilename, + ]) + }, + (progress) => { + onProgress?.( + calculateProgress(currentProgress, progress, targetProgress), + 'Creating base audio...' + ) + } + ) + + // If there is no audios, the base audio is the final one + if (!audios.length) return onProgress?.(100, 'Prepared audios...') + + currentProgress = targetProgress + targetProgress = 50 + + // Mix audios based on their start times + + const audioInputFiles = ['-i', baseFilename] + for (let index = 0; index < audios.length; index++) { + onProgress?.(currentProgress, 'Creating base audio...') + console.log(TAG, `Processing audio #${index}`) + const audio = audios[index] + const expectedProgressForItem = ((1 / audios.length) * targetProgress) / 100 + if (!audio.data) continue + const audioFilename = `audio_${UUID()}.mp3` + await ffmpeg.writeFile(audioFilename, audio.data) + audioInputFiles.push('-i', audioFilename) + const delay = audio.startTimeInMs + const durationInSecs = audio.endTimeInMs - audio.startTimeInMs / 1000 + filterComplexParts.push( + `[${index + 1}:a]atrim=0:${durationInSecs},adelay=${delay}|${delay}[delayed${index}]` + ) + currentProgress += expectedProgressForItem * 100 + } + + const amixInputs = `[0:a]${audios.map((_, index) => `[delayed${index}]`).join('')}amix=inputs=${audios.length + 1}:duration=longest` + filterComplexParts.push(`${amixInputs}[a]`) + const filterComplex = filterComplexParts.join('; ') + + currentProgress = targetProgress + targetProgress = 100 + + const createFullAudioExitCode = await captureFFmpegProgress( + ffmpeg, + totalVideoDurationInMs, + async () => { + await ffmpeg.exec([ + ...audioInputFiles, + '-filter_complex', + filterComplex, + '-map', + '[a]', + '-t', + `${totalVideoDurationInMs / 1000}`, + '-loglevel', + 'verbose', + filename, + ]) + }, + (progress) => { + onProgress?.( + calculateProgress(currentProgress, progress, targetProgress), + 'Mixing audios...' + ) + } + ) + + if (createFullAudioExitCode) { + throw new Error(`${TAG}: Error while creating full audio!`) + } + onProgress?.(targetProgress, 'Prepared audios...') +} + +/** + * Creates the full silent video including empty black + * segments and loads it into ffmpeg with the given `filename`. + * @param onProgress callback to capture the progress of this method + * @throws Error if ffmpeg returns exit code 1 + */ +export async function createFullSilentVideo( + videos: FFMPegVideoInput[], + filename: string, + totalVideoDurationInMs: number, + width: number, + height: number, + excludeEmptyContent = false, + onProgress?: (progress: number, message: string) => void +) { + const ffmpeg = await loadFFmpegMt() + const fileList = 'fileList.txt' + const fileListContentArray = [] + + // Complete array of videos including concatenated empty segments + // This is helpful for cleaner progress log + let lastStartTimeVideoInMs = 0 + let videosWithGaps: FFMPegVideoInput[] + + if (!videos.length) { + videosWithGaps = [ + { + startTimeInMs: 0, + endTimeInMs: totalVideoDurationInMs, + data: null, + durationInSecs: totalVideoDurationInMs / 1000, + }, + ] + } else { + videosWithGaps = videos.reduce((arr: FFMPegVideoInput[], video, index) => { + const emptyVideoDurationInMs = + video.startTimeInMs - lastStartTimeVideoInMs + if (emptyVideoDurationInMs) { + arr.push({ + startTimeInMs: lastStartTimeVideoInMs, + endTimeInMs: lastStartTimeVideoInMs + emptyVideoDurationInMs, + data: null, + durationInSecs: emptyVideoDurationInMs / 1000, + }) + } + arr.push(video) + lastStartTimeVideoInMs = video.endTimeInMs + if ( + index == videos.length - 1 && + lastStartTimeVideoInMs < totalVideoDurationInMs + ) { + arr.push({ + startTimeInMs: lastStartTimeVideoInMs, + endTimeInMs: totalVideoDurationInMs, + data: null, + durationInSecs: + (totalVideoDurationInMs - lastStartTimeVideoInMs) / 1000, + }) + } + return arr + }, []) + } + + onProgress?.(0, 'Preparing videos...') + + // Arbitrary percentage, as `concat` is fast, + // then estimate the generation of gap videos + // as the 70% of the work + let currentProgress = 0 + let targetProgress = 70 + + for (const video of videosWithGaps) { + const expectedProgressForItem = + (((video.durationInSecs * 1000) / totalVideoDurationInMs) * + targetProgress) / + 100 + if (!video.data) { + if (excludeEmptyContent) continue + let collectedProgress = 0 + await addEmptyVideo( + video.durationInSecs, + width, + height, + `empty_video_${UUID()}.mp4`, + fileListContentArray, + (progress) => { + const subProgress = progress / 100 + currentProgress += + (expectedProgressForItem * subProgress - collectedProgress) * 100 + console.log(TAG, 'Current progress', currentProgress) + onProgress?.(currentProgress, 'Preparing videos...') + collectedProgress = expectedProgressForItem * subProgress + } + ) + } else { + const videoFilename = `video_${UUID()}.mp4` + await ffmpeg.writeFile(videoFilename, video.data) + fileListContentArray.push(`file ${videoFilename}`) + currentProgress += expectedProgressForItem * 100 + console.log(TAG, 'Current progress', currentProgress) + onProgress?.(currentProgress, 'Preparing videos...') + } + } + + onProgress?.(targetProgress, 'Concatenating videos...') + currentProgress = 70 + targetProgress = 100 + + const fileListContent = fileListContentArray.join('\n') + await ffmpeg.writeFile(fileList, fileListContent) + + const creatBaseFullVideoExitCode = await captureFFmpegProgress( + ffmpeg, + totalVideoDurationInMs, + async () => { + await ffmpeg.exec([ + '-f', + 'concat', + '-safe', + '0', + '-i', + fileList, + '-loglevel', + 'verbose', + '-c', + 'copy', + filename, + ]) + }, + (progress: number) => { + onProgress?.( + calculateProgress(currentProgress, progress, targetProgress), + 'Merging audio and video...' + ) + } + ) + + if (creatBaseFullVideoExitCode) { + throw new Error(`${TAG}: Error while creating base full video!`) + } + onProgress?.(targetProgress, 'Concatenating videos...') +} diff --git a/src/services/io/parseFileIntoSegments.ts b/src/services/io/parseFileIntoSegments.ts index 8d6d2f1e..15519349 100644 --- a/src/services/io/parseFileIntoSegments.ts +++ b/src/services/io/parseFileIntoSegments.ts @@ -5,6 +5,7 @@ import { ClapOutputType, ClapSegmentCategory, ClapSegmentStatus, + isValidNumber, newSegment, UUID, } from '@aitube/clap' @@ -13,6 +14,9 @@ import { SegmentEditionStatus, SegmentVisibility, TimelineSegment, + useTimeline, + TimelineStore, + DEFAULT_DURATION_IN_MS_PER_STEP, } from '@aitube/timeline' import { blobToBase64DataUri } from '@/lib/utils/blobToBase64DataUri' @@ -22,12 +26,21 @@ import { ResourceCategory, ResourceType } from '@aitube/clapper-services' export async function parseFileIntoSegments({ file, + track, + startTimeInMs: maybeStartTimeInMs, + endTimeInMs: maybeEndTimeInMs, }: { /** * The file to import */ file: File + + track?: number + startTimeInMs?: number + endTimeInMs?: number }): Promise { + const timeline: TimelineStore = useTimeline.getState() + const { cursorTimestampAtInMs } = timeline // console.log(`parseFileIntoSegments(): filename = ${file.name}`) // console.log(`parseFileIntoSegments(): file size = ${file.size} bytes`) // console.log(`parseFileIntoSegments(): file type = ${file.type}`) @@ -38,16 +51,69 @@ export async function parseFileIntoSegments({ 'TODO: open a popup to ask if this is a voice character sample, dialogue, music etc' ) - let type: ResourceType = 'misc' - let resourceCategory: ResourceCategory = 'misc' - const newSegments: TimelineSegment[] = [] switch (file.type) { - case 'image/webp': - type = 'image' - resourceCategory = 'control_image' + case 'image/jpeg': + case 'image/png': + case 'image/avif': + case 'image/heic': + case 'image/webp': { + const type: ResourceType = 'image' + const resourceCategory: ResourceCategory = 'control_image' + + // ok let's stop for a minute there: + // if someone drops a .mp3, and assuming we don't yet have the UI to select the category, + // do you think it should be a SOUND, a VOICE or a MUSIC by default? + // I expect people will use AI service providers for sound and voice, + // maybe in some case music too, but there are also many people + // who will want to use their own track eg. to create a music video + const category = ClapSegmentCategory.STORYBOARD + + const assetUrl = await blobToBase64DataUri(file) + + const startTimeInMs = isValidNumber(maybeStartTimeInMs) + ? maybeStartTimeInMs! + : cursorTimestampAtInMs + const durationInSteps = 4 + const durationInMs = durationInSteps * DEFAULT_DURATION_IN_MS_PER_STEP + const endTimeInMs = isValidNumber(maybeEndTimeInMs) + ? maybeEndTimeInMs! + : startTimeInMs + durationInMs + + const newSegmentData: Partial = { + prompt: 'Storyboard', // note: this can be set later with an automatic captioning worker + startTimeInMs, // start time of the segment + endTimeInMs, // end time of the segment (startTimeInMs + durationInMs) + status: ClapSegmentStatus.COMPLETED, + // track: findFreeTrack({ segments, startTimeInMs, endTimeInMs }), // track row index + label: `${file.name}`, // a short label to name the segment (optional, can be human or LLM-defined) + category, + assetUrl, + assetDurationInMs: endTimeInMs, + assetSourceType: ClapAssetSource.DATA, + assetFileFormat: `${file.type}`, + } + + const timelineSegment = await clapSegmentToTimelineSegment( + newSegment(newSegmentData) + ) + + if (isValidNumber(track)) { + timelineSegment.track = track + } + + timelineSegment.outputType = ClapOutputType.IMAGE + + // we assume we want it to be immediately visible + timelineSegment.visibility = SegmentVisibility.VISIBLE + + // console.log("newSegment:", audioSegment) + + // poof! type disappears.. it's magic + newSegments.push(timelineSegment) break + } case 'audio/mpeg': // this is the "official" one case 'audio/mp3': // this is just an alias @@ -56,10 +122,10 @@ export async function parseFileIntoSegments({ case 'audio/x-mp4': // should be rare, normally is is audio/mp4 case 'audio/m4a': // shouldn't exist case 'audio/x-m4a': // should be rare, normally is is audio/mp4 - case 'audio/webm': + case 'audio/webm': { // for background track, or as an inspiration track, or a voice etc - type = 'audio' - resourceCategory = 'background_music' + const type: ResourceType = 'audio' + const resourceCategory: ResourceCategory = 'background_music' // TODO: add caption analysis const { durationInMs, durationInSteps, bpm, audioBuffer } = @@ -71,11 +137,12 @@ export async function parseFileIntoSegments({ }) // TODO: use the correct drop time - const startTimeInMs = 0 - const startTimeInSteps = 1 - - const endTimeInSteps = durationInSteps - const endTimeInMs = startTimeInMs + durationInMs + const startTimeInMs = isValidNumber(maybeStartTimeInMs) + ? maybeStartTimeInMs! + : 0 + const endTimeInMs = isValidNumber(maybeEndTimeInMs) + ? maybeEndTimeInMs! + : startTimeInMs + durationInMs // ok let's stop for a minute there: // if someone drops a .mp3, and assuming we don't yet have the UI to select the category, @@ -92,6 +159,7 @@ export async function parseFileIntoSegments({ startTimeInMs, // start time of the segment endTimeInMs, // end time of the segment (startTimeInMs + durationInMs) status: ClapSegmentStatus.COMPLETED, + track, // track: findFreeTrack({ segments, startTimeInMs, endTimeInMs }), // track row index label: `${file.name} (${Math.round(durationInMs / 1000)}s @ ${Math.round(bpm * 100) / 100} BPM)`, // a short label to name the segment (optional, can be human or LLM-defined) category, @@ -104,6 +172,11 @@ export async function parseFileIntoSegments({ const timelineSegment = await clapSegmentToTimelineSegment( newSegment(newSegmentData) ) + + if (isValidNumber(track)) { + timelineSegment.track = track + } + timelineSegment.outputType = ClapOutputType.AUDIO timelineSegment.outputGain = 1.0 timelineSegment.audioBuffer = audioBuffer @@ -116,28 +189,31 @@ export async function parseFileIntoSegments({ // poof! type disappears.. it's magic newSegments.push(timelineSegment) break + } - case 'text/plain': + case 'text/plain': { // for dialogue, prompts.. - type = 'text' - resourceCategory = 'text_prompt' + const type: ResourceType = 'text' + const resourceCategory: ResourceCategory = 'text_prompt' break + } - default: + default: { console.log(`unrecognized file type "${file.type}"`) break + } } // note: we always upload the files, because even if it is an unhandled format (eg. a PDF) // this can still be part of the project as a resource for humans (inspiration, guidelines etc) + /* const id = UUID() const fileName = `${id}.${extension}` const storage = `resources` const filePath = `${type}/${fileName}` - /* const { data, error } = await supabase .storage .from('avatars') diff --git a/src/services/io/useIO.ts b/src/services/io/useIO.ts index 9e11f667..8e056198 100644 --- a/src/services/io/useIO.ts +++ b/src/services/io/useIO.ts @@ -18,6 +18,7 @@ import { TimelineSegment, removeFinalVideosAndConvertToTimelineSegments, getFinalVideo, + DEFAULT_DURATION_IN_MS_PER_STEP, } from '@aitube/timeline' import { ParseScriptProgressUpdate, parseScriptToClap } from '@aitube/broadway' import { IOStore, TaskCategory, TaskVisibility } from '@aitube/clapper-services' @@ -40,11 +41,11 @@ import { formatSegmentForExport, } from '@/lib/utils/formatSegmentForExport' import { sleep } from '@/lib/utils/sleep' -import { - FFMPegAudioInput, - FFMPegVideoInput, - createFullVideo, -} from './createFullVideo' +import { FFMPegAudioInput, FFMPegVideoInput } from './ffmpegUtils' +import { createFullVideo } from './createFullVideo' +import { extractFramesFromVideo } from './extractFramesFromVideo' +import { extractCaptionsFromFrames } from './extractCaptionsFromFrames' +import { base64DataUriToFile } from '@/lib/utils/base64DataUriToFile' export const useIO = create((set, get) => ({ ...getDefaultIOState(), @@ -107,6 +108,93 @@ export const useIO = create((set, get) => ({ } const isVideoFile = fileType.startsWith('video/') + if (isVideoFile) { + const storyboardExtractionTask = useTasks.getState().add({ + category: TaskCategory.IMPORT, + visibility: TaskVisibility.BLOCKER, + initialMessage: `Extracting storyboards..`, + successMessage: `Extracting storyboards.. 100% done`, + value: 0, + }) + + const frames = await extractFramesFromVideo(file, { + format: 'png', // in theory we could also use 'jpg', but this freezes FFmpeg + maxWidth: 1024, + maxHeight: 576, + sceneSamplingRate: 100, + onProgress: (progress: number) => { + storyboardExtractionTask.setProgress({ + message: `Extracting storyboards.. ${progress}% done`, + value: progress, + }) + }, + }) + + // optional: reset the project + await timeline.setClap(newClap()) + + const track = 1 + let i = 0 + let startTimeInMs = 0 + const durationInSteps = 4 + const durationInMs = durationInSteps * DEFAULT_DURATION_IN_MS_PER_STEP + let endTimeInMs = startTimeInMs + durationInMs + + for (const frame of frames) { + const frameFile = base64DataUriToFile(frame, `storyboard_${i++}.png`) + const newSegments = await parseFileIntoSegments({ + file: frameFile, + startTimeInMs, + endTimeInMs, + track, + }) + + startTimeInMs += durationInMs + endTimeInMs += durationInMs + + console.log('calling timeline.addSegments with:', newSegments) + await timeline.addSegments({ + segments: newSegments, + track, + }) + } + + storyboardExtractionTask.success() + + const enableCaptioning = false + + if (enableCaptioning) { + const captioningTask = useTasks.getState().add({ + category: TaskCategory.IMPORT, + // visibility: TaskVisibility.BLOCKER, + + // since this is very long task, we can run it in the background + visibility: TaskVisibility.BACKGROUND, + initialMessage: `Analyzing storyboards..`, + successMessage: `Analyzing storyboards.. 100% done`, + value: 0, + }) + + console.log('calling extractCaptionsFromFrames() with:', frames) + const captions = await extractCaptionsFromFrames( + frames, + ( + progress: number, + storyboardIndex: number, + nbStoryboards: number + ) => { + captioningTask.setProgress({ + message: `Analyzing storyboards (${progress}%)`, + value: progress, + }) + } + ) + console.log('captions:', captions) + // TODO: add + + captioningTask.success() + } + } } }, openScreenplay: async (