From 5bd8810dc28a10d8f5a82b0d188b87a090ce43ef Mon Sep 17 00:00:00 2001 From: Julian Bilcke Date: Mon, 22 Jul 2024 22:57:08 +0200 Subject: [PATCH] first version of the video analyzer --- package-lock.json | 176 ++++----- package.json | 4 +- public/images/onboarding/get-started.png | 3 + public/images/onboarding/get-started.xcf | 3 + public/images/onboarding/pick-an-example.png | 3 + public/images/onboarding/pick-an-example.xcf | 3 + src/app/main.tsx | 67 +++- .../toolbars/top-menu/file/index.tsx | 60 ++- src/components/toolbars/top-menu/index.tsx | 9 +- src/lib/core/constants.ts | 2 +- src/lib/hooks/useOpenFilePicker.ts | 36 +- src/lib/utils/base64DataUriToFile.ts | 2 +- src/services/io/extractFramesFromVideo.ts | 104 +++-- src/services/io/extractScenesFromVideo.ts | 369 ++++++++++++++++++ src/services/io/fileDataToBase64.ts | 13 + src/services/io/parseFileIntoSegments.ts | 53 ++- src/services/io/useIO.ts | 249 ++++++++---- src/services/resolver/useResolver.ts | 10 +- src/services/ui/getDefaultUIState.ts | 2 + src/services/ui/useUI.ts | 7 + 20 files changed, 932 insertions(+), 243 deletions(-) create mode 100644 public/images/onboarding/get-started.png create mode 100644 public/images/onboarding/get-started.xcf create mode 100644 public/images/onboarding/pick-an-example.png create mode 100644 public/images/onboarding/pick-an-example.xcf create mode 100644 src/services/io/extractScenesFromVideo.ts create mode 100644 src/services/io/fileDataToBase64.ts diff --git a/package-lock.json b/package-lock.json index b76f0f9d..6474c59d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,9 +11,9 @@ "dependencies": { "@aitube/broadway": "0.0.22", "@aitube/clap": "0.0.30", - "@aitube/clapper-services": "0.0.29", + "@aitube/clapper-services": "0.0.34", "@aitube/engine": "0.0.26", - "@aitube/timeline": "0.0.43", + "@aitube/timeline": "0.0.44", "@fal-ai/serverless-client": "^0.13.0", "@ffmpeg/ffmpeg": "^0.12.10", "@ffmpeg/util": "^0.12.1", @@ -161,12 +161,12 @@ } }, "node_modules/@aitube/clapper-services": { - "version": "0.0.29", - "resolved": "https://registry.npmjs.org/@aitube/clapper-services/-/clapper-services-0.0.29.tgz", - "integrity": "sha512-61UH/TQwPcvXArEkPnGNm+IQulaW3zNh73pzihdU2kkqufGzUYCNSd/jHJh9dLqQm3lZtm6QMN2RReFrzGuLNQ==", + "version": "0.0.34", + "resolved": "https://registry.npmjs.org/@aitube/clapper-services/-/clapper-services-0.0.34.tgz", + "integrity": "sha512-d0HruUyWRIXozO67W+2iEUTuBdbojGPn9BnIf6cvxkVbywLwy4hKaN+SD+yQwOi/jqoqu+TTQYUoWSF93JDVEQ==", "peerDependencies": { "@aitube/clap": "0.0.30", - "@aitube/timeline": "0.0.43", + "@aitube/timeline": "0.0.44", "@monaco-editor/react": "4.6.0", "monaco-editor": "0.50.0", "react": "*", @@ -192,9 +192,9 @@ } }, "node_modules/@aitube/timeline": { - "version": "0.0.43", - "resolved": "https://registry.npmjs.org/@aitube/timeline/-/timeline-0.0.43.tgz", - "integrity": "sha512-TnzKrB955YeDKOMWsnniGbQ+qulCmGptMfhNjDLEqA6jRcsnPVUFCR2dQBqWGNn6KfFPXmDvSi0Sihy7Oj98Aw==", + "version": "0.0.44", + "resolved": "https://registry.npmjs.org/@aitube/timeline/-/timeline-0.0.44.tgz", + "integrity": "sha512-iELTtmLONWR7zuGLLr9cJRlMuNoBXWxZzgGerDeXa5VyQhDmjj4shLOlZLP78PiIVHMdRwZr16IN6ob899VmMw==", "dependencies": { "date-fns": "^3.6.0", "react-virtualized-auto-sizer": "^1.0.24" @@ -3348,20 +3348,20 @@ } }, "node_modules/@floating-ui/core": { - "version": "1.6.4", - "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.6.4.tgz", - "integrity": "sha512-a4IowK4QkXl4SCWTGUR0INAfEOX3wtsYw3rKK5InQEHMGObkR8Xk44qYQD9P4r6HHw0iIfK6GUKECmY8sTkqRA==", + "version": "1.6.5", + "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.6.5.tgz", + "integrity": "sha512-8GrTWmoFhm5BsMZOTHeGD2/0FLKLQQHvO/ZmQga4tKempYRLz8aqJGqXVuQgisnMObq2YZ2SgkwctN1LOOxcqA==", "dependencies": { - "@floating-ui/utils": "^0.2.4" + "@floating-ui/utils": "^0.2.5" } }, "node_modules/@floating-ui/dom": { - "version": "1.6.7", - "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.6.7.tgz", - "integrity": "sha512-wmVfPG5o2xnKDU4jx/m4w5qva9FWHcnZ8BvzEe90D/RpwsJaTAVYPEPdQ8sbr/N8zZTAHlZUTQdqg8ZUbzHmng==", + "version": "1.6.8", + "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.6.8.tgz", + "integrity": "sha512-kx62rP19VZ767Q653wsP1XZCGIirkE09E0QUGNYTM/ttbbQHqcGPdSfWFxUyyNLc/W6aoJRBajOSXhP6GXjC0Q==", "dependencies": { "@floating-ui/core": "^1.6.0", - "@floating-ui/utils": "^0.2.4" + "@floating-ui/utils": "^0.2.5" } }, "node_modules/@floating-ui/react-dom": { @@ -3377,9 +3377,9 @@ } }, "node_modules/@floating-ui/utils": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.4.tgz", - "integrity": "sha512-dWO2pw8hhi+WrXq1YJy2yCuWoL20PddgGaqTgVe4cOS9Q6qklXCiA1tJEqX6BEwRNSCP84/afac9hd4MS+zEUA==" + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.5.tgz", + "integrity": "sha512-sTcG+QZ6fdEUObICavU+aB3Mp8HY4n14wYHdxK4fXjPmv3PXZZeY5RaguJmGyeH/CJQhX3fqKUtS4qc1LoHwhQ==" }, "node_modules/@gar/promisify": { "version": "1.1.3", @@ -3430,9 +3430,9 @@ } }, "node_modules/@huggingface/inference/node_modules/@huggingface/tasks": { - "version": "0.11.2", - "resolved": "https://registry.npmjs.org/@huggingface/tasks/-/tasks-0.11.2.tgz", - "integrity": "sha512-vlwUJsj/QJcR/oLXvV+JBKheaVk9pqfAPYiS136cjHEDTeTW5/+ePpM6uKOc56oxqwrUjh5T0JylHJU8vyqr1A==" + "version": "0.11.3", + "resolved": "https://registry.npmjs.org/@huggingface/tasks/-/tasks-0.11.3.tgz", + "integrity": "sha512-IYq4OdlySdscjkFwm6iIqP1ZgKl4OGhvQFJWI7Yxpq2V8RmXcgIjiqk/65S6Ap7i+eyCdlOC4qweVy/ICNE0JA==" }, "node_modules/@huggingface/jinja": { "version": "0.2.2", @@ -4206,15 +4206,15 @@ } }, "node_modules/@langchain/core": { - "version": "0.2.17", - "resolved": "https://registry.npmjs.org/@langchain/core/-/core-0.2.17.tgz", - "integrity": "sha512-WnFiZ7R/ZUVeHO2IgcSL7Tu+CjApa26Iy99THJP5fax/NF8UQCc/ZRcw2Sb/RUuRPVm6ALDass0fSQE1L9YNJg==", + "version": "0.2.18", + "resolved": "https://registry.npmjs.org/@langchain/core/-/core-0.2.18.tgz", + "integrity": "sha512-ru542BwNcsnDfjTeDbIkFIchwa54ctHZR+kVrC8U9NPS9/36iM8p8ruprOV7Zccj/oxtLE5UpEhV+9MZhVcFlA==", "dependencies": { "ansi-styles": "^5.0.0", "camelcase": "6", "decamelize": "1.2.0", "js-tiktoken": "^1.0.12", - "langsmith": "~0.1.30", + "langsmith": "~0.1.39", "ml-distance": "^4.0.0", "mustache": "^4.2.0", "p-queue": "^6.6.2", @@ -6141,9 +6141,9 @@ "integrity": "sha512-iQVztO09ZVfsletMiY+DpT/JRiBntdsdJ4uqk3UJFhrhS8mIC9ZOZbmfGSRs/kdbNPQkVyzucceDicQ/3Mlj9g==" }, "node_modules/@react-three/drei": { - "version": "9.108.4", - "resolved": "https://registry.npmjs.org/@react-three/drei/-/drei-9.108.4.tgz", - "integrity": "sha512-YyPVG7+np6G8CJRVVdEfgK+bou7cvp8v9R7k4NSHsoi5EokFPG03tkCjniRiz5SzQyN+E8kCiMogI9oZaop5+g==", + "version": "9.109.0", + "resolved": "https://registry.npmjs.org/@react-three/drei/-/drei-9.109.0.tgz", + "integrity": "sha512-LlJ1k0DO5UvBdjuv6WuSP5jXb1mXsQY3VeQTfzivCsHJH9pUsbxutLL7mk84w9MI7cZytv2Qcx2nU2HBm0eNpQ==", "dependencies": { "@babel/runtime": "^7.11.2", "@mediapipe/tasks-vision": "0.10.8", @@ -7313,9 +7313,9 @@ } }, "node_modules/@testing-library/dom": { - "version": "10.3.2", - "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-10.3.2.tgz", - "integrity": "sha512-0bxIdP9mmPiOJ6wHLj8bdJRq+51oddObeCGdEf6PNEhYd93ZYAN+lPRnEOVFtheVwDM7+p+tza3LAQgp0PTudg==", + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-10.4.0.tgz", + "integrity": "sha512-pemlzrSESWbdAloYml3bAJMEfNh1Z7EduzqPKprCH5S341frlpYnUEW0H72dLxa6IsYr+mPno20GiSm+h9dEdQ==", "dev": true, "peer": true, "dependencies": { @@ -7844,13 +7844,13 @@ } }, "node_modules/@vitest/expect": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-2.0.3.tgz", - "integrity": "sha512-X6AepoOYePM0lDNUPsGXTxgXZAl3EXd0GYe/MZyVE4HzkUqyUVC6S3PrY5mClDJ6/7/7vALLMV3+xD/Ko60Hqg==", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-2.0.4.tgz", + "integrity": "sha512-39jr5EguIoanChvBqe34I8m1hJFI4+jxvdOpD7gslZrVQBKhh8H9eD7J/LJX4zakrw23W+dITQTDqdt43xVcJw==", "dev": true, "dependencies": { - "@vitest/spy": "2.0.3", - "@vitest/utils": "2.0.3", + "@vitest/spy": "2.0.4", + "@vitest/utils": "2.0.4", "chai": "^5.1.1", "tinyrainbow": "^1.2.0" }, @@ -7859,9 +7859,9 @@ } }, "node_modules/@vitest/pretty-format": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-2.0.3.tgz", - "integrity": "sha512-URM4GLsB2xD37nnTyvf6kfObFafxmycCL8un3OC9gaCs5cti2u+5rJdIflZ2fUJUen4NbvF6jCufwViAFLvz1g==", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-2.0.4.tgz", + "integrity": "sha512-RYZl31STbNGqf4l2eQM1nvKPXE0NhC6Eq0suTTePc4mtMQ1Fn8qZmjV4emZdEdG2NOWGKSCrHZjmTqDCDoeFBw==", "dev": true, "dependencies": { "tinyrainbow": "^1.2.0" @@ -7871,12 +7871,12 @@ } }, "node_modules/@vitest/runner": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-2.0.3.tgz", - "integrity": "sha512-EmSP4mcjYhAcuBWwqgpjR3FYVeiA4ROzRunqKltWjBfLNs1tnMLtF+qtgd5ClTwkDP6/DGlKJTNa6WxNK0bNYQ==", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-2.0.4.tgz", + "integrity": "sha512-Gk+9Su/2H2zNfNdeJR124gZckd5st4YoSuhF1Rebi37qTXKnqYyFCd9KP4vl2cQHbtuVKjfEKrNJxHHCW8thbQ==", "dev": true, "dependencies": { - "@vitest/utils": "2.0.3", + "@vitest/utils": "2.0.4", "pathe": "^1.1.2" }, "funding": { @@ -7884,12 +7884,12 @@ } }, "node_modules/@vitest/snapshot": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-2.0.3.tgz", - "integrity": "sha512-6OyA6v65Oe3tTzoSuRPcU6kh9m+mPL1vQ2jDlPdn9IQoUxl8rXhBnfICNOC+vwxWY684Vt5UPgtcA2aPFBb6wg==", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-2.0.4.tgz", + "integrity": "sha512-or6Mzoz/pD7xTvuJMFYEtso1vJo1S5u6zBTinfl+7smGUhqybn6VjzCDMhmTyVOFWwkCMuNjmNNxnyXPgKDoPw==", "dev": true, "dependencies": { - "@vitest/pretty-format": "2.0.3", + "@vitest/pretty-format": "2.0.4", "magic-string": "^0.30.10", "pathe": "^1.1.2" }, @@ -7898,9 +7898,9 @@ } }, "node_modules/@vitest/spy": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-2.0.3.tgz", - "integrity": "sha512-sfqyAw/ypOXlaj4S+w8689qKM1OyPOqnonqOc9T91DsoHbfN5mU7FdifWWv3MtQFf0lEUstEwR9L/q/M390C+A==", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-2.0.4.tgz", + "integrity": "sha512-uTXU56TNoYrTohb+6CseP8IqNwlNdtPwEO0AWl+5j7NelS6x0xZZtP0bDWaLvOfUbaYwhhWp1guzXUxkC7mW7Q==", "dev": true, "dependencies": { "tinyspy": "^3.0.0" @@ -7910,12 +7910,12 @@ } }, "node_modules/@vitest/utils": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-2.0.3.tgz", - "integrity": "sha512-c/UdELMuHitQbbc/EVctlBaxoYAwQPQdSNwv7z/vHyBKy2edYZaFgptE27BRueZB7eW8po+cllotMNTDpL3HWg==", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-2.0.4.tgz", + "integrity": "sha512-Zc75QuuoJhOBnlo99ZVUkJIuq4Oj0zAkrQ2VzCqNCx6wAwViHEh5Fnp4fiJTE9rA+sAoXRf00Z9xGgfEzV6fzQ==", "dev": true, "dependencies": { - "@vitest/pretty-format": "2.0.3", + "@vitest/pretty-format": "2.0.4", "estree-walker": "^3.0.3", "loupe": "^3.1.1", "tinyrainbow": "^1.2.0" @@ -7926,7 +7926,7 @@ }, "node_modules/@xenova/transformers": { "version": "3.0.0-alpha.0", - "resolved": "git+ssh://git@github.com/xenova/transformers.js.git#96f19b062429ee4569ceeb4695aea90f0f456e63", + "resolved": "git+ssh://git@github.com/xenova/transformers.js.git#c6aeb4be1bc1cdfa72e9d050f77b97dc9c8af362", "dependencies": { "@huggingface/jinja": "^0.2.2", "onnxruntime-web": "^1.18.0", @@ -15062,9 +15062,9 @@ } }, "node_modules/msw/node_modules/type-fest": { - "version": "4.22.1", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.22.1.tgz", - "integrity": "sha512-9tHNEa0Ov81YOopiVkcCJVz5TM6AEQ+CHHjFIktqPnE3NV0AHIkx+gh9tiCl58m/66wWxkOC9eltpa75J4lQPA==", + "version": "4.23.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.23.0.tgz", + "integrity": "sha512-ZiBujro2ohr5+Z/hZWHESLz3g08BBdrdLMieYFULJO+tWc437sn8kQsWLJoZErY8alNhxre9K4p3GURAG11n+w==", "engines": { "node": ">=16" }, @@ -15452,9 +15452,9 @@ } }, "node_modules/node-releases": { - "version": "2.0.17", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.17.tgz", - "integrity": "sha512-Ww6ZlOiEQfPfXM45v17oabk77Z7mg5bOt7AjDyzy7RjK9OrLrLC8dyZQoAPEOtFX9SaNf1Tdvr5gRJWdTJj7GA==" + "version": "2.0.18", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.18.tgz", + "integrity": "sha512-d9VeXT4SJ7ZeOqGX6R5EM022wpL+eWPooLI+5UpWn2jCT1aosUQEhQP214x33Wkwx3JQMvIm+tIoVOdodFS40g==" }, "node_modules/nopt": { "version": "7.2.1", @@ -15877,9 +15877,9 @@ } }, "node_modules/onnxruntime-node/node_modules/tar": { - "version": "7.4.0", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.0.tgz", - "integrity": "sha512-XQs0S8fuAkQWuqhDeCdMlJXDX80D7EOVLDPVFkna9yQfzS+PHKgfxcei0jf6/+QAWcjqrnC8uM3fSAnrQl+XYg==", + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.1.tgz", + "integrity": "sha512-dDJzpQf7Nud96mCs3wtw+XUiWGpi9WHxytSusrg0lYlj/Kr11DnB5hfw5bNDQNzx52JJ2Vy+7l8AFivp6H7ETA==", "optional": true, "dependencies": { "@isaacs/fs-minipass": "^4.0.0", @@ -15916,9 +15916,9 @@ } }, "node_modules/openai": { - "version": "4.52.7", - "resolved": "https://registry.npmjs.org/openai/-/openai-4.52.7.tgz", - "integrity": "sha512-dgxA6UZHary6NXUHEDj5TWt8ogv0+ibH+b4pT5RrWMjiRZVylNwLcw/2ubDrX5n0oUmHX/ZgudMJeemxzOvz7A==", + "version": "4.53.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-4.53.0.tgz", + "integrity": "sha512-XoMaJsSLuedW5eoMEMmZbdNoXgML3ujcU5KfwRnC6rnbmZkHE2Q4J/SArwhqCxQRqJwHnQUj1LpiROmKPExZJA==", "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", @@ -16939,9 +16939,9 @@ } }, "node_modules/query-string": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/query-string/-/query-string-9.0.0.tgz", - "integrity": "sha512-4EWwcRGsO2H+yzq6ddHcVqkCQ2EFUSfDMEjF8ryp8ReymyZhIuaFRGLomeOQLkrzacMHoyky2HW0Qe30UbzkKw==", + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/query-string/-/query-string-9.1.0.tgz", + "integrity": "sha512-t6dqMECpCkqfyv2FfwVS1xcB6lgXW/0XZSaKdsCNGYkqMO76AFiJEg4vINzoDKcZa6MS7JX+OHIjwh06K5vczw==", "dependencies": { "decode-uri-component": "^0.4.1", "filter-obj": "^5.1.0", @@ -19852,9 +19852,9 @@ } }, "node_modules/vite-node": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-2.0.3.tgz", - "integrity": "sha512-14jzwMx7XTcMB+9BhGQyoEAmSl0eOr3nrnn+Z12WNERtOvLN+d2scbRUvyni05rT3997Bg+rZb47NyP4IQPKXg==", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-2.0.4.tgz", + "integrity": "sha512-ZpJVkxcakYtig5iakNeL7N3trufe3M6vGuzYAr4GsbCTwobDeyPJpE4cjDhhPluv8OvQCFzu2LWp6GkoKRITXA==", "dev": true, "dependencies": { "cac": "^6.7.14", @@ -19888,18 +19888,18 @@ } }, "node_modules/vitest": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-2.0.3.tgz", - "integrity": "sha512-o3HRvU93q6qZK4rI2JrhKyZMMuxg/JRt30E6qeQs6ueaiz5hr1cPj+Sk2kATgQzMMqsa2DiNI0TIK++1ULx8Jw==", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-2.0.4.tgz", + "integrity": "sha512-luNLDpfsnxw5QSW4bISPe6tkxVvv5wn2BBs/PuDRkhXZ319doZyLOBr1sjfB5yCEpTiU7xCAdViM8TNVGPwoog==", "dev": true, "dependencies": { "@ampproject/remapping": "^2.3.0", - "@vitest/expect": "2.0.3", - "@vitest/pretty-format": "^2.0.3", - "@vitest/runner": "2.0.3", - "@vitest/snapshot": "2.0.3", - "@vitest/spy": "2.0.3", - "@vitest/utils": "2.0.3", + "@vitest/expect": "2.0.4", + "@vitest/pretty-format": "^2.0.4", + "@vitest/runner": "2.0.4", + "@vitest/snapshot": "2.0.4", + "@vitest/spy": "2.0.4", + "@vitest/utils": "2.0.4", "chai": "^5.1.1", "debug": "^4.3.5", "execa": "^8.0.1", @@ -19910,8 +19910,8 @@ "tinypool": "^1.0.0", "tinyrainbow": "^1.2.0", "vite": "^5.0.0", - "vite-node": "2.0.3", - "why-is-node-running": "^2.2.2" + "vite-node": "2.0.4", + "why-is-node-running": "^2.3.0" }, "bin": { "vitest": "vitest.mjs" @@ -19925,8 +19925,8 @@ "peerDependencies": { "@edge-runtime/vm": "*", "@types/node": "^18.0.0 || >=20.0.0", - "@vitest/browser": "2.0.3", - "@vitest/ui": "2.0.3", + "@vitest/browser": "2.0.4", + "@vitest/ui": "2.0.4", "happy-dom": "*", "jsdom": "*" }, diff --git a/package.json b/package.json index 3db6581f..d4ab2d19 100644 --- a/package.json +++ b/package.json @@ -37,9 +37,9 @@ "dependencies": { "@aitube/broadway": "0.0.22", "@aitube/clap": "0.0.30", - "@aitube/clapper-services": "0.0.29", + "@aitube/clapper-services": "0.0.34", "@aitube/engine": "0.0.26", - "@aitube/timeline": "0.0.43", + "@aitube/timeline": "0.0.44", "@fal-ai/serverless-client": "^0.13.0", "@ffmpeg/ffmpeg": "^0.12.10", "@ffmpeg/util": "^0.12.1", diff --git a/public/images/onboarding/get-started.png b/public/images/onboarding/get-started.png new file mode 100644 index 00000000..4b06e587 --- /dev/null +++ b/public/images/onboarding/get-started.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:208b7aa03f52a824308fff17c9ec5d854d28ba051bba46ac2b886c7fffe3e688 +size 24468 diff --git a/public/images/onboarding/get-started.xcf b/public/images/onboarding/get-started.xcf new file mode 100644 index 00000000..ea6a421d --- /dev/null +++ b/public/images/onboarding/get-started.xcf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:864be2d20925acf9ff343ac45c3d877c656eb2b474f335316526181a204a82be +size 70110 diff --git a/public/images/onboarding/pick-an-example.png b/public/images/onboarding/pick-an-example.png new file mode 100644 index 00000000..6029a554 --- /dev/null +++ b/public/images/onboarding/pick-an-example.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84bda0aa814064c9768a737ca07fe58568129240292b4f1783ef6bb08495b267 +size 30473 diff --git a/public/images/onboarding/pick-an-example.xcf b/public/images/onboarding/pick-an-example.xcf new file mode 100644 index 00000000..f5faa6b3 --- /dev/null +++ b/public/images/onboarding/pick-an-example.xcf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29d46c5ed0ae1b1a4d9d11cf1103baf52b50ce204221d3667b7f481394ed75ba +size 107646 diff --git a/src/app/main.tsx b/src/app/main.tsx index 9a90dcc9..9488ecec 100644 --- a/src/app/main.tsx +++ b/src/app/main.tsx @@ -5,7 +5,6 @@ import { ReflexContainer, ReflexSplitter, ReflexElement } from 'react-reflex' import { useSearchParams } from 'next/navigation' import { DndProvider, useDrop } from 'react-dnd' import { HTML5Backend, NativeTypes } from 'react-dnd-html5-backend' -import { useTimeline } from '@aitube/timeline' import { Toaster } from '@/components/ui/sonner' import { cn } from '@/lib/utils' @@ -14,22 +13,23 @@ import { Monitor } from '@/components/monitor' import { SettingsDialog } from '@/components/settings' import { LoadingDialog } from '@/components/dialogs/loader/LoadingDialog' -import { useUI } from '@/services/ui' +import { useUI, useIO } from '@/services' import { TopBar } from '@/components/toolbars/top-bar' import { Timeline } from '@/components/core/timeline' -import { useIO } from '@/services/io/useIO' import { ChatView } from '@/components/assistant/ChatView' import { Editors } from '@/components/editors/Editors' +import { useTheme } from '@/services/ui/useTheme' type DroppableThing = { files: File[] } function MainContent() { const ref = useRef(null) - const isEmpty = useTimeline((s) => s.isEmpty) + const showWelcomeScreen = useUI((s) => s.showWelcomeScreen) const showTimeline = useUI((s) => s.showTimeline) const showAssistant = useUI((s) => s.showAssistant) - + const theme = useTheme() const openFiles = useIO((s) => s.openFiles) + const isTopMenuOpen = useUI((s) => s.isTopMenuOpen) const [{ isOver, canDrop }, connectFileDrop] = useDrop({ accept: [NativeTypes.FILE], @@ -67,8 +67,7 @@ function MainContent() {
@@ -109,6 +108,60 @@ function MainContent() {
+
+
+
+ +
+
+ +
+
+

+ Welcome to{' '} + + Clapper + + . +

+
+

A free and open-source AI video editor,

+

designed for the age of generative filmmaking.

+
+
+
+
+ diff --git a/src/components/toolbars/top-menu/file/index.tsx b/src/components/toolbars/top-menu/file/index.tsx index cf9a5c59..1cfc1711 100644 --- a/src/components/toolbars/top-menu/file/index.tsx +++ b/src/components/toolbars/top-menu/file/index.tsx @@ -17,6 +17,7 @@ import { import { useOpenFilePicker, useQueryStringParams } from '@/lib/hooks' import { IframeWarning } from '@/components/dialogs/iframe-warning' import { useIO, useUI } from '@/services' +import { newClap } from '@aitube/clap' export function TopMenuFile() { const { clapUrl } = useQueryStringParams({ @@ -44,6 +45,9 @@ export function TopMenuFile() { const hasBetaAccess = useUI((s) => s.hasBetaAccess) + const showWelcomeScreen = useUI((s) => s.showWelcomeScreen) + const setShowWelcomeScreen = useUI((s) => s.setShowWelcomeScreen) + useEffect(() => { ;(async () => { if (!clapUrl) { @@ -67,21 +71,21 @@ export function TopMenuFile() { File - {hasBetaAccess && ( - { - openClapUrl('/samples/claps/empty_project.clap') - }} - > - New Project⌘N - - )} + { + setClap(newClap()) + setShowWelcomeScreen(false) + }} + > + New Project⌘N + + { openFilePicker() }} > - Open file (.clap, .txt)⌘O + Open project (.clap)⌘O { @@ -92,7 +96,7 @@ export function TopMenuFile() { - Examples + Import an example { @@ -157,20 +161,46 @@ export function TopMenuFile() { { - saveVideoFile() + openFilePicker() + }} + > + Import screenplay (.txt) + + { + openFilePicker() + }} + > + Import video (.mp4) + + {/* + In case we want to show a video import wizard UI: + + { + openFilePicker() }} > - Export project to MP4 + Import video (.mp4) + */} + + + { + saveVideoFile() + }} + > + Export full video (.mp4) + { saveZipFile() }} > - Export project to .zip + Export all assets (.zip) - {/* { saveKdenline() diff --git a/src/components/toolbars/top-menu/index.tsx b/src/components/toolbars/top-menu/index.tsx index b54642b1..a3eacfdf 100644 --- a/src/components/toolbars/top-menu/index.tsx +++ b/src/components/toolbars/top-menu/index.tsx @@ -19,10 +19,17 @@ import { TopMenuPlugins } from './plugins' export function TopMenu() { const isBusyResolving = useResolver((s) => s.isBusyResolving) + const setIsTopMenuOpen = useUI((s) => s.setIsTopMenuOpen) + const hasBetaAccess = useUI((s) => s.hasBetaAccess) return ( - + { + setIsTopMenuOpen(!!value) + }} + > {hasBetaAccess && } diff --git a/src/lib/core/constants.ts b/src/lib/core/constants.ts index 8942edfc..89e86475 100644 --- a/src/lib/core/constants.ts +++ b/src/lib/core/constants.ts @@ -3,7 +3,7 @@ export const HARD_LIMIT_NB_MAX_ASSETS_TO_GENERATE_IN_PARALLEL = 32 export const APP_NAME = 'Clapper.app' -export const APP_REVISION = 'r20240722-0205' +export const APP_REVISION = 'r20240722-2258' export const APP_DOMAIN = 'Clapper.app' export const APP_LINK = 'https://clapper.app' diff --git a/src/lib/hooks/useOpenFilePicker.ts b/src/lib/hooks/useOpenFilePicker.ts index 1d5583d1..572655c8 100644 --- a/src/lib/hooks/useOpenFilePicker.ts +++ b/src/lib/hooks/useOpenFilePicker.ts @@ -4,15 +4,24 @@ import { useFilePicker } from 'use-file-picker' import { parseFileName } from '@/services/io/parseFileName' import { useIO } from '@/services/io/useIO' -const supportedExtensions = ['clap', 'txt'] +const defaultSupportedExtensions = ['clap', 'txt', 'mp4', 'mp3'] -export function useOpenFilePicker() { +export function useOpenFilePicker( + { + supportedExtensions = defaultSupportedExtensions, + }: { + supportedExtensions: string[] + } = { + supportedExtensions: defaultSupportedExtensions, + } +) { const [isLoading, setIsLoading] = useState(false) const openClapBlob = useIO((s) => s.openClapBlob) const openScreenplay = useIO((s) => s.openScreenplay) + const openVideo = useIO((s) => s.openVideo) const { openFilePicker, filesContent, loading } = useFilePicker({ - accept: ['clap', 'txt'].map((ext) => `.${ext}`), + accept: supportedExtensions.map((ext) => `.${ext}`), readAs: 'ArrayBuffer', }) @@ -27,7 +36,7 @@ export function useOpenFilePicker() { const { fileName, projectName, extension } = parseFileName(input) - if (!supportedExtensions.includes(extension)) { + if (!defaultSupportedExtensions.includes(extension)) { console.error(`unsupported extension "${extension}"`) return } @@ -52,10 +61,27 @@ export function useOpenFilePicker() { } finally { setIsLoading(false) } + } else if (extension === 'mp4') { + try { + setIsLoading(true) + await openVideo(projectName, fileName, blob) + } catch (err) { + console.error('failed to load the Clap file:', err) + } finally { + setIsLoading(false) + } + } else if (extension === 'mp3') { + alert('Initializing a project from a mp3 is not supported yet') } } fn() - }, [fileData?.name, fileData?.content, openClapBlob, openScreenplay]) + }, [ + fileData?.name, + fileData?.content, + openClapBlob, + openScreenplay, + openVideo, + ]) return { openFilePicker, diff --git a/src/lib/utils/base64DataUriToFile.ts b/src/lib/utils/base64DataUriToFile.ts index 184b9aca..34fdd0b0 100644 --- a/src/lib/utils/base64DataUriToFile.ts +++ b/src/lib/utils/base64DataUriToFile.ts @@ -1,5 +1,5 @@ export function base64DataUriToFile(dataUrl: string, fileName: string) { - var arr = dataUrl.split(',') + var arr = `${dataUrl || ''}`.split(',') const st = `${arr[0] || ''}` const mime = `${st.match(/:(.*?);/)?.[1] || ''}` const bstr = atob(arr[arr.length - 1]) diff --git a/src/services/io/extractFramesFromVideo.ts b/src/services/io/extractFramesFromVideo.ts index 528ae7db..3a556aeb 100644 --- a/src/services/io/extractFramesFromVideo.ts +++ b/src/services/io/extractFramesFromVideo.ts @@ -1,6 +1,7 @@ 'use client' import { FFmpeg } from '@ffmpeg/ffmpeg' +import { FileData } from '@ffmpeg/ffmpeg/dist/esm/types' import { toBlobURL } from '@ffmpeg/util' import mediaInfoFactory, { Track, @@ -12,6 +13,7 @@ import mediaInfoFactory, { MenuTrack, OtherTrack, } from 'mediainfo.js' +import { fileDataToBase64 } from './fileDataToBase64' interface FrameExtractorOptions { format: 'png' | 'jpg' @@ -20,6 +22,7 @@ interface FrameExtractorOptions { sceneSamplingRate: number // Percentage of additional frames between scene changes (0-100) onProgress?: (progress: number) => void // Callback function for progress updates debug?: boolean + autoCrop?: boolean // New option to enable automatic cropping } export async function extractFramesFromVideo( @@ -114,28 +117,57 @@ export async function extractFramesFromVideo( if (options.debug) { console.log('input.mp4 written!') } - // Prepare FFmpeg command - const sceneFilter = `select='gt(scene,0.4)'` - const additionalFramesFilter = `select='not(mod(n,${Math.floor(100 / options.sceneSamplingRate)}))'` - const scaleFilter = `scale='min(${options.maxWidth},iw)':min'(${options.maxHeight},ih)':force_original_aspect_ratio=decrease` - let lastProgress = 0 - ffmpeg.on('log', ({ message }) => { + let cropParams = '' + + if (options.autoCrop) { + // First pass: Detect crop parameters + const cropDetectCommand = [ + '-i', + 'input.mp4', + '-vf', + 'cropdetect=limit=0.1:round=2:reset=0', + '-f', + 'null', + '-t', + '10', // Analyze first 10 seconds + '-', + ] + if (options.debug) { - console.log('FFmpeg log:', message) + console.log( + 'Executing crop detection command:', + cropDetectCommand.join(' ') + ) } - const timeMatch = message.match(/time=(\d{2}):(\d{2}):(\d{2}\.\d{2})/) - if (timeMatch) { - const [, hours, minutes, seconds] = timeMatch - const currentTime = - parseInt(hours) * 3600 + parseInt(minutes) * 60 + parseFloat(seconds) - const progress = Math.min(100, Math.round((currentTime / duration) * 100)) - if (progress > lastProgress) { - lastProgress = progress - options.onProgress?.(progress) + + ffmpeg.on('log', ({ message }) => { + const cropMatch = message.match(/crop=(\d+:\d+:\d+:\d+)/) + if (cropMatch) { + cropParams = cropMatch[1] } + }) + + await ffmpeg.exec(cropDetectCommand) + + if (options.debug) { + console.log('Detected crop parameters:', cropParams) } - }) + + if (!cropParams) { + console.warn('No crop parameters detected. Proceeding without cropping.') + } + } + + // Main processing command + const sceneFilter = `select='gt(scene,0.2)'` + const additionalFramesFilter = `select='not(mod(n,${Math.floor(100 / options.sceneSamplingRate)}))'` + const scaleFilter = `scale='min(${options.maxWidth},iw)':min'(${options.maxHeight},ih)':force_original_aspect_ratio=decrease` + + let filterChain = `${sceneFilter},${additionalFramesFilter},${scaleFilter}` + if (options.autoCrop && cropParams) { + filterChain = `crop=${cropParams},${filterChain}` + } const ffmpegCommand = [ '-i', @@ -143,7 +175,7 @@ export async function extractFramesFromVideo( '-loglevel', 'verbose', '-vf', - `${sceneFilter},${additionalFramesFilter},${scaleFilter}`, + filterChain, '-vsync', '2', '-q:v', @@ -156,9 +188,30 @@ export async function extractFramesFromVideo( ] if (options.debug) { - console.log('Executing FFmpeg command:', ffmpegCommand.join(' ')) + console.log('Executing main FFmpeg command:', ffmpegCommand.join(' ')) } + let lastProgress = 0 + ffmpeg.on('log', ({ message }) => { + if (options.debug) { + console.log('FFmpeg log:', message) + } + const timeMatch = message.match(/time=(\d{2}):(\d{2}):(\d{2}\.\d{2})/) + if (timeMatch) { + const [, hours, minutes, seconds] = timeMatch + const currentTime = + parseInt(hours) * 3600 + parseInt(minutes) * 60 + parseFloat(seconds) + const progress = Math.min(100, Math.round((currentTime / duration) * 100)) + if (progress > lastProgress) { + lastProgress = progress + options.onProgress?.(progress) + } + } + }) + + if (options.debug) { + console.log('Executing FFmpeg command:', ffmpegCommand.join(' ')) + } try { await ffmpeg.exec(ffmpegCommand) } catch (error) { @@ -189,16 +242,9 @@ export async function extractFramesFromVideo( console.log(`Processing frame file: ${file.name}`) } try { - const frameData = await ffmpeg.readFile(file.name) - - // Convert Uint8Array to Base64 string without using btoa - let binary = '' - const bytes = new Uint8Array(frameData as any) - const len = bytes.byteLength - for (let i = 0; i < len; i++) { - binary += String.fromCharCode(bytes[i]) - } - const base64Frame = window.btoa(binary) + const frameData: FileData = await ffmpeg.readFile(file.name) + + const base64Frame = fileDataToBase64(frameData) frames.push(`data:image/${options.format};base64,${base64Frame}`) diff --git a/src/services/io/extractScenesFromVideo.ts b/src/services/io/extractScenesFromVideo.ts new file mode 100644 index 00000000..741a06de --- /dev/null +++ b/src/services/io/extractScenesFromVideo.ts @@ -0,0 +1,369 @@ +'use client' + +import { FFmpeg } from '@ffmpeg/ffmpeg' +import { toBlobURL } from '@ffmpeg/util' +import mediaInfoFactory, { VideoTrack, AudioTrack } from 'mediainfo.js' +import { fileDataToBase64 } from './fileDataToBase64' + +interface ExtractorOptions { + frameFormat: 'png' | 'jpg' + maxWidth: number + maxHeight: number + framesPerScene: number + onProgress?: (progress: number) => void + debug?: boolean + autoCrop?: boolean + sceneThreshold?: number + minSceneDuration?: number +} + +interface SceneData { + sceneIndex: number + startTimeInMs: number + endTimeInMs: number + video: string + frames: string[] +} + +export async function extractScenesFromVideo( + videoBlob: Blob, + options: ExtractorOptions +): Promise { + const ffmpeg = new FFmpeg() + const baseURL = 'https://unpkg.com/@ffmpeg/core@0.12.6/dist/umd' + + try { + console.log(`getting duration..`) + + const duration = await getVideoDuration(videoBlob) + if (!duration) { + throw new Error(`couldn't get the video duration`) + } + if (options.debug) { + console.log('Video duration in seconds:', duration) + } + + console.log(`loading FFmpeg..`) + + await ffmpeg.load({ + coreURL: await toBlobURL(`${baseURL}/ffmpeg-core.js`, 'text/javascript'), + wasmURL: await toBlobURL( + `${baseURL}/ffmpeg-core.wasm`, + 'application/wasm' + ), + }) + + if (options.debug) { + console.log('FFmpeg loaded') + } + + const videoUint8Array = new Uint8Array(await videoBlob.arrayBuffer()) + await ffmpeg.writeFile('input.mp4', videoUint8Array) + + console.log(`detecting crop parameters..`) + + let cropParams = '' + if (options.autoCrop) { + cropParams = await detectCropParameters(ffmpeg, options) + } + + const sceneThreshold = options.sceneThreshold || 0.2 + const minSceneDuration = options.minSceneDuration || 1 + + const sceneDetectionFilter = `select='gt(scene,${sceneThreshold})'` + const scaleFilter = `scale='min(${options.maxWidth},iw)':min'(${options.maxHeight},ih)':force_original_aspect_ratio=decrease` + + let filterChain = `${sceneDetectionFilter},${scaleFilter}` + if (cropParams) { + filterChain = `crop=${cropParams},${filterChain}` + } + console.log(`detecting scenes..`) + + const sceneTimestamps = await detectScenes( + ffmpeg, + filterChain, + options, + duration + ) + + console.log(`detected ${sceneTimestamps.length} scenes`) + + const scenes: SceneData[] = [] + + for (let i = 0; i < sceneTimestamps.length; i++) { + const startTime = sceneTimestamps[i] + const endTime = + i < sceneTimestamps.length - 1 + ? sceneTimestamps[i + 1] + : duration * 1000 + const sceneDuration = endTime - startTime + console.log(`processing scene ${i}`) + + try { + const sceneData = await processScene( + ffmpeg, + i, + startTime, + endTime, + sceneDuration, + options + ) + scenes.push(sceneData) + } catch (error) { + console.error(`Error processing scene ${i}:`, error) + } + + options.onProgress?.(Math.round(((i + 1) / sceneTimestamps.length) * 100)) + } + + if (options.debug) { + console.log(`Total scenes processed: ${scenes.length}`) + } + + return scenes + } catch (error) { + console.error('Error in extractFramesAndScenesFromVideo:', error) + throw error + } finally { + try { + await ffmpeg.terminate() + } catch (error) { + console.error('Error terminating FFmpeg:', error) + } + } +} + +async function getVideoDuration( + videoBlob: Blob, + debug: boolean = false +): Promise { + // Initialize MediaInfo + const mediaInfo = await mediaInfoFactory({ + format: 'object', + locateFile: () => { + return '/wasm/MediaInfoModule.wasm' + }, + }) + + // Get video duration using MediaInfo + const getSize = () => videoBlob.size + const readChunk = (chunkSize: number, offset: number) => + new Promise((resolve, reject) => { + const reader = new FileReader() + reader.onload = (event) => { + if (event.target?.result instanceof ArrayBuffer) { + resolve(new Uint8Array(event.target.result)) + } else { + reject(new Error('Failed to read chunk')) + } + } + reader.onerror = (error) => reject(error) + reader.readAsArrayBuffer(videoBlob.slice(offset, offset + chunkSize)) + }) + + if (debug) { + console.log('calling await mediaInfo.analyzeData(getSize, readChunk)') + } + + const result = await mediaInfo.analyzeData(getSize, readChunk) + if (debug) { + console.log('result = ', result) + } + + let duration: number = 0 + + for (const track of result.media?.track || []) { + if (debug) { + console.log('track = ', track) + } + + let maybeDuration: number = 0 + if (track['@type'] === 'Audio') { + const audioTrack = track as AudioTrack + maybeDuration = audioTrack.Duration + ? parseFloat(`${audioTrack.Duration || 0}`) + : 0 + } else if (track['@type'] === 'Video') { + const videoTrack = track as VideoTrack + maybeDuration = videoTrack.Duration + ? parseFloat(`${videoTrack.Duration || 0}`) + : 0 + } + if ( + typeof maybeDuration === 'number' && + isFinite(maybeDuration) && + !isNaN(maybeDuration) + ) { + duration = maybeDuration + } + } + return duration +} + +async function detectCropParameters( + ffmpeg: FFmpeg, + options: ExtractorOptions +): Promise { + const cropDetectCommand = [ + '-i', + 'input.mp4', + '-vf', + 'cropdetect=limit=0.1:round=2:reset=0', + '-f', + 'null', + '-t', + '10', + '-', + ] + + if (options.debug) { + console.log( + 'Executing crop detection command:', + cropDetectCommand.join(' ') + ) + } + + let cropParams = '' + ffmpeg.on('log', ({ message }) => { + const cropMatch = message.match(/crop=(\d+:\d+:\d+:\d+)/) + if (cropMatch) { + cropParams = cropMatch[1] + } + }) + + await ffmpeg.exec(cropDetectCommand) + + if (options.debug) { + console.log('Detected crop parameters:', cropParams) + } + + return cropParams +} + +async function detectScenes( + ffmpeg: FFmpeg, + filterChain: string, + options: ExtractorOptions, + duration: number +): Promise { + const extractScenesCommand = [ + '-i', + 'input.mp4', + '-filter_complex', + `${filterChain},metadata=print:file=scenes.txt`, + '-f', + 'null', + '-', + ] + + if (options.debug) { + console.log( + 'Executing scene detection command:', + extractScenesCommand.join(' ') + ) + } + + await ffmpeg.exec(extractScenesCommand) + + const scenesMetadata = await ffmpeg.readFile('scenes.txt') + const decodedMetadata = new TextDecoder().decode(scenesMetadata as Uint8Array) + + if (options.debug) { + console.log('Scenes metadata:', decodedMetadata) + } + + const sceneTimestamps = decodedMetadata + .split('\n') + .filter((line) => line.includes('pts_time')) + .map((line) => parseFloat(line.split('pts_time:')[1]) * 1000) // Convert to milliseconds + + // Add start and end timestamps + sceneTimestamps.unshift(0) + sceneTimestamps.push(duration * 1000) + + // Filter out scenes that are too short + const filteredScenes = sceneTimestamps.filter((timestamp, index, array) => { + if (index === 0) return true + const sceneDuration = timestamp - array[index - 1] + return sceneDuration >= (options.minSceneDuration || 1) * 1000 + }) + + return filteredScenes +} + +async function processScene( + ffmpeg: FFmpeg, + index: number, + startTime: number, + endTime: number, + duration: number, + options: ExtractorOptions +): Promise { + const extractSceneCommand = [ + '-ss', + (startTime / 1000).toString(), + '-i', + 'input.mp4', + '-t', + (duration / 1000).toString(), + '-c:v', + 'libx264', + '-preset', + 'ultrafast', + '-crf', + '23', + '-c:a', + 'aac', + `scene_${index}.mp4`, + ] + // console.log(`calling ffmpeg.exec(extractSceneCommand)`, extractSceneCommand) + await ffmpeg.exec(extractSceneCommand) + + // Calculate frame interval to get the desired number of frames + const frameInterval = Math.max( + 1, + Math.floor(duration / (1000 * options.framesPerScene)) + ) + + const extractFramesCommand = [ + '-i', + `scene_${index}.mp4`, + '-vf', + `select='not(mod(n,${frameInterval}))',setpts=N/FRAME_RATE/TB`, + '-frames:v', + options.framesPerScene.toString(), + '-vsync', + '0', + '-q:v', + '2', + '-f', + 'image2', + `scene_${index}_frame_%03d.${options.frameFormat}`, + ] + // console.log(`calling ffmpeg.exec(extractFramesCommand)`, extractFramesCommand) + await ffmpeg.exec(extractFramesCommand) + + const sceneVideo = await ffmpeg.readFile(`scene_${index}.mp4`) + const frameFiles = (await ffmpeg.listDir('/')).filter( + (file) => + file.name.startsWith(`scene_${index}_frame_`) && + file.name.endsWith(`.${options.frameFormat}`) + ) + + const frames: string[] = [] + for (const frameFile of frameFiles) { + const frameData = await ffmpeg.readFile(frameFile.name) + const base64Frame = fileDataToBase64(frameData) + frames.push(`data:image/${options.frameFormat};base64,${base64Frame}`) + } + + const base64Video = fileDataToBase64(sceneVideo) + + return { + sceneIndex: index, + startTimeInMs: Math.round(startTime), + endTimeInMs: Math.round(endTime), + video: `data:video/mp4;base64,${base64Video}`, + frames, + } +} diff --git a/src/services/io/fileDataToBase64.ts b/src/services/io/fileDataToBase64.ts new file mode 100644 index 00000000..96ab046e --- /dev/null +++ b/src/services/io/fileDataToBase64.ts @@ -0,0 +1,13 @@ +import { FileData } from '@ffmpeg/ffmpeg/dist/esm/types' + +export function fileDataToBase64(fileData: FileData): string { + // Convert Uint8Array to Base64 string without using btoa + let binary = '' + const bytes = new Uint8Array(fileData as any) + const len = bytes.byteLength + for (let i = 0; i < len; i++) { + binary += String.fromCharCode(bytes[i]) + } + + return window.btoa(binary) +} diff --git a/src/services/io/parseFileIntoSegments.ts b/src/services/io/parseFileIntoSegments.ts index 15519349..a52eca5e 100644 --- a/src/services/io/parseFileIntoSegments.ts +++ b/src/services/io/parseFileIntoSegments.ts @@ -81,7 +81,41 @@ export async function parseFileIntoSegments({ ? maybeEndTimeInMs! : startTimeInMs + durationInMs - const newSegmentData: Partial = { + const partialVideo: Partial = { + category: ClapSegmentCategory.VIDEO, + startTimeInMs, + endTimeInMs, + + prompt: 'movie', + label: 'movie', // `${file.name.split(".")[0] || "Untitled"}`, // a short label to name the segment (optional, can be human or LLM-defined) + + outputType: ClapOutputType.VIDEO, + status: ClapSegmentStatus.TO_GENERATE, + + assetUrl: '', + assetDurationInMs: durationInMs, + assetSourceType: ClapAssetSource.EMPTY, + assetFileFormat: undefined, + track: track ? track : undefined, + } + + const video = await clapSegmentToTimelineSegment(newSegment(partialVideo)) + + if (isValidNumber(track)) { + video.track = track + } + + video.outputType = ClapOutputType.VIDEO + + // we assume we want it to be immediately visible + video.visibility = SegmentVisibility.VISIBLE + + // console.log("newSegment:", audioSegment) + + // poof! type disappears.. it's magic + newSegments.push(video) + + const partialStoryboard: Partial = { prompt: 'Storyboard', // note: this can be set later with an automatic captioning worker startTimeInMs, // start time of the segment endTimeInMs, // end time of the segment (startTimeInMs + durationInMs) @@ -90,28 +124,31 @@ export async function parseFileIntoSegments({ label: `${file.name}`, // a short label to name the segment (optional, can be human or LLM-defined) category, assetUrl, - assetDurationInMs: endTimeInMs, + assetDurationInMs: durationInMs, assetSourceType: ClapAssetSource.DATA, assetFileFormat: `${file.type}`, + + // important: we try to go below + track: track ? track + 1 : undefined, } - const timelineSegment = await clapSegmentToTimelineSegment( - newSegment(newSegmentData) + const storyboard = await clapSegmentToTimelineSegment( + newSegment(partialStoryboard) ) if (isValidNumber(track)) { - timelineSegment.track = track + storyboard.track = track } - timelineSegment.outputType = ClapOutputType.IMAGE + storyboard.outputType = ClapOutputType.IMAGE // we assume we want it to be immediately visible - timelineSegment.visibility = SegmentVisibility.VISIBLE + storyboard.visibility = SegmentVisibility.VISIBLE // console.log("newSegment:", audioSegment) // poof! type disappears.. it's magic - newSegments.push(timelineSegment) + newSegments.push(storyboard) break } diff --git a/src/services/io/useIO.ts b/src/services/io/useIO.ts index e7819b55..469bf872 100644 --- a/src/services/io/useIO.ts +++ b/src/services/io/useIO.ts @@ -3,7 +3,10 @@ import { ClapAssetSource, ClapEntity, + ClapMediaOrientation, + ClapOutputType, ClapProject, + ClapSegment, ClapSegmentCategory, ClapSegmentStatus, getClapAssetSourceType, @@ -11,6 +14,7 @@ import { newSegment, parseClap, serializeClap, + UUID, } from '@aitube/clap' import { TimelineStore, @@ -19,6 +23,7 @@ import { removeFinalVideosAndConvertToTimelineSegments, getFinalVideo, DEFAULT_DURATION_IN_MS_PER_STEP, + clapSegmentToTimelineSegment, } from '@aitube/timeline' import { ParseScriptProgressUpdate, parseScriptToClap } from '@aitube/broadway' import { IOStore, TaskCategory, TaskVisibility } from '@aitube/clapper-services' @@ -43,9 +48,11 @@ import { import { sleep } from '@/lib/utils/sleep' import { FFMPegAudioInput, FFMPegVideoInput } from './ffmpegUtils' import { createFullVideo } from './createFullVideo' -import { extractFramesFromVideo } from './extractFramesFromVideo' +import { extractScenesFromVideo } from './extractScenesFromVideo' import { extractCaptionsFromFrames } from './extractCaptionsFromFrames' import { base64DataUriToFile } from '@/lib/utils/base64DataUriToFile' +import { useUI } from '../ui' +import { getTypeAndExtension } from '@/lib/utils/getTypeAndExtension' export const useIO = create((set, get) => ({ ...getDefaultIOState(), @@ -59,7 +66,7 @@ export const useIO = create((set, get) => ({ timeline.clear() }, openFiles: async (files: File[]) => { - const { openClapBlob, openScreenplay } = get() + const { openClapBlob, openScreenplay, openVideo } = get() const timeline: TimelineStore = useTimeline.getState() const { segments, addSegments } = timeline @@ -101,101 +108,166 @@ export const useIO = create((set, get) => ({ const newSegments = await parseFileIntoSegments({ file }) console.log('calling timeline.addSegments with:', newSegments) - await timeline.addSegments({ - segments: newSegments, - }) + await timeline.addSegments({ segments: newSegments }) + return } const isVideoFile = fileType.startsWith('video/') if (isVideoFile) { - const storyboardExtractionTask = useTasks.getState().add({ - category: TaskCategory.IMPORT, - visibility: TaskVisibility.BLOCKER, - initialMessage: `Extracting storyboards..`, - successMessage: `Extracting storyboards.. 100% done`, - value: 0, - }) + await openVideo(projectName, fileName, file) + return + } + } + useUI.getState().setShowWelcomeScreen(false) + }, + openVideo: async ( + projectName: string, + fileName: string, + fileContent: string | Blob + ): Promise => { + const timeline: TimelineStore = useTimeline.getState() - const frames = await extractFramesFromVideo(file, { - format: 'png', // in theory we could also use 'jpg', but this freezes FFmpeg - maxWidth: 1024, - maxHeight: 576, - sceneSamplingRate: 100, - onProgress: (progress: number) => { - storyboardExtractionTask.setProgress({ - message: `Extracting storyboards.. ${progress}% done`, - value: progress, - }) - }, + const sceneExtractionTask = useTasks.getState().add({ + category: TaskCategory.IMPORT, + visibility: TaskVisibility.BLOCKER, + initialMessage: `Starting up, can take a few minutes..`, + successMessage: `Extracting scenes.. 100%`, + value: 0, + }) + + const file = + typeof fileContent === 'string' + ? base64DataUriToFile(fileContent, fileName) + : fileContent + + const scenes = await extractScenesFromVideo(file, { + frameFormat: 'png', // in theory we could also use 'jpg', but this freezes FFmpeg + maxWidth: 1024, + maxHeight: 576, + framesPerScene: 1, + autoCrop: true, + sceneThreshold: 0.1, + minSceneDuration: 1, + debug: true, + onProgress: (progress: number) => { + sceneExtractionTask.setProgress({ + message: `Extracting scenes.. ${progress}%`, + value: progress, }) + }, + }) - // optional: reset the project - // await timeline.setClap(newClap()) - - const track = 1 - let i = 0 - let startTimeInMs = 0 - const durationInSteps = 4 - const durationInMs = durationInSteps * DEFAULT_DURATION_IN_MS_PER_STEP - let endTimeInMs = startTimeInMs + durationInMs - - for (const frame of frames) { - const frameFile = base64DataUriToFile(frame, `storyboard_${i++}.png`) - const newSegments = await parseFileIntoSegments({ - file: frameFile, - startTimeInMs, - endTimeInMs, - track, - }) + // optional: reset the project + // await timeline.setClap(newClap()) + + let currentStoryboardIndex = 0 + let startTimeInMs = 0 + const durationInSteps = 4 + const durationInMs = durationInSteps * DEFAULT_DURATION_IN_MS_PER_STEP + let endTimeInMs = startTimeInMs + durationInMs + + // TODO: extract info from the original video to determine things like + // the orientation, duration.. + timeline.setClap( + newClap({ + meta: { + id: UUID(), + title: projectName, + description: `${projectName} (${fileName})`, + synopsis: '', + licence: + "This OpenClap file is just a conversion from the original screenplay and doesn't claim any copyright or intellectual property. All rights reserved to the original intellectual property and copyright holders. Using OpenClap isn't piracy.", + + orientation: ClapMediaOrientation.LANDSCAPE, + durationInMs: frames.length * durationInMs, + + // TODO: those should come from the Clapper user settings + + width: 1024, + height: 576, + + defaultVideoModel: '', // <-- we should deprecate this no? + extraPositivePrompt: '', + screenplay: '', + isLoop: false, + isInteractive: false, + }, + }) + ) - startTimeInMs += durationInMs - endTimeInMs += durationInMs + for (const scene of scenes) { + console.log('parsing scene:', scene) + try { + const frameFile = base64DataUriToFile( + scene.frames[0], + `storyboard_${++currentStoryboardIndex}.png` + ) - console.log('calling timeline.addSegments with:', newSegments) - await timeline.addSegments({ - segments: newSegments, - track, - }) - } + const assetDurationInMs = scene.endTimeInMs - scene.startTimeInMs + + // this returns multiple segments (video, image..) + const newSegments = await parseFileIntoSegments({ + file: frameFile, + startTimeInMs: scene.startTimeInMs, + endTimeInMs: scene.endTimeInMs, + }) - storyboardExtractionTask.success() + for (const newSegment of newSegments) { + newSegment.assetDurationInMs = assetDurationInMs + if (newSegment.category === ClapSegmentCategory.VIDEO) { + const { assetFileFormat, outputType } = getTypeAndExtension( + scene.video + ) + newSegment.assetFileFormat = assetFileFormat + newSegment.assetUrl = scene.video + newSegment.status = ClapSegmentStatus.COMPLETED + newSegment.outputType = outputType + } + } + await timeline.addSegments({ segments: newSegments }) + } catch (err) { + console.error(`failed to process scene:`, scene) + console.error(err) + } + } - const enableCaptioning = false + sceneExtractionTask.success() - if (enableCaptioning) { - const captioningTask = useTasks.getState().add({ - category: TaskCategory.IMPORT, - // visibility: TaskVisibility.BLOCKER, + const enableCaptioning = false - // since this is very long task, we can run it in the background - visibility: TaskVisibility.BACKGROUND, - initialMessage: `Analyzing storyboards..`, - successMessage: `Analyzing storyboards.. 100% done`, - value: 0, - }) + if (enableCaptioning) { + const captioningTask = useTasks.getState().add({ + category: TaskCategory.IMPORT, + // visibility: TaskVisibility.BLOCKER, - console.log('calling extractCaptionsFromFrames() with:', frames) - const captions = await extractCaptionsFromFrames( - frames, - ( - progress: number, - storyboardIndex: number, - nbStoryboards: number - ) => { - captioningTask.setProgress({ - message: `Analyzing storyboards (${progress}%)`, - value: progress, - }) - } - ) - console.log('captions:', captions) - // TODO: add + // since this is very long task, we can run it in the background + visibility: TaskVisibility.BACKGROUND, + initialMessage: `Analyzing storyboards..`, + successMessage: `Analyzing storyboards.. 100% done`, + value: 0, + }) - captioningTask.success() + console.log('calling extractCaptionsFromFrames() with:', frames) + /* + const captions = await extractCaptionsFromFrames( + frames, + (progress: number, storyboardIndex: number, nbStoryboards: number) => { + captioningTask.setProgress({ + message: `Analyzing storyboards (${progress}%)`, + value: progress, + }) } - } + ) + + console.log('captions:', captions) + */ + // TODO: add + + captioningTask.success() } + + useUI.getState().setShowWelcomeScreen(false) }, openScreenplay: async ( projectName: string, @@ -270,6 +342,7 @@ export const useIO = create((set, get) => ({ task.fail(`${err || 'unknown screenplay import error'}`) } finally { } + useUI.getState().setShowWelcomeScreen(false) }, openScreenplayUrl: async (url: string) => { const timeline: TimelineStore = useTimeline.getState() @@ -326,6 +399,7 @@ export const useIO = create((set, get) => ({ } catch (err) { task.fail(`${err || 'unknown error'}`) } + useUI.getState().setShowWelcomeScreen(false) }, saveAnyFile: (blob: Blob, fileName: string) => { // Create an object URL for the compressed clap blob @@ -391,6 +465,7 @@ export const useIO = create((set, get) => ({ } catch (err) { task.fail(`${err || 'unknown error'}`) } + useUI.getState().setShowWelcomeScreen(false) }, openClapBlob: async (projectName: string, fileName: string, blob: Blob) => { const timeline: TimelineStore = useTimeline.getState() @@ -423,6 +498,7 @@ export const useIO = create((set, get) => ({ } catch (err) { task.fail(`${err || 'unknown error'}`) } + useUI.getState().setShowWelcomeScreen(false) }, saveClap: async () => { const { saveAnyFile } = get() @@ -726,7 +802,9 @@ export const useIO = create((set, get) => ({ } }, - openMLT: async (file: File) => {}, + openMLT: async (file: File) => { + useUI.getState().setShowWelcomeScreen(false) + }, saveMLT: async () => {}, generateMLT: async (): Promise => { const timeline: TimelineStore = useTimeline.getState() @@ -1001,7 +1079,9 @@ export const useIO = create((set, get) => ({ ` }, - openKdenline: async (file: File) => {}, + openKdenline: async (file: File) => { + useUI.getState().setShowWelcomeScreen(false) + }, saveKdenline: async () => { const { saveAnyFile } = get() @@ -1062,7 +1142,9 @@ export const useIO = create((set, get) => ({ */ }, - openOpenTimelineIO: async (file: File) => {}, + openOpenTimelineIO: async (file: File) => { + useUI.getState().setShowWelcomeScreen(false) + }, saveOpenTimelineIO: async () => {}, @@ -1086,6 +1168,9 @@ export const useIO = create((set, get) => ({ } const { entities } = await parseClap(file) + + useUI.getState().setShowWelcomeScreen(false) + return entities }, })) diff --git a/src/services/resolver/useResolver.ts b/src/services/resolver/useResolver.ts index 8b003045..8b10e089 100644 --- a/src/services/resolver/useResolver.ts +++ b/src/services/resolver/useResolver.ts @@ -597,10 +597,12 @@ export const useResolver = create((set, get) => ({ ) as TimelineSegment if (newSegment.outputType === ClapOutputType.AUDIO) { - try { - newSegment.audioBuffer = await getAudioBuffer(newSegment.assetUrl) - } catch (err) { - console.error(`failed to load the audio file: ${err}`) + if (newSegment.assetUrl) { + try { + newSegment.audioBuffer = await getAudioBuffer(newSegment.assetUrl) + } catch (err) { + console.error(`failed to load the audio file: ${err}`) + } } } diff --git a/src/services/ui/getDefaultUIState.ts b/src/services/ui/getDefaultUIState.ts index fb382a45..64465f3a 100644 --- a/src/services/ui/getDefaultUIState.ts +++ b/src/services/ui/getDefaultUIState.ts @@ -6,6 +6,8 @@ import { export function getDefaultUIState(): UIState { const state: UIState = { + isTopMenuOpen: false, + showWelcomeScreen: true, hasBetaAccess: false, themeName: 'backstage', showApiKeys: false, diff --git a/src/services/ui/useUI.ts b/src/services/ui/useUI.ts index febe161f..c09d1d8d 100644 --- a/src/services/ui/useUI.ts +++ b/src/services/ui/useUI.ts @@ -21,6 +21,13 @@ export const useUI = create()( persist( (set, get) => ({ ...getDefaultUIState(), + setIsTopMenuOpen: (isTopMenuOpen: boolean) => { + set({ isTopMenuOpen }) + }, + setShowWelcomeScreen: (showWelcomeScreen: boolean) => { + console.log('setShowWelcomeScreen called with:', showWelcomeScreen) + set({ showWelcomeScreen: showWelcomeScreen }) + }, setHasBetaAccess: (hasBetaAccess: boolean) => { set({ hasBetaAccess }) },