neutral theme, comfy auth, dynamic workflows

Author: jbilcke-hf

documentation/design/README.md

@@ -0,0 +1,70 @@
+# Design
+
+(those are just quick notes for now, we can elaborate later)
+
+## Designing Clapper for the Web
+
+### Targeted devices
+
+Currently Clapper works best on a laptop,
+since the UI has been developed around the presence of a relatively wide screen,
+a mouse and a touch pad (for horizontal scrolling in the timeline).
+
+However I think we should try to better support the following environments:
+
+- Tiny laptop (12", 11")
+- Tablet (without a mouse)
+- Desktop computer with multiple screens
+
+(mobile has its own chapter, see below in this document)
+
+### Use Web APIs in priority
+
+We should try to use standard Web APIs (ratified by the W3C) as much as possible, with polyfills in case some browsers don't implement them yet.
+
+We can use experimental standards (eg. WebGPU),
+but since they are unstable and not supported by all browsers, they should not be mandatory to the experience.
+
+## Designing Clapper for desktop
+
+The experience on Desktop should be similar to the one in a browser, but we can do some changes:
+
+### Customizing the look of the app window
+
+Electron offers us to hide or customize the app's window, so we should do it.
+
+We don't have to follow the design principles of the underlying operating system (some apps don't care eg. video games, Spotify, Slack, FLStudio, Discord..) but it may be necessary for some operations (file pickers, installer, window management etc).
+
+### Use the native file system
+
+The big benefit of running Clapper as a desktop application is that we can access the file system, meaning we can work with files of arbitrary file length (note: for this kind of file system manipulation, we will have to use extra code eg. NodeJS)
+
+This can also be used for performance optimization such as using temporary files, or pre-computing things and store them in a cache for the next time Clapper is opened.
+
+### Download additional data for local use
+
+By running Clapper on the user's device, we can also make it download data of arbitrary size.
+
+This can help with various use cases, such as running AI models locally.
+
+For instance the desktop app LMStudio can download models from Hugging Face.
+
+### Call external or embedded native libraries
+
+We have complete freedom to ship Clapper with embedded native tools eg. a database or a native library (eg. a C++ library to run a LLM locally).
+
+We could even use Python scripts with Clapper.
+
+Please note however that we will have to make sure anything we embed works on various operating systems (Windows/macOS/Linux) so this requires dedicated skills and maintenance.
+
+### Use System Notifications
+
+When a job is pending, finished, a software update is ready etc.
+
+## Clapper Design For Mobile
+
+Currently Clapper can *run* on mobile, but it is not designed for it.
+
+So things like the top menu etc will be pretty much unuseable, unless we adapt them.
+
+

package-lock.json

@@ -38,7 +38,7 @@
   "dependencies": {
     "@aitube/broadway": "0.2.3",
     "@aitube/clap": "0.2.3",
-    "@aitube/clapper-services": "0.2.3-1",
+    "@aitube/clapper-services": "0.2.3-2",
     "@aitube/client": "0.2.3",
     "@aitube/engine": "0.2.3",
     "@aitube/timeline": "0.2.3",
@@ -60,7 +60,7 @@
     "@radix-ui/react-avatar": "^1.0.4",
     "@radix-ui/react-checkbox": "^1.0.4",
     "@radix-ui/react-collapsible": "^1.0.3",
-    "@radix-ui/react-dialog": "^1.0.5",
+    "@radix-ui/react-dialog": "^1.1.1",
     "@radix-ui/react-dropdown-menu": "^2.0.6",
     "@radix-ui/react-icons": "^1.3.0",
     "@radix-ui/react-label": "^2.0.2",
@@ -93,7 +93,7 @@
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.1",
     "cmdk": "^0.2.1",
-    "comfydeploy": "^0.0.19-beta.13",
+    "comfydeploy": "^0.0.21",
     "date-fns": "^3.6.0",
     "dotenv": "^16.4.5",
     "fflate": "^0.8.2",
@@ -116,6 +116,7 @@
     "react-dnd-html5-backend": "^16.0.1",
     "react-dom": "^18.3.1",
     "react-drag-drop-files": "^2.3.10",
+    "react-error-boundary": "^4.0.13",
     "react-hook-consent": "^3.5.3",
     "react-hotkeys-hook": "^4.5.0",
     "react-icons": "^5.2.1",
@@ -126,7 +127,7 @@
     "replicate": "^0.32.0",
     "sharp": "0.33.4",
     "sonner": "^1.5.0",
-    "tailwind-merge": "^2.4.0",
+    "tailwind-merge": "^2.5.2",
     "tailwindcss-animate": "^1.0.7",
     "three": "^0.164.1",
     "ts-node": "^10.9.2",
@@ -136,7 +137,7 @@
     "web-audio-beat-detector": "^8.2.12",
     "yaml": "^2.4.5",
     "zustand": "4.5.2",
-    "zx": "^8.1.3"
+    "zx": "^8.1.4"
   },
   "devDependencies": {
     "@electron-forge/cli": "^7.4.0",

package.json

@@ -1,7 +1,11 @@
-import { ClapSegmentStatus, getClapAssetSourceType } from '@aitube/clap'
+import {
+  ClapSegmentCategory,
+  ClapSegmentStatus,
+  getClapAssetSourceType,
+} from '@aitube/clap'
 
 import { ResolveRequest } from '@aitube/clapper-services'
-import { getComfyWorkflow } from '../comfyui/getComfyWorkflow'
+import { getComfyWorkflow } from '../../../../../services/editors/workflow-editor/workflows/comfyui/getComfyWorkflow'
 import { runWorkflow } from './runWorkflow'
 import { TimelineSegment } from '@aitube/timeline'
 
@@ -11,22 +15,55 @@ export async function resolveSegment(
   if (!request.settings.replicateApiKey) {
     throw new Error(`Missing API key for "Replicate.com"`)
   }
-  const workflow = getComfyWorkflow(request)
 
   const segment: TimelineSegment = request.segment
 
-  try {
-    segment.assetUrl = await runWorkflow({
-      apiKey: request.settings.replicateApiKey,
-      workflow,
-    })
-    segment.assetSourceType = getClapAssetSourceType(segment.assetUrl)
-  } catch (err) {
-    console.error(`failed to call Replicate: `, err)
-    segment.assetUrl = ''
-    segment.assetSourceType = getClapAssetSourceType(segment.assetUrl)
-    segment.status = ClapSegmentStatus.TO_GENERATE
-    // request.segment.status = ClapSegmentStatus.ERROR
+  if (request.segment.category === ClapSegmentCategory.STORYBOARD) {
+    const inputFields =
+      request.settings.imageGenerationWorkflow.inputFields || []
+
+    // since this is a random "wild" workflow, it is possible
+    // that the field name is a bit different
+    // we try to look into the workflow input fields
+    // to find the best match
+    const promptFields = [
+      inputFields.find((f) => f.id === 'prompt'), // exactMatch,
+      inputFields.find((f) => f.id.includes('prompt')), // similarName,
+      inputFields.find((f) => f.type === 'string'), // similarType
+    ].filter((x) => typeof x !== 'undefined')
+
+    const promptField = promptFields[0]
+    if (!promptField) {
+      throw new Error(
+        `this workflow doesn't seem to have a parameter called "prompt"`
+      )
+    }
+
+    console.log(`TODO: inject parameters into the final request object`)
+    const workflow = `{}` // <- the final workflow sent to Replicate
+
+    try {
+      segment.assetUrl = await runWorkflow({
+        apiKey: request.settings.replicateApiKey,
+        workflow,
+      })
+      segment.assetSourceType = getClapAssetSourceType(segment.assetUrl)
+    } catch (err) {
+      console.error(`failed to call Replicate: `, err)
+      segment.assetUrl = ''
+      segment.assetSourceType = getClapAssetSourceType(segment.assetUrl)
+      segment.status = ClapSegmentStatus.TO_GENERATE
+      // request.segment.status = ClapSegmentStatus.ERROR
+    }
+  } else if (request.segment.category === ClapSegmentCategory.VIDEO) {
+    // TODO do the same for video
+    throw new Error(
+      `Clapper doesn't support ${request.segment.category} generation for provider "Comfy.icu". Please open a pull request with (working code) to solve this!`
+    )
+  } else {
+    throw new Error(
+      `Clapper doesn't support ${request.segment.category} generation for provider "Comfy.icu". Please open a pull request with (working code) to solve this!`
+    )
   }
 
   return segment