fix replicate

jbilcke-hf · Aug 18, 2024 · 193dbf1 · 193dbf1
1 parent 17ee0ab
commit 193dbf1
Show file tree

Hide file tree

Showing 54 changed files with 1,023 additions and 337 deletions.
diff --git a/bun.lockb b/bun.lockb
diff --git a/package.json b/package.json
@@ -1,22 +1,12 @@
 {
   "name": "@aitube/clapper-monorepo",
   "version": "0.2.4",
-  "private": true,
   "description": "A monorepo for the Clapper project. Individual packages are in the packages directory.",
-  "workspaces": [
-    "packages/clap",
-    "packages/timeline",
-    "packages/api-client",
-    "packages/io",
-    "packages/colors",
-    "packages/engine",
-    "packages/broadway",
-    "packages/clapper-services",
-    "packages/app"
-  ],
   "engines": {
     "bun": ">=1.0.0"
   },
+  "packageManager": "[email protected]",
+  "private": true,
   "scripts": {
     "dev": "bun run --cwd packages/app dev",
     "start": "bun run --cwd packages/app start",
@@ -36,10 +26,20 @@
     "test:all": "bun run --cwd packages/clap test && bun run --cwd packages/timeline test && bun run --cwd packages/api-client test && bun run --cwd packages/io test && bun run --cwd packages/colors test && bun run --cwd packages/engine test && bun run --cwd packages/broadway test && bun run --cwd packages/clapper-services test && bun run --cwd packages/app test",
     "format": "bun run --cwd packages/app format"
   },
-  "packageManager": "[email protected]",
   "trustedDependencies": [
     "@aitube/clapper",
     "onnxruntime-node",
     "protobufjs"
+  ],
+  "workspaces": [
+    "packages/clap",
+    "packages/timeline",
+    "packages/api-client",
+    "packages/io",
+    "packages/colors",
+    "packages/engine",
+    "packages/broadway",
+    "packages/clapper-services",
+    "packages/app"
   ]
 }
diff --git a/packages/api-client/package.json b/packages/api-client/package.json
@@ -39,6 +39,8 @@
     "dist/**/*.d.ts"
   ],
   "dependencies": {
+    "@aitube/clap": "workspace:*",
+    "@types/bun": "latest",
     "query-string": "^9.0.0"
   }
 }
diff --git a/packages/app/package.json b/packages/app/package.json
@@ -41,7 +41,7 @@
     "@aitube/clapper-services": "workspace:*",
     "@aitube/engine": "workspace:*",
     "@aitube/timeline": "workspace:*",
-    "@fal-ai/serverless-client": "^0.13.0",
+    "@fal-ai/serverless-client": "^0.14.2",
     "@ffmpeg/ffmpeg": "^0.12.10",
     "@ffmpeg/util": "^0.12.1",
     "@gradio/client": "^1.5.0",

diff --git a/packages/app/src/app/api/resolve/providers/replicate/index.ts b/packages/app/src/app/api/resolve/providers/replicate/index.ts
@@ -1,8 +1,11 @@
 import Replicate from 'replicate'
 
-import { ClapSegmentCategory } from '@aitube/clap'
+import { ClapMediaOrientation, ClapSegmentCategory } from '@aitube/clap'
 import { ResolveRequest } from '@aitube/clapper-services'
 import { TimelineSegment } from '@aitube/timeline'
+import { getWorkflowInputValues } from '../getWorkflowInputValues'
+import { defaultLoraModels } from '@/services/editors/workflow-editor/workflows/common/loras'
+import { getWorkflowLora } from '@/services/editors/workflow-editor/workflows/common/loras/getWorkflowLora'
 
 export async function resolveSegment(
   request: ResolveRequest
@@ -12,29 +15,57 @@ export async function resolveSegment(
   }
   const replicate = new Replicate({ auth: request.settings.replicateApiKey })
 
-  if (request.segment.category !== ClapSegmentCategory.STORYBOARD) {
-    throw new Error(
-      `Clapper doesn't support ${request.segment.category} generation for provider "Replicate". Please open a pull request with (working code) to solve this!`
-    )
-  }
-
   const segment = request.segment
 
-  // this mapping isn't great, we should use something auto-adapting
-  // like we are doing for Hugging Face (match the fields etc)
-  if (request.segment.category === ClapSegmentCategory.STORYBOARD) {
+  if (request.segment.category == ClapSegmentCategory.STORYBOARD) {
+
+    const { workflowValues } = getWorkflowInputValues(
+      request.settings.imageGenerationWorkflow
+    )
+
     let params: object = {
       prompt: request.prompts.image.positive,
       width: request.meta.width,
       height: request.meta.height,
+      disable_safety_checker: !request.settings.censorNotForAllAudiencesContent,
     }
+
+    const aspectRatio =
+    request.meta.orientation === ClapMediaOrientation.SQUARE
+      ? "1:1"
+      : request.meta.orientation === ClapMediaOrientation.PORTRAIT
+        ? "9:16"
+        : "16:9"
+
     if (
       request.settings.imageGenerationWorkflow.data === 'fofr/pulid-lightning'
     ) {
       params = {
         ...params,
         face_image: request.prompts.image.identity,
       }
+    } else if (
+      request.settings.imageGenerationWorkflow.data === 'lucataco/flux-dev-lora'
+    ) {
+
+      // note: this isn't the right place to do this, because maybe the LoRAs are dynamic
+      const loraModel = getWorkflowLora(request.settings.imageGenerationWorkflow)
+
+      params = {
+        // for some reason this model doesn't support arbitrary width and height,
+        // at least not at the time of writing..
+        aspect_ratio: aspectRatio,
+
+        hf_lora: workflowValues['hf_lora'] || '',
+
+        prompt: [
+          loraModel?.trigger,
+          request.prompts.image.positive
+        ].filter(x => x).join(' '),
+
+        disable_safety_checker: !request.settings.censorNotForAllAudiencesContent,
+      }
+
     } else if (
       request.settings.imageGenerationWorkflow.data === 'zsxkib/pulid'
     ) {
@@ -43,32 +74,44 @@ export async function resolveSegment(
         main_face_image: request.prompts.image.identity,
       }
     }
+
+    /*
+    console.log("debug:", {
+      model: request.settings.imageGenerationWorkflow.data,
+      params,
+    })
+      */
     const response = (await replicate.run(
-      request.settings.imageGenerationWorkflow as any,
+      request.settings.imageGenerationWorkflow.data as any,
       { input: params }
     )) as any
-    segment.assetUrl = `${response.output || ''}`
+
+
+    segment.assetUrl = `${response[0] || ''}`
+
   } else if (request.segment.category === ClapSegmentCategory.DIALOGUE) {
     const response = (await replicate.run(
       request.settings.voiceGenerationWorkflow.data as any,
       {
         input: {
           text: request.prompts.voice.positive,
           audio: request.prompts.voice.identity,
+          disable_safety_checker: !request.settings.censorNotForAllAudiencesContent,
         },
       }
     )) as any
-    segment.assetUrl = `${response.output || ''}`
+    segment.assetUrl = `${response[0] || ''}`
   } else if (request.segment.category === ClapSegmentCategory.VIDEO) {
     const response = (await replicate.run(
       request.settings.videoGenerationWorkflow.data as any,
       {
         input: {
           image: request.prompts.video.image,
+          disable_safety_checker: !request.settings.censorNotForAllAudiencesContent,
         },
       }
     )) as any
-    segment.assetUrl = `${response.output || ''}`
+    segment.assetUrl = `${response[0] || ''}`
   } else {
     throw new Error(
       `Clapper doesn't support ${request.segment.category} generation for provider "Replicate". Please open a pull request with (working code) to solve this!`

diff --git a/packages/app/src/app/api/resolve/route.ts b/packages/app/src/app/api/resolve/route.ts
@@ -132,6 +132,9 @@ export async function POST(req: NextRequest) {
       segment.outputType === ClapOutputType.AUDIO ||
       segment.outputType === ClapOutputType.VIDEO
     ) {
+
+
+      // TODO this should be down in the browser side, so that we can scale better
       const { durationInMs, hasAudio } = await getMediaInfo(segment.assetUrl)
       segment.assetDurationInMs = durationInMs
 

diff --git a/packages/app/src/components/toolbars/top-menu/lists/AssistantWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/AssistantWorkflows.tsx
@@ -21,18 +21,20 @@ import {
   ClapWorkflowProviderLogo,
   ClapWorkflowProviderName,
 } from '@/components/core/providers'
+import { parseWorkflow } from '@/services/settings/workflows/parseWorkflow'
+
+const category = ClapWorkflowCategory.ASSISTANT
 
 export function AssistantWorkflows() {
-  const workflowId = useSettings((s) => s.assistantWorkflow)
-  const setWorkflowId = useSettings((s) => s.setAssistantWorkflow)
+  const assistantWorkflow = useSettings((s) => s.assistantWorkflow)
+  const setAssistantWorkflow = useSettings((s) => s.setAssistantWorkflow)
   const availableWorkflows = useWorkflowEditor((s) => s.availableWorkflows)
 
-  const { workflows, providers, nbProviders } = findWorkflows(
-    availableWorkflows,
-    { category: ClapWorkflowCategory.ASSISTANT }
-  )
+  const { providers, nbProviders } = findWorkflows(availableWorkflows, {
+    category,
+  })
 
-  const { workflow } = findWorkflows(workflows, { workflowId })
+  const workflow = parseWorkflow(assistantWorkflow, category)
 
   if (!nbProviders) {
     return null
@@ -65,15 +67,15 @@ export function AssistantWorkflows() {
               {workflows?.map((w) => (
                 <MenubarCheckboxItem
                   key={w.id}
-                  checked={workflowId === w.id}
+                  checked={workflow.id === w.id}
                   disabled={hasNoPublicAPI(w)}
                   onClick={(e) => {
                     if (hasNoPublicAPI(w)) {
                       e.stopPropagation()
                       e.preventDefault()
                       return false
                     }
-                    setWorkflowId(w.id)
+                    setAssistantWorkflow(w)
                     e.stopPropagation()
                     e.preventDefault()
                     return false

diff --git a/packages/app/src/components/toolbars/top-menu/lists/ImageDepthWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/ImageDepthWorkflows.tsx
@@ -21,18 +21,20 @@ import {
   ClapWorkflowProviderLogo,
   ClapWorkflowProviderName,
 } from '@/components/core/providers'
+import { parseWorkflow } from '@/services/settings/workflows/parseWorkflow'
+
+const category = ClapWorkflowCategory.IMAGE_DEPTH_MAPPING
 
 export function ImageDepthWorkflows() {
-  const workflowId = useSettings((s) => s.imageDepthWorkflow)
-  const setWorkflowId = useSettings((s) => s.setImageDepthWorkflow)
+  const imageDepthWorkflow = useSettings((s) => s.imageDepthWorkflow)
+  const setImageDepthWorkflow = useSettings((s) => s.setImageDepthWorkflow)
   const availableWorkflows = useWorkflowEditor((s) => s.availableWorkflows)
 
-  const { workflows, providers, nbProviders } = findWorkflows(
-    availableWorkflows,
-    { category: ClapWorkflowCategory.IMAGE_DEPTH_MAPPING }
-  )
+  const { providers, nbProviders } = findWorkflows(availableWorkflows, {
+    category,
+  })
 
-  const { workflow } = findWorkflows(workflows, { workflowId })
+  const workflow = parseWorkflow(imageDepthWorkflow, category)
 
   if (!nbProviders) {
     return null
@@ -65,15 +67,15 @@ export function ImageDepthWorkflows() {
               {workflows?.map((w) => (
                 <MenubarCheckboxItem
                   key={w.id}
-                  checked={workflowId === w.id}
+                  checked={workflow.id === w.id}
                   disabled={hasNoPublicAPI(w)}
                   onClick={(e) => {
                     if (hasNoPublicAPI(w)) {
                       e.stopPropagation()
                       e.preventDefault()
                       return false
                     }
-                    setWorkflowId(w.id)
+                    setImageDepthWorkflow(w)
                     e.stopPropagation()
                     e.preventDefault()
                     return false