added demo live-portrait workflow

jbilcke-hf · Aug 26, 2024 · e4ff8a6 · e4ff8a6
1 parent 8168440
commit e4ff8a6
Show file tree

Hide file tree

Showing 18 changed files with 249 additions and 96 deletions.
diff --git a/packages/app/src/app/api/resolve/providers/falai/index.ts b/packages/app/src/app/api/resolve/providers/falai/index.ts
@@ -10,7 +10,7 @@ import {
   FalAiVideoResponse,
 } from './types'
 import { getWorkflowInputValues } from '../getWorkflowInputValues'
-import { sampleVoice } from '@/lib/core/constants'
+import { sampleDrivingVideo, sampleVoice } from '@/lib/core/constants'
 import { getWorkflowLora } from '@/services/editors/workflow-editor/workflows/common/loras/getWorkflowLora'
 
 export async function resolveSegment(
@@ -174,7 +174,38 @@ export async function resolveSegment(
     model = request.settings.videoGenerationWorkflow.data || ''
 
     // console.log(`request.settings.falAiModelForVideo = `, request.settings.falAiModelForVideo)
-    if (model !== 'fal-ai/stable-video') {
+    if (model === 'fal-ai/live-portrait') {
+      const result = (await fal.run(model, {
+        input: {
+          image_url: request.prompts.video.image,
+
+          // what we do here is that we generate an "idle" video
+          // now, the current driving video is just a dummy one I made for testing
+          // we can replace it by something better, that would reflect the current
+          // pacing of the scene (news anchor, peaceful dialogue, intense, aggressive etc)
+          video_url: sampleDrivingVideo,
+
+          sync_mode: true,
+          enable_safety_checker:
+            request.settings.censorNotForAllAudiencesContent,
+        },
+      })) as FalAiVideoResponse
+
+      if (request.settings.censorNotForAllAudiencesContent) {
+        if (
+          Array.isArray(result.has_nsfw_concepts) &&
+          result.has_nsfw_concepts.includes(true)
+        ) {
+          throw new Error(
+            `The generated content has been filtered according to your safety settings`
+          )
+        }
+      }
+
+      console.log('live portrait result:', result)
+
+      segment.assetUrl = result?.video?.url || ''
+    } else if (model !== 'fal-ai/stable-video') {
       throw new Error(
         `only "fal-ai/stable-video" is supported by Clapper for the moment`
       )

diff --git a/packages/app/src/app/api/resolve/providers/replicate/index.ts b/packages/app/src/app/api/resolve/providers/replicate/index.ts
@@ -5,6 +5,7 @@ import { ResolveRequest } from '@aitube/clapper-services'
 import { TimelineSegment } from '@aitube/timeline'
 import { getWorkflowInputValues } from '../getWorkflowInputValues'
 import { getWorkflowLora } from '@/services/editors/workflow-editor/workflows/common/loras/getWorkflowLora'
+import { sampleDrivingVideo } from '@/lib/core/constants'
 
 export async function resolveSegment(
   request: ResolveRequest
@@ -21,7 +22,6 @@ export async function resolveSegment(
       request.settings.imageGenerationWorkflow
     )
 
-
     const aspectRatio =
       request.meta.orientation === ClapMediaOrientation.SQUARE
         ? '1:1'
@@ -99,17 +99,68 @@ export async function resolveSegment(
     )) as any
     segment.assetUrl = `${response[0] || ''}`
   } else if (request.segment.category === ClapSegmentCategory.VIDEO) {
-    const response = (await replicate.run(
-      request.settings.videoGenerationWorkflow.data as any,
-      {
-        input: {
-          image: request.prompts.video.image,
-          disable_safety_checker:
-            !request.settings.censorNotForAllAudiencesContent,
-        },
-      }
-    )) as any
-    segment.assetUrl = `${response[0] || ''}`
+    const model = request.settings.videoGenerationWorkflow.data as any
+
+    if (model.startsWith("fofr/live-portrait")) {
+      const response = (await replicate.run(
+        request.settings.videoGenerationWorkflow.data as any,
+        {
+          input: {
+            // TODO use the workflows fields to do this
+            face_image: request.prompts.video.image,
+
+            // what we do here is that we generate an "idle" video
+            // now, the current driving video is just a dummy one I made for testing
+            // we can replace it by something better, that would reflect the current
+            // pacing of the scene (news anchor, peaceful dialogue, intense, aggressive etc)
+            driving_video: sampleDrivingVideo,
+
+            // Select every nth frame from the driving video. Set to 1 to use all frames.
+            // default: 1
+            video_select_every_n_frames: 1,
+
+            // Size of the output image
+            // min: 64, max: 2048
+            // default: 512
+            live_portrait_dsize: 512,
+
+            // Scaling factor for the face
+            // min: 1, max: 4
+            // default: 2.3
+            live_portrait_scale: 2.3,
+
+
+            // Enable stitching
+            // default: true
+            live_portrait_stitching: true,
+
+            // Use relative positioning
+            // default: true
+            live_portrait_relative: true,
+
+            // there are a lot of other params, check them here:
+            // https://replicate.com/fofr/live-portrait
+
+
+            disable_safety_checker:
+              !request.settings.censorNotForAllAudiencesContent,
+          },
+        }
+      )) as any
+      segment.assetUrl = `${response[0] || ''}`
+    } else {
+      const response = (await replicate.run(
+        request.settings.videoGenerationWorkflow.data as any,
+        {
+          input: {
+            image: request.prompts.video.image,
+            disable_safety_checker:
+              !request.settings.censorNotForAllAudiencesContent,
+          },
+        }
+      )) as any
+      segment.assetUrl = `${response[0] || ''}`
+    }
   } else {
     throw new Error(
       `Clapper doesn't support ${request.segment.category} generation for provider "Replicate". Please open a pull request with (working code) to solve this!`

diff --git a/packages/app/src/app/api/resolve/route.ts b/packages/app/src/app/api/resolve/route.ts
@@ -177,30 +177,29 @@ export async function POST(req: NextRequest) {
     const faceSwap: ProviderFn | undefined =
       faceswapProviders[faceswapProvider] || undefined
 
-      if (faceSwap) {
-        try {
-          await faceSwap(request)
+    if (faceSwap) {
+      try {
+        await faceSwap(request)
 
-          // we clean-up and parse the output from all the resolvers:
-          // this will download files hosted on CDNs, convert WAV files to MP3 etc
+        // we clean-up and parse the output from all the resolvers:
+        // this will download files hosted on CDNs, convert WAV files to MP3 etc
 
-          segment.assetUrl = await decodeOutput(segment.assetUrl)
+        segment.assetUrl = await decodeOutput(segment.assetUrl)
 
-          segment.assetSourceType = getClapAssetSourceType(segment.assetUrl)
+        segment.assetSourceType = getClapAssetSourceType(segment.assetUrl)
 
-          segment.status = ClapSegmentStatus.COMPLETED
+        segment.status = ClapSegmentStatus.COMPLETED
 
-          const { assetFileFormat, outputType } = getTypeAndExtension(
-            segment.assetUrl
-          )
+        const { assetFileFormat, outputType } = getTypeAndExtension(
+          segment.assetUrl
+        )
 
-          segment.assetFileFormat = assetFileFormat
-          segment.outputType = outputType
-
-        } catch (err) {
-          console.error(`failed to run the faceswap (${err})`)
-        }
+        segment.assetFileFormat = assetFileFormat
+        segment.outputType = outputType
+      } catch (err) {
+        console.error(`failed to run the faceswap (${err})`)
       }
+    }
   }
 
   return NextResponse.json(segment)

diff --git a/packages/app/src/components/toolbars/top-menu/lists/AssistantWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/AssistantWorkflows.tsx
@@ -47,11 +47,13 @@ export function AssistantWorkflows() {
           ai&nbsp;assistant
         </Tag>
         <div className={cn(`flex flex-row items-center space-x-2`)}>
-          {workflow?.provider && <ClapWorkflowProviderLogo
-            provider={workflow?.provider}
-            height={18}
-            className={cn(`rounded-full`)}
-          />}
+          {workflow?.provider && (
+            <ClapWorkflowProviderLogo
+              provider={workflow?.provider}
+              height={18}
+              className={cn(`rounded-full`)}
+            />
+          )}
           <div>{workflow?.label || 'None'}</div>
         </div>
       </MenubarSubTrigger>

diff --git a/packages/app/src/components/toolbars/top-menu/lists/ImageDepthWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/ImageDepthWorkflows.tsx
@@ -47,11 +47,13 @@ export function ImageDepthWorkflows() {
           depth&nbsp;mapper
         </Tag>
         <div className={cn(`flex flex-row items-center space-x-2`)}>
-          {workflow?.provider && <ClapWorkflowProviderLogo
-            provider={workflow?.provider}
-            height={18}
-            className={cn(`rounded-full`)}
-          />}
+          {workflow?.provider && (
+            <ClapWorkflowProviderLogo
+              provider={workflow?.provider}
+              height={18}
+              className={cn(`rounded-full`)}
+            />
+          )}
           <div>{workflow?.label || 'None'}</div>
         </div>
       </MenubarSubTrigger>

diff --git a/packages/app/src/components/toolbars/top-menu/lists/ImageFaceswapWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/ImageFaceswapWorkflows.tsx
@@ -49,11 +49,13 @@ export function ImageFaceswapWorkflows() {
           face&nbsp;swap
         </Tag>
         <div className={cn(`flex flex-row items-center space-x-2`)}>
-          {workflow?.provider && <ClapWorkflowProviderLogo
-            provider={workflow?.provider}
-            height={18}
-            className={cn(`rounded-full`)}
-          />}
+          {workflow?.provider && (
+            <ClapWorkflowProviderLogo
+              provider={workflow?.provider}
+              height={18}
+              className={cn(`rounded-full`)}
+            />
+          )}
           <div>{workflow?.label || 'None'}</div>
         </div>
       </MenubarSubTrigger>

diff --git a/packages/app/src/components/toolbars/top-menu/lists/ImageGenerationWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/ImageGenerationWorkflows.tsx
@@ -59,11 +59,13 @@ export function ImageGenerationWorkflows() {
           generate&nbsp;image
         </Tag>
         <div className={cn(`flex flex-row items-center space-x-2`)}>
-          {workflow?.provider && <ClapWorkflowProviderLogo
-            provider={workflow?.provider}
-            height={18}
-            className={cn(`rounded-full`)}
-          />}
+          {workflow?.provider && (
+            <ClapWorkflowProviderLogo
+              provider={workflow?.provider}
+              height={18}
+              className={cn(`rounded-full`)}
+            />
+          )}
           <div>{workflow?.label || 'None'}</div>
         </div>
       </MenubarSubTrigger>

diff --git a/packages/app/src/components/toolbars/top-menu/lists/ImageSegmentationWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/ImageSegmentationWorkflows.tsx
@@ -51,11 +51,13 @@ export function ImageSegmentationWorkflows() {
           segmentation
         </Tag>
         <div className={cn(`flex flex-row items-center space-x-2`)}>
-          {workflow?.provider && <ClapWorkflowProviderLogo
-            provider={workflow?.provider}
-            height={18}
-            className={cn(`rounded-full`)}
-          />}
+          {workflow?.provider && (
+            <ClapWorkflowProviderLogo
+              provider={workflow?.provider}
+              height={18}
+              className={cn(`rounded-full`)}
+            />
+          )}
           <div>{workflow?.label || 'None'}</div>
         </div>
       </MenubarSubTrigger>

diff --git a/packages/app/src/components/toolbars/top-menu/lists/ImageUpscalingWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/ImageUpscalingWorkflows.tsx
@@ -49,11 +49,13 @@ export function ImageUpscalingWorkflows() {
           upscale&nbsp;image
         </Tag>
         <div className={cn(`flex flex-row items-center space-x-2`)}>
-          {workflow?.provider && <ClapWorkflowProviderLogo
-            provider={workflow?.provider}
-            height={18}
-            className={cn(`rounded-full`)}
-          />}
+          {workflow?.provider && (
+            <ClapWorkflowProviderLogo
+              provider={workflow?.provider}
+              height={18}
+              className={cn(`rounded-full`)}
+            />
+          )}
           <div>{workflow?.label || 'None'}</div>
         </div>
       </MenubarSubTrigger>

diff --git a/packages/app/src/components/toolbars/top-menu/lists/MusicGenerationWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/MusicGenerationWorkflows.tsx
@@ -49,11 +49,13 @@ export function MusicGenerationWorkflows() {
           generate&nbsp;music
         </Tag>
         <div className={cn(`flex flex-row items-center space-x-2`)}>
-          {workflow?.provider && <ClapWorkflowProviderLogo
-            provider={workflow?.provider}
-            height={18}
-            className={cn(`rounded-full`)}
-          />}
+          {workflow?.provider && (
+            <ClapWorkflowProviderLogo
+              provider={workflow?.provider}
+              height={18}
+              className={cn(`rounded-full`)}
+            />
+          )}
           <div>{workflow?.label || 'None'}</div>
         </div>
       </MenubarSubTrigger>

diff --git a/packages/app/src/components/toolbars/top-menu/lists/SoundGenerationWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/SoundGenerationWorkflows.tsx
@@ -49,11 +49,13 @@ export function SoundGenerationWorkflows() {
           generate&nbsp;sound
         </Tag>
         <div className={cn(`flex flex-row items-center space-x-2`)}>
-          {workflow?.provider && <ClapWorkflowProviderLogo
-            provider={workflow?.provider}
-            height={18}
-            className={cn(`rounded-full`)}
-          />}
+          {workflow?.provider && (
+            <ClapWorkflowProviderLogo
+              provider={workflow?.provider}
+              height={18}
+              className={cn(`rounded-full`)}
+            />
+          )}
           <div>{workflow?.label || 'None'}</div>
         </div>
       </MenubarSubTrigger>

diff --git a/packages/app/src/components/toolbars/top-menu/lists/VideoDepthWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/VideoDepthWorkflows.tsx
@@ -47,11 +47,13 @@ export function VideoDepthWorkflows() {
           depth&nbsp;mapper
         </Tag>
         <div className={cn(`flex flex-row items-center space-x-2`)}>
-          {workflow?.provider && <ClapWorkflowProviderLogo
-            provider={workflow?.provider}
-            height={18}
-            className={cn(`rounded-full`)}
-          />}
+          {workflow?.provider && (
+            <ClapWorkflowProviderLogo
+              provider={workflow?.provider}
+              height={18}
+              className={cn(`rounded-full`)}
+            />
+          )}
           <div>{workflow?.label || 'None'}</div>
         </div>
       </MenubarSubTrigger>

diff --git a/packages/app/src/components/toolbars/top-menu/lists/VideoGenerationWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/VideoGenerationWorkflows.tsx
@@ -49,11 +49,13 @@ export function VideoGenerationWorkflows() {
           generate&nbsp;video
         </Tag>
         <div className={cn(`flex flex-row items-center space-x-2`)}>
-          {workflow?.provider && <ClapWorkflowProviderLogo
-            provider={workflow?.provider}
-            height={18}
-            className={cn(`rounded-full`)}
-          />}
+          {workflow?.provider && (
+            <ClapWorkflowProviderLogo
+              provider={workflow?.provider}
+              height={18}
+              className={cn(`rounded-full`)}
+            />
+          )}
           <div>{workflow?.label || 'None'}</div>
         </div>
       </MenubarSubTrigger>