Skip to content

Commit e4ff8a6

Browse files
jbilcke-hfjbilcke-hf
committed
added demo live-portrait workflow
1 parent 8168440 commit e4ff8a6

18 files changed

+249
-96
lines changed

‎packages/app/src/app/api/resolve/providers/falai/index.ts

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import {
1010
FalAiVideoResponse,
1111
} from './types'
1212
import { getWorkflowInputValues } from '../getWorkflowInputValues'
13-
import { sampleVoice } from '@/lib/core/constants'
13+
import { sampleDrivingVideo, sampleVoice } from '@/lib/core/constants'
1414
import { getWorkflowLora } from '@/services/editors/workflow-editor/workflows/common/loras/getWorkflowLora'
1515

1616
export async function resolveSegment(
@@ -174,7 +174,38 @@ export async function resolveSegment(
174174
model = request.settings.videoGenerationWorkflow.data || ''
175175

176176
// console.log(`request.settings.falAiModelForVideo = `, request.settings.falAiModelForVideo)
177-
if (model !== 'fal-ai/stable-video') {
177+
if (model === 'fal-ai/live-portrait') {
178+
const result = (await fal.run(model, {
179+
input: {
180+
image_url: request.prompts.video.image,
181+
182+
// what we do here is that we generate an "idle" video
183+
// now, the current driving video is just a dummy one I made for testing
184+
// we can replace it by something better, that would reflect the current
185+
// pacing of the scene (news anchor, peaceful dialogue, intense, aggressive etc)
186+
video_url: sampleDrivingVideo,
187+
188+
sync_mode: true,
189+
enable_safety_checker:
190+
request.settings.censorNotForAllAudiencesContent,
191+
},
192+
})) as FalAiVideoResponse
193+
194+
if (request.settings.censorNotForAllAudiencesContent) {
195+
if (
196+
Array.isArray(result.has_nsfw_concepts) &&
197+
result.has_nsfw_concepts.includes(true)
198+
) {
199+
throw new Error(
200+
`The generated content has been filtered according to your safety settings`
201+
)
202+
}
203+
}
204+
205+
console.log('live portrait result:', result)
206+
207+
segment.assetUrl = result?.video?.url || ''
208+
} else if (model !== 'fal-ai/stable-video') {
178209
throw new Error(
179210
`only "fal-ai/stable-video" is supported by Clapper for the moment`
180211
)

‎packages/app/src/app/api/resolve/providers/replicate/index.ts

Lines changed: 63 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import { ResolveRequest } from '@aitube/clapper-services'
55
import { TimelineSegment } from '@aitube/timeline'
66
import { getWorkflowInputValues } from '../getWorkflowInputValues'
77
import { getWorkflowLora } from '@/services/editors/workflow-editor/workflows/common/loras/getWorkflowLora'
8+
import { sampleDrivingVideo } from '@/lib/core/constants'
89

910
export async function resolveSegment(
1011
request: ResolveRequest
@@ -21,7 +22,6 @@ export async function resolveSegment(
2122
request.settings.imageGenerationWorkflow
2223
)
2324

24-
2525
const aspectRatio =
2626
request.meta.orientation === ClapMediaOrientation.SQUARE
2727
? '1:1'
@@ -99,17 +99,68 @@ export async function resolveSegment(
9999
)) as any
100100
segment.assetUrl = `${response[0] || ''}`
101101
} else if (request.segment.category === ClapSegmentCategory.VIDEO) {
102-
const response = (await replicate.run(
103-
request.settings.videoGenerationWorkflow.data as any,
104-
{
105-
input: {
106-
image: request.prompts.video.image,
107-
disable_safety_checker:
108-
!request.settings.censorNotForAllAudiencesContent,
109-
},
110-
}
111-
)) as any
112-
segment.assetUrl = `${response[0] || ''}`
102+
const model = request.settings.videoGenerationWorkflow.data as any
103+
104+
if (model.startsWith("fofr/live-portrait")) {
105+
const response = (await replicate.run(
106+
request.settings.videoGenerationWorkflow.data as any,
107+
{
108+
input: {
109+
// TODO use the workflows fields to do this
110+
face_image: request.prompts.video.image,
111+
112+
// what we do here is that we generate an "idle" video
113+
// now, the current driving video is just a dummy one I made for testing
114+
// we can replace it by something better, that would reflect the current
115+
// pacing of the scene (news anchor, peaceful dialogue, intense, aggressive etc)
116+
driving_video: sampleDrivingVideo,
117+
118+
// Select every nth frame from the driving video. Set to 1 to use all frames.
119+
// default: 1
120+
video_select_every_n_frames: 1,
121+
122+
// Size of the output image
123+
// min: 64, max: 2048
124+
// default: 512
125+
live_portrait_dsize: 512,
126+
127+
// Scaling factor for the face
128+
// min: 1, max: 4
129+
// default: 2.3
130+
live_portrait_scale: 2.3,
131+
132+
133+
// Enable stitching
134+
// default: true
135+
live_portrait_stitching: true,
136+
137+
// Use relative positioning
138+
// default: true
139+
live_portrait_relative: true,
140+
141+
// there are a lot of other params, check them here:
142+
// https://replicate.com/fofr/live-portrait
143+
144+
145+
disable_safety_checker:
146+
!request.settings.censorNotForAllAudiencesContent,
147+
},
148+
}
149+
)) as any
150+
segment.assetUrl = `${response[0] || ''}`
151+
} else {
152+
const response = (await replicate.run(
153+
request.settings.videoGenerationWorkflow.data as any,
154+
{
155+
input: {
156+
image: request.prompts.video.image,
157+
disable_safety_checker:
158+
!request.settings.censorNotForAllAudiencesContent,
159+
},
160+
}
161+
)) as any
162+
segment.assetUrl = `${response[0] || ''}`
163+
}
113164
} else {
114165
throw new Error(
115166
`Clapper doesn't support ${request.segment.category} generation for provider "Replicate". Please open a pull request with (working code) to solve this!`

‎packages/app/src/app/api/resolve/route.ts

Lines changed: 16 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -177,30 +177,29 @@ export async function POST(req: NextRequest) {
177177
const faceSwap: ProviderFn | undefined =
178178
faceswapProviders[faceswapProvider] || undefined
179179

180-
if (faceSwap) {
181-
try {
182-
await faceSwap(request)
180+
if (faceSwap) {
181+
try {
182+
await faceSwap(request)
183183

184-
// we clean-up and parse the output from all the resolvers:
185-
// this will download files hosted on CDNs, convert WAV files to MP3 etc
184+
// we clean-up and parse the output from all the resolvers:
185+
// this will download files hosted on CDNs, convert WAV files to MP3 etc
186186

187-
segment.assetUrl = await decodeOutput(segment.assetUrl)
187+
segment.assetUrl = await decodeOutput(segment.assetUrl)
188188

189-
segment.assetSourceType = getClapAssetSourceType(segment.assetUrl)
189+
segment.assetSourceType = getClapAssetSourceType(segment.assetUrl)
190190

191-
segment.status = ClapSegmentStatus.COMPLETED
191+
segment.status = ClapSegmentStatus.COMPLETED
192192

193-
const { assetFileFormat, outputType } = getTypeAndExtension(
194-
segment.assetUrl
195-
)
193+
const { assetFileFormat, outputType } = getTypeAndExtension(
194+
segment.assetUrl
195+
)
196196

197-
segment.assetFileFormat = assetFileFormat
198-
segment.outputType = outputType
199-
200-
} catch (err) {
201-
console.error(`failed to run the faceswap (${err})`)
202-
}
197+
segment.assetFileFormat = assetFileFormat
198+
segment.outputType = outputType
199+
} catch (err) {
200+
console.error(`failed to run the faceswap (${err})`)
203201
}
202+
}
204203
}
205204

206205
return NextResponse.json(segment)

‎packages/app/src/components/toolbars/top-menu/lists/AssistantWorkflows.tsx

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,11 +47,13 @@ export function AssistantWorkflows() {
4747
ai assistant
4848
</Tag>
4949
<div className={cn(`flex flex-row items-center space-x-2`)}>
50-
{workflow?.provider && <ClapWorkflowProviderLogo
51-
provider={workflow?.provider}
52-
height={18}
53-
className={cn(`rounded-full`)}
54-
/>}
50+
{workflow?.provider && (
51+
<ClapWorkflowProviderLogo
52+
provider={workflow?.provider}
53+
height={18}
54+
className={cn(`rounded-full`)}
55+
/>
56+
)}
5557
<div>{workflow?.label || 'None'}</div>
5658
</div>
5759
</MenubarSubTrigger>

‎packages/app/src/components/toolbars/top-menu/lists/ImageDepthWorkflows.tsx

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,11 +47,13 @@ export function ImageDepthWorkflows() {
4747
depth&nbsp;mapper
4848
</Tag>
4949
<div className={cn(`flex flex-row items-center space-x-2`)}>
50-
{workflow?.provider && <ClapWorkflowProviderLogo
51-
provider={workflow?.provider}
52-
height={18}
53-
className={cn(`rounded-full`)}
54-
/>}
50+
{workflow?.provider && (
51+
<ClapWorkflowProviderLogo
52+
provider={workflow?.provider}
53+
height={18}
54+
className={cn(`rounded-full`)}
55+
/>
56+
)}
5557
<div>{workflow?.label || 'None'}</div>
5658
</div>
5759
</MenubarSubTrigger>

‎packages/app/src/components/toolbars/top-menu/lists/ImageFaceswapWorkflows.tsx

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -49,11 +49,13 @@ export function ImageFaceswapWorkflows() {
4949
face&nbsp;swap
5050
</Tag>
5151
<div className={cn(`flex flex-row items-center space-x-2`)}>
52-
{workflow?.provider && <ClapWorkflowProviderLogo
53-
provider={workflow?.provider}
54-
height={18}
55-
className={cn(`rounded-full`)}
56-
/>}
52+
{workflow?.provider && (
53+
<ClapWorkflowProviderLogo
54+
provider={workflow?.provider}
55+
height={18}
56+
className={cn(`rounded-full`)}
57+
/>
58+
)}
5759
<div>{workflow?.label || 'None'}</div>
5860
</div>
5961
</MenubarSubTrigger>

‎packages/app/src/components/toolbars/top-menu/lists/ImageGenerationWorkflows.tsx

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -59,11 +59,13 @@ export function ImageGenerationWorkflows() {
5959
generate&nbsp;image
6060
</Tag>
6161
<div className={cn(`flex flex-row items-center space-x-2`)}>
62-
{workflow?.provider && <ClapWorkflowProviderLogo
63-
provider={workflow?.provider}
64-
height={18}
65-
className={cn(`rounded-full`)}
66-
/>}
62+
{workflow?.provider && (
63+
<ClapWorkflowProviderLogo
64+
provider={workflow?.provider}
65+
height={18}
66+
className={cn(`rounded-full`)}
67+
/>
68+
)}
6769
<div>{workflow?.label || 'None'}</div>
6870
</div>
6971
</MenubarSubTrigger>

‎packages/app/src/components/toolbars/top-menu/lists/ImageSegmentationWorkflows.tsx

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -51,11 +51,13 @@ export function ImageSegmentationWorkflows() {
5151
segmentation
5252
</Tag>
5353
<div className={cn(`flex flex-row items-center space-x-2`)}>
54-
{workflow?.provider && <ClapWorkflowProviderLogo
55-
provider={workflow?.provider}
56-
height={18}
57-
className={cn(`rounded-full`)}
58-
/>}
54+
{workflow?.provider && (
55+
<ClapWorkflowProviderLogo
56+
provider={workflow?.provider}
57+
height={18}
58+
className={cn(`rounded-full`)}
59+
/>
60+
)}
5961
<div>{workflow?.label || 'None'}</div>
6062
</div>
6163
</MenubarSubTrigger>

‎packages/app/src/components/toolbars/top-menu/lists/ImageUpscalingWorkflows.tsx

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -49,11 +49,13 @@ export function ImageUpscalingWorkflows() {
4949
upscale&nbsp;image
5050
</Tag>
5151
<div className={cn(`flex flex-row items-center space-x-2`)}>
52-
{workflow?.provider && <ClapWorkflowProviderLogo
53-
provider={workflow?.provider}
54-
height={18}
55-
className={cn(`rounded-full`)}
56-
/>}
52+
{workflow?.provider && (
53+
<ClapWorkflowProviderLogo
54+
provider={workflow?.provider}
55+
height={18}
56+
className={cn(`rounded-full`)}
57+
/>
58+
)}
5759
<div>{workflow?.label || 'None'}</div>
5860
</div>
5961
</MenubarSubTrigger>

‎packages/app/src/components/toolbars/top-menu/lists/MusicGenerationWorkflows.tsx

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -49,11 +49,13 @@ export function MusicGenerationWorkflows() {
4949
generate&nbsp;music
5050
</Tag>
5151
<div className={cn(`flex flex-row items-center space-x-2`)}>
52-
{workflow?.provider && <ClapWorkflowProviderLogo
53-
provider={workflow?.provider}
54-
height={18}
55-
className={cn(`rounded-full`)}
56-
/>}
52+
{workflow?.provider && (
53+
<ClapWorkflowProviderLogo
54+
provider={workflow?.provider}
55+
height={18}
56+
className={cn(`rounded-full`)}
57+
/>
58+
)}
5759
<div>{workflow?.label || 'None'}</div>
5860
</div>
5961
</MenubarSubTrigger>

‎packages/app/src/components/toolbars/top-menu/lists/SoundGenerationWorkflows.tsx

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -49,11 +49,13 @@ export function SoundGenerationWorkflows() {
4949
generate&nbsp;sound
5050
</Tag>
5151
<div className={cn(`flex flex-row items-center space-x-2`)}>
52-
{workflow?.provider && <ClapWorkflowProviderLogo
53-
provider={workflow?.provider}
54-
height={18}
55-
className={cn(`rounded-full`)}
56-
/>}
52+
{workflow?.provider && (
53+
<ClapWorkflowProviderLogo
54+
provider={workflow?.provider}
55+
height={18}
56+
className={cn(`rounded-full`)}
57+
/>
58+
)}
5759
<div>{workflow?.label || 'None'}</div>
5860
</div>
5961
</MenubarSubTrigger>

‎packages/app/src/components/toolbars/top-menu/lists/VideoDepthWorkflows.tsx

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,11 +47,13 @@ export function VideoDepthWorkflows() {
4747
depth&nbsp;mapper
4848
</Tag>
4949
<div className={cn(`flex flex-row items-center space-x-2`)}>
50-
{workflow?.provider && <ClapWorkflowProviderLogo
51-
provider={workflow?.provider}
52-
height={18}
53-
className={cn(`rounded-full`)}
54-
/>}
50+
{workflow?.provider && (
51+
<ClapWorkflowProviderLogo
52+
provider={workflow?.provider}
53+
height={18}
54+
className={cn(`rounded-full`)}
55+
/>
56+
)}
5557
<div>{workflow?.label || 'None'}</div>
5658
</div>
5759
</MenubarSubTrigger>

‎packages/app/src/components/toolbars/top-menu/lists/VideoGenerationWorkflows.tsx

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -49,11 +49,13 @@ export function VideoGenerationWorkflows() {
4949
generate&nbsp;video
5050
</Tag>
5151
<div className={cn(`flex flex-row items-center space-x-2`)}>
52-
{workflow?.provider && <ClapWorkflowProviderLogo
53-
provider={workflow?.provider}
54-
height={18}
55-
className={cn(`rounded-full`)}
56-
/>}
52+
{workflow?.provider && (
53+
<ClapWorkflowProviderLogo
54+
provider={workflow?.provider}
55+
height={18}
56+
className={cn(`rounded-full`)}
57+
/>
58+
)}
5759
<div>{workflow?.label || 'None'}</div>
5860
</div>
5961
</MenubarSubTrigger>

0 commit comments

Comments
 (0)