diff --git a/packages/app/src/app/api/resolve/providers/falai/index.ts b/packages/app/src/app/api/resolve/providers/falai/index.ts index 435ee0f0..ecb65741 100644 --- a/packages/app/src/app/api/resolve/providers/falai/index.ts +++ b/packages/app/src/app/api/resolve/providers/falai/index.ts @@ -10,7 +10,7 @@ import { FalAiVideoResponse, } from './types' import { getWorkflowInputValues } from '../getWorkflowInputValues' -import { sampleVoice } from '@/lib/core/constants' +import { sampleDrivingVideo, sampleVoice } from '@/lib/core/constants' import { getWorkflowLora } from '@/services/editors/workflow-editor/workflows/common/loras/getWorkflowLora' export async function resolveSegment( @@ -174,7 +174,38 @@ export async function resolveSegment( model = request.settings.videoGenerationWorkflow.data || '' // console.log(`request.settings.falAiModelForVideo = `, request.settings.falAiModelForVideo) - if (model !== 'fal-ai/stable-video') { + if (model === 'fal-ai/live-portrait') { + const result = (await fal.run(model, { + input: { + image_url: request.prompts.video.image, + + // what we do here is that we generate an "idle" video + // now, the current driving video is just a dummy one I made for testing + // we can replace it by something better, that would reflect the current + // pacing of the scene (news anchor, peaceful dialogue, intense, aggressive etc) + video_url: sampleDrivingVideo, + + sync_mode: true, + enable_safety_checker: + request.settings.censorNotForAllAudiencesContent, + }, + })) as FalAiVideoResponse + + if (request.settings.censorNotForAllAudiencesContent) { + if ( + Array.isArray(result.has_nsfw_concepts) && + result.has_nsfw_concepts.includes(true) + ) { + throw new Error( + `The generated content has been filtered according to your safety settings` + ) + } + } + + console.log('live portrait result:', result) + + segment.assetUrl = result?.video?.url || '' + } else if (model !== 'fal-ai/stable-video') { throw new Error( `only "fal-ai/stable-video" is supported by Clapper for the moment` ) diff --git a/packages/app/src/app/api/resolve/providers/replicate/index.ts b/packages/app/src/app/api/resolve/providers/replicate/index.ts index 46ef034d..9230e2df 100644 --- a/packages/app/src/app/api/resolve/providers/replicate/index.ts +++ b/packages/app/src/app/api/resolve/providers/replicate/index.ts @@ -5,6 +5,7 @@ import { ResolveRequest } from '@aitube/clapper-services' import { TimelineSegment } from '@aitube/timeline' import { getWorkflowInputValues } from '../getWorkflowInputValues' import { getWorkflowLora } from '@/services/editors/workflow-editor/workflows/common/loras/getWorkflowLora' +import { sampleDrivingVideo } from '@/lib/core/constants' export async function resolveSegment( request: ResolveRequest @@ -21,7 +22,6 @@ export async function resolveSegment( request.settings.imageGenerationWorkflow ) - const aspectRatio = request.meta.orientation === ClapMediaOrientation.SQUARE ? '1:1' @@ -99,17 +99,68 @@ export async function resolveSegment( )) as any segment.assetUrl = `${response[0] || ''}` } else if (request.segment.category === ClapSegmentCategory.VIDEO) { - const response = (await replicate.run( - request.settings.videoGenerationWorkflow.data as any, - { - input: { - image: request.prompts.video.image, - disable_safety_checker: - !request.settings.censorNotForAllAudiencesContent, - }, - } - )) as any - segment.assetUrl = `${response[0] || ''}` + const model = request.settings.videoGenerationWorkflow.data as any + + if (model.startsWith("fofr/live-portrait")) { + const response = (await replicate.run( + request.settings.videoGenerationWorkflow.data as any, + { + input: { + // TODO use the workflows fields to do this + face_image: request.prompts.video.image, + + // what we do here is that we generate an "idle" video + // now, the current driving video is just a dummy one I made for testing + // we can replace it by something better, that would reflect the current + // pacing of the scene (news anchor, peaceful dialogue, intense, aggressive etc) + driving_video: sampleDrivingVideo, + + // Select every nth frame from the driving video. Set to 1 to use all frames. + // default: 1 + video_select_every_n_frames: 1, + + // Size of the output image + // min: 64, max: 2048 + // default: 512 + live_portrait_dsize: 512, + + // Scaling factor for the face + // min: 1, max: 4 + // default: 2.3 + live_portrait_scale: 2.3, + + + // Enable stitching + // default: true + live_portrait_stitching: true, + + // Use relative positioning + // default: true + live_portrait_relative: true, + + // there are a lot of other params, check them here: + // https://replicate.com/fofr/live-portrait + + + disable_safety_checker: + !request.settings.censorNotForAllAudiencesContent, + }, + } + )) as any + segment.assetUrl = `${response[0] || ''}` + } else { + const response = (await replicate.run( + request.settings.videoGenerationWorkflow.data as any, + { + input: { + image: request.prompts.video.image, + disable_safety_checker: + !request.settings.censorNotForAllAudiencesContent, + }, + } + )) as any + segment.assetUrl = `${response[0] || ''}` + } } else { throw new Error( `Clapper doesn't support ${request.segment.category} generation for provider "Replicate". Please open a pull request with (working code) to solve this!` diff --git a/packages/app/src/app/api/resolve/route.ts b/packages/app/src/app/api/resolve/route.ts index 46a5f5b3..2d32ba3d 100644 --- a/packages/app/src/app/api/resolve/route.ts +++ b/packages/app/src/app/api/resolve/route.ts @@ -177,30 +177,29 @@ export async function POST(req: NextRequest) { const faceSwap: ProviderFn | undefined = faceswapProviders[faceswapProvider] || undefined - if (faceSwap) { - try { - await faceSwap(request) + if (faceSwap) { + try { + await faceSwap(request) - // we clean-up and parse the output from all the resolvers: - // this will download files hosted on CDNs, convert WAV files to MP3 etc + // we clean-up and parse the output from all the resolvers: + // this will download files hosted on CDNs, convert WAV files to MP3 etc - segment.assetUrl = await decodeOutput(segment.assetUrl) + segment.assetUrl = await decodeOutput(segment.assetUrl) - segment.assetSourceType = getClapAssetSourceType(segment.assetUrl) + segment.assetSourceType = getClapAssetSourceType(segment.assetUrl) - segment.status = ClapSegmentStatus.COMPLETED + segment.status = ClapSegmentStatus.COMPLETED - const { assetFileFormat, outputType } = getTypeAndExtension( - segment.assetUrl - ) + const { assetFileFormat, outputType } = getTypeAndExtension( + segment.assetUrl + ) - segment.assetFileFormat = assetFileFormat - segment.outputType = outputType - - } catch (err) { - console.error(`failed to run the faceswap (${err})`) - } + segment.assetFileFormat = assetFileFormat + segment.outputType = outputType + } catch (err) { + console.error(`failed to run the faceswap (${err})`) } + } } return NextResponse.json(segment) diff --git a/packages/app/src/components/toolbars/top-menu/lists/AssistantWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/AssistantWorkflows.tsx index 361f8f6d..4f95a82d 100644 --- a/packages/app/src/components/toolbars/top-menu/lists/AssistantWorkflows.tsx +++ b/packages/app/src/components/toolbars/top-menu/lists/AssistantWorkflows.tsx @@ -47,11 +47,13 @@ export function AssistantWorkflows() { ai assistant
- {workflow?.provider && } + {workflow?.provider && ( + + )}
{workflow?.label || 'None'}
diff --git a/packages/app/src/components/toolbars/top-menu/lists/ImageDepthWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/ImageDepthWorkflows.tsx index e9f7894b..69d9324d 100644 --- a/packages/app/src/components/toolbars/top-menu/lists/ImageDepthWorkflows.tsx +++ b/packages/app/src/components/toolbars/top-menu/lists/ImageDepthWorkflows.tsx @@ -47,11 +47,13 @@ export function ImageDepthWorkflows() { depth mapper
- {workflow?.provider && } + {workflow?.provider && ( + + )}
{workflow?.label || 'None'}
diff --git a/packages/app/src/components/toolbars/top-menu/lists/ImageFaceswapWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/ImageFaceswapWorkflows.tsx index 57721b7a..87b3fcbd 100644 --- a/packages/app/src/components/toolbars/top-menu/lists/ImageFaceswapWorkflows.tsx +++ b/packages/app/src/components/toolbars/top-menu/lists/ImageFaceswapWorkflows.tsx @@ -49,11 +49,13 @@ export function ImageFaceswapWorkflows() { face swap
- {workflow?.provider && } + {workflow?.provider && ( + + )}
{workflow?.label || 'None'}
diff --git a/packages/app/src/components/toolbars/top-menu/lists/ImageGenerationWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/ImageGenerationWorkflows.tsx index 404a5114..f9df57fd 100644 --- a/packages/app/src/components/toolbars/top-menu/lists/ImageGenerationWorkflows.tsx +++ b/packages/app/src/components/toolbars/top-menu/lists/ImageGenerationWorkflows.tsx @@ -59,11 +59,13 @@ export function ImageGenerationWorkflows() { generate image
- {workflow?.provider && } + {workflow?.provider && ( + + )}
{workflow?.label || 'None'}
diff --git a/packages/app/src/components/toolbars/top-menu/lists/ImageSegmentationWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/ImageSegmentationWorkflows.tsx index 4ecd6322..ac939398 100644 --- a/packages/app/src/components/toolbars/top-menu/lists/ImageSegmentationWorkflows.tsx +++ b/packages/app/src/components/toolbars/top-menu/lists/ImageSegmentationWorkflows.tsx @@ -51,11 +51,13 @@ export function ImageSegmentationWorkflows() { segmentation
- {workflow?.provider && } + {workflow?.provider && ( + + )}
{workflow?.label || 'None'}
diff --git a/packages/app/src/components/toolbars/top-menu/lists/ImageUpscalingWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/ImageUpscalingWorkflows.tsx index 01f7ce6c..a1c70afb 100644 --- a/packages/app/src/components/toolbars/top-menu/lists/ImageUpscalingWorkflows.tsx +++ b/packages/app/src/components/toolbars/top-menu/lists/ImageUpscalingWorkflows.tsx @@ -49,11 +49,13 @@ export function ImageUpscalingWorkflows() { upscale image
- {workflow?.provider && } + {workflow?.provider && ( + + )}
{workflow?.label || 'None'}
diff --git a/packages/app/src/components/toolbars/top-menu/lists/MusicGenerationWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/MusicGenerationWorkflows.tsx index 8194e1f4..f0c68197 100644 --- a/packages/app/src/components/toolbars/top-menu/lists/MusicGenerationWorkflows.tsx +++ b/packages/app/src/components/toolbars/top-menu/lists/MusicGenerationWorkflows.tsx @@ -49,11 +49,13 @@ export function MusicGenerationWorkflows() { generate music
- {workflow?.provider && } + {workflow?.provider && ( + + )}
{workflow?.label || 'None'}
diff --git a/packages/app/src/components/toolbars/top-menu/lists/SoundGenerationWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/SoundGenerationWorkflows.tsx index eb63032d..5b203b28 100644 --- a/packages/app/src/components/toolbars/top-menu/lists/SoundGenerationWorkflows.tsx +++ b/packages/app/src/components/toolbars/top-menu/lists/SoundGenerationWorkflows.tsx @@ -49,11 +49,13 @@ export function SoundGenerationWorkflows() { generate sound
- {workflow?.provider && } + {workflow?.provider && ( + + )}
{workflow?.label || 'None'}
diff --git a/packages/app/src/components/toolbars/top-menu/lists/VideoDepthWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/VideoDepthWorkflows.tsx index 24fa0055..1dd598f7 100644 --- a/packages/app/src/components/toolbars/top-menu/lists/VideoDepthWorkflows.tsx +++ b/packages/app/src/components/toolbars/top-menu/lists/VideoDepthWorkflows.tsx @@ -47,11 +47,13 @@ export function VideoDepthWorkflows() { depth mapper
- {workflow?.provider && } + {workflow?.provider && ( + + )}
{workflow?.label || 'None'}
diff --git a/packages/app/src/components/toolbars/top-menu/lists/VideoGenerationWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/VideoGenerationWorkflows.tsx index ad3ceb55..650e3088 100644 --- a/packages/app/src/components/toolbars/top-menu/lists/VideoGenerationWorkflows.tsx +++ b/packages/app/src/components/toolbars/top-menu/lists/VideoGenerationWorkflows.tsx @@ -49,11 +49,13 @@ export function VideoGenerationWorkflows() { generate video
- {workflow?.provider && } + {workflow?.provider && ( + + )}
{workflow?.label || 'None'}
diff --git a/packages/app/src/components/toolbars/top-menu/lists/VideoSegmentationWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/VideoSegmentationWorkflows.tsx index 94478193..0dd4f3d8 100644 --- a/packages/app/src/components/toolbars/top-menu/lists/VideoSegmentationWorkflows.tsx +++ b/packages/app/src/components/toolbars/top-menu/lists/VideoSegmentationWorkflows.tsx @@ -51,11 +51,13 @@ export function VideoSegmentationWorkflows() { segmentation
- {workflow?.provider && } + {workflow?.provider && ( + + )}
{workflow?.label || 'None'}
diff --git a/packages/app/src/components/toolbars/top-menu/lists/VideoUpscalingWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/VideoUpscalingWorkflows.tsx index 2408cbfc..e32d3d4e 100644 --- a/packages/app/src/components/toolbars/top-menu/lists/VideoUpscalingWorkflows.tsx +++ b/packages/app/src/components/toolbars/top-menu/lists/VideoUpscalingWorkflows.tsx @@ -49,11 +49,13 @@ export function VideoUpscalingWorkflows() { upscale video
- {workflow?.provider && } + {workflow?.provider && ( + + )}
{workflow?.label || 'None'}
diff --git a/packages/app/src/components/toolbars/top-menu/lists/VoiceGenerationWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/VoiceGenerationWorkflows.tsx index c4fad92a..dd7d4b6f 100644 --- a/packages/app/src/components/toolbars/top-menu/lists/VoiceGenerationWorkflows.tsx +++ b/packages/app/src/components/toolbars/top-menu/lists/VoiceGenerationWorkflows.tsx @@ -49,11 +49,13 @@ export function VoiceGenerationWorkflows() { generate voice
- {workflow?.provider && } + {workflow?.provider && ( + + )}
{workflow?.label || 'None'}
diff --git a/packages/app/src/services/editors/workflow-editor/workflows/common/defaultValues.ts b/packages/app/src/services/editors/workflow-editor/workflows/common/defaultValues.ts index d56476b7..1f5e5031 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/common/defaultValues.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/common/defaultValues.ts @@ -157,6 +157,26 @@ export const genericVideoUrl: ClapInputField = { defaultValue: '', } +export const genericFaceImage: ClapInputField = { + id: 'face_image', + label: 'Face Image URL', + description: 'Face Image URL', + category: ClapInputCategory.IMAGE_URL, + type: 'string', + allowedValues: [], + defaultValue: '', +} + +export const genericDrivingVideo: ClapInputField = { + id: 'driving_video', + label: 'Driving Video URL', + description: 'Driving Video URL', + category: ClapInputCategory.VIDEO_URL, + type: 'string', + allowedValues: [], + defaultValue: '', +} + export const genericVoice: ClapInputField = { id: 'voice', label: 'Voice', diff --git a/packages/app/src/services/editors/workflow-editor/workflows/replicate/defaultWorkflows.ts b/packages/app/src/services/editors/workflow-editor/workflows/replicate/defaultWorkflows.ts index e7f4b937..47d08efe 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/replicate/defaultWorkflows.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/replicate/defaultWorkflows.ts @@ -7,9 +7,12 @@ import { import { genericBaseImageUrl, + genericDrivingVideo, + genericFaceImage, genericHeight1024, genericHeight2048, genericImage, + genericImageUrl, genericLora, genericPrompt, genericSwapImage, @@ -27,6 +30,29 @@ import { // -> we can create a ticket to fix this // ------------------------------------------------------------------------------ export const defaultWorkflows: ClapWorkflow[] = [ + { + id: 'replicate://fofr/live-portrait', + label: 'Live Portrait by @fofr', + description: '', + tags: ['live portrait'], + author: '@fofr', + thumbnailUrl: '', + nonCommercial: true, + engine: ClapWorkflowEngine.REST_API, + category: ClapWorkflowCategory.VIDEO_GENERATION, + provider: ClapWorkflowProvider.REPLICATE, + data: 'fofr/live-portrait:067dd98cc3e5cb396c4a9efb4bba3eec6c4a9d271211325c477518fc6485e146', + schema: '', + inputFields: [genericFaceImage, genericDrivingVideo], + inputValues: { + [genericFaceImage.id]: genericFaceImage.defaultValue, + [genericDrivingVideo.id]: genericDrivingVideo.defaultValue, + + // there are a lot of other params, check them here: + // https://replicate.com/fofr/live-portrait + + }, + }, { id: 'replicate://cdingram/face-swap', label: 'Face Swap by @cdingram',