diff --git a/packages/app/src/app/api/resolve/providers/falai/index.ts b/packages/app/src/app/api/resolve/providers/falai/index.ts
index 435ee0f0..ecb65741 100644
--- a/packages/app/src/app/api/resolve/providers/falai/index.ts
+++ b/packages/app/src/app/api/resolve/providers/falai/index.ts
@@ -10,7 +10,7 @@ import {
FalAiVideoResponse,
} from './types'
import { getWorkflowInputValues } from '../getWorkflowInputValues'
-import { sampleVoice } from '@/lib/core/constants'
+import { sampleDrivingVideo, sampleVoice } from '@/lib/core/constants'
import { getWorkflowLora } from '@/services/editors/workflow-editor/workflows/common/loras/getWorkflowLora'
export async function resolveSegment(
@@ -174,7 +174,38 @@ export async function resolveSegment(
model = request.settings.videoGenerationWorkflow.data || ''
// console.log(`request.settings.falAiModelForVideo = `, request.settings.falAiModelForVideo)
- if (model !== 'fal-ai/stable-video') {
+ if (model === 'fal-ai/live-portrait') {
+ const result = (await fal.run(model, {
+ input: {
+ image_url: request.prompts.video.image,
+
+ // what we do here is that we generate an "idle" video
+ // now, the current driving video is just a dummy one I made for testing
+ // we can replace it by something better, that would reflect the current
+ // pacing of the scene (news anchor, peaceful dialogue, intense, aggressive etc)
+ video_url: sampleDrivingVideo,
+
+ sync_mode: true,
+ enable_safety_checker:
+ request.settings.censorNotForAllAudiencesContent,
+ },
+ })) as FalAiVideoResponse
+
+ if (request.settings.censorNotForAllAudiencesContent) {
+ if (
+ Array.isArray(result.has_nsfw_concepts) &&
+ result.has_nsfw_concepts.includes(true)
+ ) {
+ throw new Error(
+ `The generated content has been filtered according to your safety settings`
+ )
+ }
+ }
+
+ console.log('live portrait result:', result)
+
+ segment.assetUrl = result?.video?.url || ''
+ } else if (model !== 'fal-ai/stable-video') {
throw new Error(
`only "fal-ai/stable-video" is supported by Clapper for the moment`
)
diff --git a/packages/app/src/app/api/resolve/providers/replicate/index.ts b/packages/app/src/app/api/resolve/providers/replicate/index.ts
index 46ef034d..9230e2df 100644
--- a/packages/app/src/app/api/resolve/providers/replicate/index.ts
+++ b/packages/app/src/app/api/resolve/providers/replicate/index.ts
@@ -5,6 +5,7 @@ import { ResolveRequest } from '@aitube/clapper-services'
import { TimelineSegment } from '@aitube/timeline'
import { getWorkflowInputValues } from '../getWorkflowInputValues'
import { getWorkflowLora } from '@/services/editors/workflow-editor/workflows/common/loras/getWorkflowLora'
+import { sampleDrivingVideo } from '@/lib/core/constants'
export async function resolveSegment(
request: ResolveRequest
@@ -21,7 +22,6 @@ export async function resolveSegment(
request.settings.imageGenerationWorkflow
)
-
const aspectRatio =
request.meta.orientation === ClapMediaOrientation.SQUARE
? '1:1'
@@ -99,17 +99,68 @@ export async function resolveSegment(
)) as any
segment.assetUrl = `${response[0] || ''}`
} else if (request.segment.category === ClapSegmentCategory.VIDEO) {
- const response = (await replicate.run(
- request.settings.videoGenerationWorkflow.data as any,
- {
- input: {
- image: request.prompts.video.image,
- disable_safety_checker:
- !request.settings.censorNotForAllAudiencesContent,
- },
- }
- )) as any
- segment.assetUrl = `${response[0] || ''}`
+ const model = request.settings.videoGenerationWorkflow.data as any
+
+ if (model.startsWith("fofr/live-portrait")) {
+ const response = (await replicate.run(
+ request.settings.videoGenerationWorkflow.data as any,
+ {
+ input: {
+ // TODO use the workflows fields to do this
+ face_image: request.prompts.video.image,
+
+ // what we do here is that we generate an "idle" video
+ // now, the current driving video is just a dummy one I made for testing
+ // we can replace it by something better, that would reflect the current
+ // pacing of the scene (news anchor, peaceful dialogue, intense, aggressive etc)
+ driving_video: sampleDrivingVideo,
+
+ // Select every nth frame from the driving video. Set to 1 to use all frames.
+ // default: 1
+ video_select_every_n_frames: 1,
+
+ // Size of the output image
+ // min: 64, max: 2048
+ // default: 512
+ live_portrait_dsize: 512,
+
+ // Scaling factor for the face
+ // min: 1, max: 4
+ // default: 2.3
+ live_portrait_scale: 2.3,
+
+
+ // Enable stitching
+ // default: true
+ live_portrait_stitching: true,
+
+ // Use relative positioning
+ // default: true
+ live_portrait_relative: true,
+
+ // there are a lot of other params, check them here:
+ // https://replicate.com/fofr/live-portrait
+
+
+ disable_safety_checker:
+ !request.settings.censorNotForAllAudiencesContent,
+ },
+ }
+ )) as any
+ segment.assetUrl = `${response[0] || ''}`
+ } else {
+ const response = (await replicate.run(
+ request.settings.videoGenerationWorkflow.data as any,
+ {
+ input: {
+ image: request.prompts.video.image,
+ disable_safety_checker:
+ !request.settings.censorNotForAllAudiencesContent,
+ },
+ }
+ )) as any
+ segment.assetUrl = `${response[0] || ''}`
+ }
} else {
throw new Error(
`Clapper doesn't support ${request.segment.category} generation for provider "Replicate". Please open a pull request with (working code) to solve this!`
diff --git a/packages/app/src/app/api/resolve/route.ts b/packages/app/src/app/api/resolve/route.ts
index 46a5f5b3..2d32ba3d 100644
--- a/packages/app/src/app/api/resolve/route.ts
+++ b/packages/app/src/app/api/resolve/route.ts
@@ -177,30 +177,29 @@ export async function POST(req: NextRequest) {
const faceSwap: ProviderFn | undefined =
faceswapProviders[faceswapProvider] || undefined
- if (faceSwap) {
- try {
- await faceSwap(request)
+ if (faceSwap) {
+ try {
+ await faceSwap(request)
- // we clean-up and parse the output from all the resolvers:
- // this will download files hosted on CDNs, convert WAV files to MP3 etc
+ // we clean-up and parse the output from all the resolvers:
+ // this will download files hosted on CDNs, convert WAV files to MP3 etc
- segment.assetUrl = await decodeOutput(segment.assetUrl)
+ segment.assetUrl = await decodeOutput(segment.assetUrl)
- segment.assetSourceType = getClapAssetSourceType(segment.assetUrl)
+ segment.assetSourceType = getClapAssetSourceType(segment.assetUrl)
- segment.status = ClapSegmentStatus.COMPLETED
+ segment.status = ClapSegmentStatus.COMPLETED
- const { assetFileFormat, outputType } = getTypeAndExtension(
- segment.assetUrl
- )
+ const { assetFileFormat, outputType } = getTypeAndExtension(
+ segment.assetUrl
+ )
- segment.assetFileFormat = assetFileFormat
- segment.outputType = outputType
-
- } catch (err) {
- console.error(`failed to run the faceswap (${err})`)
- }
+ segment.assetFileFormat = assetFileFormat
+ segment.outputType = outputType
+ } catch (err) {
+ console.error(`failed to run the faceswap (${err})`)
}
+ }
}
return NextResponse.json(segment)
diff --git a/packages/app/src/components/toolbars/top-menu/lists/AssistantWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/AssistantWorkflows.tsx
index 361f8f6d..4f95a82d 100644
--- a/packages/app/src/components/toolbars/top-menu/lists/AssistantWorkflows.tsx
+++ b/packages/app/src/components/toolbars/top-menu/lists/AssistantWorkflows.tsx
@@ -47,11 +47,13 @@ export function AssistantWorkflows() {
ai assistant
- {workflow?.provider &&
}
+ {workflow?.provider && (
+
+ )}
{workflow?.label || 'None'}
diff --git a/packages/app/src/components/toolbars/top-menu/lists/ImageDepthWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/ImageDepthWorkflows.tsx
index e9f7894b..69d9324d 100644
--- a/packages/app/src/components/toolbars/top-menu/lists/ImageDepthWorkflows.tsx
+++ b/packages/app/src/components/toolbars/top-menu/lists/ImageDepthWorkflows.tsx
@@ -47,11 +47,13 @@ export function ImageDepthWorkflows() {
depth mapper
- {workflow?.provider &&
}
+ {workflow?.provider && (
+
+ )}
{workflow?.label || 'None'}
diff --git a/packages/app/src/components/toolbars/top-menu/lists/ImageFaceswapWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/ImageFaceswapWorkflows.tsx
index 57721b7a..87b3fcbd 100644
--- a/packages/app/src/components/toolbars/top-menu/lists/ImageFaceswapWorkflows.tsx
+++ b/packages/app/src/components/toolbars/top-menu/lists/ImageFaceswapWorkflows.tsx
@@ -49,11 +49,13 @@ export function ImageFaceswapWorkflows() {
face swap
- {workflow?.provider &&
}
+ {workflow?.provider && (
+
+ )}
{workflow?.label || 'None'}
diff --git a/packages/app/src/components/toolbars/top-menu/lists/ImageGenerationWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/ImageGenerationWorkflows.tsx
index 404a5114..f9df57fd 100644
--- a/packages/app/src/components/toolbars/top-menu/lists/ImageGenerationWorkflows.tsx
+++ b/packages/app/src/components/toolbars/top-menu/lists/ImageGenerationWorkflows.tsx
@@ -59,11 +59,13 @@ export function ImageGenerationWorkflows() {
generate image
- {workflow?.provider &&
}
+ {workflow?.provider && (
+
+ )}
{workflow?.label || 'None'}
diff --git a/packages/app/src/components/toolbars/top-menu/lists/ImageSegmentationWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/ImageSegmentationWorkflows.tsx
index 4ecd6322..ac939398 100644
--- a/packages/app/src/components/toolbars/top-menu/lists/ImageSegmentationWorkflows.tsx
+++ b/packages/app/src/components/toolbars/top-menu/lists/ImageSegmentationWorkflows.tsx
@@ -51,11 +51,13 @@ export function ImageSegmentationWorkflows() {
segmentation
- {workflow?.provider &&
}
+ {workflow?.provider && (
+
+ )}
{workflow?.label || 'None'}
diff --git a/packages/app/src/components/toolbars/top-menu/lists/ImageUpscalingWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/ImageUpscalingWorkflows.tsx
index 01f7ce6c..a1c70afb 100644
--- a/packages/app/src/components/toolbars/top-menu/lists/ImageUpscalingWorkflows.tsx
+++ b/packages/app/src/components/toolbars/top-menu/lists/ImageUpscalingWorkflows.tsx
@@ -49,11 +49,13 @@ export function ImageUpscalingWorkflows() {
upscale image
- {workflow?.provider &&
}
+ {workflow?.provider && (
+
+ )}
{workflow?.label || 'None'}
diff --git a/packages/app/src/components/toolbars/top-menu/lists/MusicGenerationWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/MusicGenerationWorkflows.tsx
index 8194e1f4..f0c68197 100644
--- a/packages/app/src/components/toolbars/top-menu/lists/MusicGenerationWorkflows.tsx
+++ b/packages/app/src/components/toolbars/top-menu/lists/MusicGenerationWorkflows.tsx
@@ -49,11 +49,13 @@ export function MusicGenerationWorkflows() {
generate music
- {workflow?.provider &&
}
+ {workflow?.provider && (
+
+ )}
{workflow?.label || 'None'}
diff --git a/packages/app/src/components/toolbars/top-menu/lists/SoundGenerationWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/SoundGenerationWorkflows.tsx
index eb63032d..5b203b28 100644
--- a/packages/app/src/components/toolbars/top-menu/lists/SoundGenerationWorkflows.tsx
+++ b/packages/app/src/components/toolbars/top-menu/lists/SoundGenerationWorkflows.tsx
@@ -49,11 +49,13 @@ export function SoundGenerationWorkflows() {
generate sound
- {workflow?.provider &&
}
+ {workflow?.provider && (
+
+ )}
{workflow?.label || 'None'}
diff --git a/packages/app/src/components/toolbars/top-menu/lists/VideoDepthWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/VideoDepthWorkflows.tsx
index 24fa0055..1dd598f7 100644
--- a/packages/app/src/components/toolbars/top-menu/lists/VideoDepthWorkflows.tsx
+++ b/packages/app/src/components/toolbars/top-menu/lists/VideoDepthWorkflows.tsx
@@ -47,11 +47,13 @@ export function VideoDepthWorkflows() {
depth mapper
- {workflow?.provider &&
}
+ {workflow?.provider && (
+
+ )}
{workflow?.label || 'None'}
diff --git a/packages/app/src/components/toolbars/top-menu/lists/VideoGenerationWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/VideoGenerationWorkflows.tsx
index ad3ceb55..650e3088 100644
--- a/packages/app/src/components/toolbars/top-menu/lists/VideoGenerationWorkflows.tsx
+++ b/packages/app/src/components/toolbars/top-menu/lists/VideoGenerationWorkflows.tsx
@@ -49,11 +49,13 @@ export function VideoGenerationWorkflows() {
generate video
- {workflow?.provider &&
}
+ {workflow?.provider && (
+
+ )}
{workflow?.label || 'None'}
diff --git a/packages/app/src/components/toolbars/top-menu/lists/VideoSegmentationWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/VideoSegmentationWorkflows.tsx
index 94478193..0dd4f3d8 100644
--- a/packages/app/src/components/toolbars/top-menu/lists/VideoSegmentationWorkflows.tsx
+++ b/packages/app/src/components/toolbars/top-menu/lists/VideoSegmentationWorkflows.tsx
@@ -51,11 +51,13 @@ export function VideoSegmentationWorkflows() {
segmentation
- {workflow?.provider &&
}
+ {workflow?.provider && (
+
+ )}
{workflow?.label || 'None'}
diff --git a/packages/app/src/components/toolbars/top-menu/lists/VideoUpscalingWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/VideoUpscalingWorkflows.tsx
index 2408cbfc..e32d3d4e 100644
--- a/packages/app/src/components/toolbars/top-menu/lists/VideoUpscalingWorkflows.tsx
+++ b/packages/app/src/components/toolbars/top-menu/lists/VideoUpscalingWorkflows.tsx
@@ -49,11 +49,13 @@ export function VideoUpscalingWorkflows() {
upscale video
- {workflow?.provider &&
}
+ {workflow?.provider && (
+
+ )}
{workflow?.label || 'None'}
diff --git a/packages/app/src/components/toolbars/top-menu/lists/VoiceGenerationWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/VoiceGenerationWorkflows.tsx
index c4fad92a..dd7d4b6f 100644
--- a/packages/app/src/components/toolbars/top-menu/lists/VoiceGenerationWorkflows.tsx
+++ b/packages/app/src/components/toolbars/top-menu/lists/VoiceGenerationWorkflows.tsx
@@ -49,11 +49,13 @@ export function VoiceGenerationWorkflows() {
generate voice
- {workflow?.provider &&
}
+ {workflow?.provider && (
+
+ )}
{workflow?.label || 'None'}
diff --git a/packages/app/src/services/editors/workflow-editor/workflows/common/defaultValues.ts b/packages/app/src/services/editors/workflow-editor/workflows/common/defaultValues.ts
index d56476b7..1f5e5031 100644
--- a/packages/app/src/services/editors/workflow-editor/workflows/common/defaultValues.ts
+++ b/packages/app/src/services/editors/workflow-editor/workflows/common/defaultValues.ts
@@ -157,6 +157,26 @@ export const genericVideoUrl: ClapInputField = {
defaultValue: '',
}
+export const genericFaceImage: ClapInputField = {
+ id: 'face_image',
+ label: 'Face Image URL',
+ description: 'Face Image URL',
+ category: ClapInputCategory.IMAGE_URL,
+ type: 'string',
+ allowedValues: [],
+ defaultValue: '',
+}
+
+export const genericDrivingVideo: ClapInputField = {
+ id: 'driving_video',
+ label: 'Driving Video URL',
+ description: 'Driving Video URL',
+ category: ClapInputCategory.VIDEO_URL,
+ type: 'string',
+ allowedValues: [],
+ defaultValue: '',
+}
+
export const genericVoice: ClapInputField = {
id: 'voice',
label: 'Voice',
diff --git a/packages/app/src/services/editors/workflow-editor/workflows/replicate/defaultWorkflows.ts b/packages/app/src/services/editors/workflow-editor/workflows/replicate/defaultWorkflows.ts
index e7f4b937..47d08efe 100644
--- a/packages/app/src/services/editors/workflow-editor/workflows/replicate/defaultWorkflows.ts
+++ b/packages/app/src/services/editors/workflow-editor/workflows/replicate/defaultWorkflows.ts
@@ -7,9 +7,12 @@ import {
import {
genericBaseImageUrl,
+ genericDrivingVideo,
+ genericFaceImage,
genericHeight1024,
genericHeight2048,
genericImage,
+ genericImageUrl,
genericLora,
genericPrompt,
genericSwapImage,
@@ -27,6 +30,29 @@ import {
// -> we can create a ticket to fix this
// ------------------------------------------------------------------------------
export const defaultWorkflows: ClapWorkflow[] = [
+ {
+ id: 'replicate://fofr/live-portrait',
+ label: 'Live Portrait by @fofr',
+ description: '',
+ tags: ['live portrait'],
+ author: '@fofr',
+ thumbnailUrl: '',
+ nonCommercial: true,
+ engine: ClapWorkflowEngine.REST_API,
+ category: ClapWorkflowCategory.VIDEO_GENERATION,
+ provider: ClapWorkflowProvider.REPLICATE,
+ data: 'fofr/live-portrait:067dd98cc3e5cb396c4a9efb4bba3eec6c4a9d271211325c477518fc6485e146',
+ schema: '',
+ inputFields: [genericFaceImage, genericDrivingVideo],
+ inputValues: {
+ [genericFaceImage.id]: genericFaceImage.defaultValue,
+ [genericDrivingVideo.id]: genericDrivingVideo.defaultValue,
+
+ // there are a lot of other params, check them here:
+ // https://replicate.com/fofr/live-portrait
+
+ },
+ },
{
id: 'replicate://cdingram/face-swap',
label: 'Face Swap by @cdingram',