Skip to content

Commit 193dbf1

Browse files
jbilcke-hfjbilcke-hf
committed
fix replicate
1 parent 17ee0ab commit 193dbf1

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+1023
-337
lines changed

‎bun.lockb

472 Bytes
Binary file not shown.

‎package.json

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,12 @@
11
{
22
"name": "@aitube/clapper-monorepo",
33
"version": "0.2.4",
4-
"private": true,
54
"description": "A monorepo for the Clapper project. Individual packages are in the packages directory.",
6-
"workspaces": [
7-
"packages/clap",
8-
"packages/timeline",
9-
"packages/api-client",
10-
"packages/io",
11-
"packages/colors",
12-
"packages/engine",
13-
"packages/broadway",
14-
"packages/clapper-services",
15-
"packages/app"
16-
],
175
"engines": {
186
"bun": ">=1.0.0"
197
},
8+
"packageManager": "[email protected]",
9+
"private": true,
2010
"scripts": {
2111
"dev": "bun run --cwd packages/app dev",
2212
"start": "bun run --cwd packages/app start",
@@ -36,10 +26,20 @@
3626
"test:all": "bun run --cwd packages/clap test && bun run --cwd packages/timeline test && bun run --cwd packages/api-client test && bun run --cwd packages/io test && bun run --cwd packages/colors test && bun run --cwd packages/engine test && bun run --cwd packages/broadway test && bun run --cwd packages/clapper-services test && bun run --cwd packages/app test",
3727
"format": "bun run --cwd packages/app format"
3828
},
39-
"packageManager": "[email protected]",
4029
"trustedDependencies": [
4130
"@aitube/clapper",
4231
"onnxruntime-node",
4332
"protobufjs"
33+
],
34+
"workspaces": [
35+
"packages/clap",
36+
"packages/timeline",
37+
"packages/api-client",
38+
"packages/io",
39+
"packages/colors",
40+
"packages/engine",
41+
"packages/broadway",
42+
"packages/clapper-services",
43+
"packages/app"
4444
]
4545
}

‎packages/api-client/package.json

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@
3939
"dist/**/*.d.ts"
4040
],
4141
"dependencies": {
42+
"@aitube/clap": "workspace:*",
43+
"@types/bun": "latest",
4244
"query-string": "^9.0.0"
4345
}
4446
}

‎packages/app/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@
4141
"@aitube/clapper-services": "workspace:*",
4242
"@aitube/engine": "workspace:*",
4343
"@aitube/timeline": "workspace:*",
44-
"@fal-ai/serverless-client": "^0.13.0",
44+
"@fal-ai/serverless-client": "^0.14.2",
4545
"@ffmpeg/ffmpeg": "^0.12.10",
4646
"@ffmpeg/util": "^0.12.1",
4747
"@gradio/client": "^1.5.0",

‎packages/app/src/app/api/resolve/providers/replicate/index.ts

Lines changed: 57 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
11
import Replicate from 'replicate'
22

3-
import { ClapSegmentCategory } from '@aitube/clap'
3+
import { ClapMediaOrientation, ClapSegmentCategory } from '@aitube/clap'
44
import { ResolveRequest } from '@aitube/clapper-services'
55
import { TimelineSegment } from '@aitube/timeline'
6+
import { getWorkflowInputValues } from '../getWorkflowInputValues'
7+
import { defaultLoraModels } from '@/services/editors/workflow-editor/workflows/common/loras'
8+
import { getWorkflowLora } from '@/services/editors/workflow-editor/workflows/common/loras/getWorkflowLora'
69

710
export async function resolveSegment(
811
request: ResolveRequest
@@ -12,29 +15,57 @@ export async function resolveSegment(
1215
}
1316
const replicate = new Replicate({ auth: request.settings.replicateApiKey })
1417

15-
if (request.segment.category !== ClapSegmentCategory.STORYBOARD) {
16-
throw new Error(
17-
`Clapper doesn't support ${request.segment.category} generation for provider "Replicate". Please open a pull request with (working code) to solve this!`
18-
)
19-
}
20-
2118
const segment = request.segment
2219

23-
// this mapping isn't great, we should use something auto-adapting
24-
// like we are doing for Hugging Face (match the fields etc)
25-
if (request.segment.category === ClapSegmentCategory.STORYBOARD) {
20+
if (request.segment.category == ClapSegmentCategory.STORYBOARD) {
21+
22+
const { workflowValues } = getWorkflowInputValues(
23+
request.settings.imageGenerationWorkflow
24+
)
25+
2626
let params: object = {
2727
prompt: request.prompts.image.positive,
2828
width: request.meta.width,
2929
height: request.meta.height,
30+
disable_safety_checker: !request.settings.censorNotForAllAudiencesContent,
3031
}
32+
33+
const aspectRatio =
34+
request.meta.orientation === ClapMediaOrientation.SQUARE
35+
? "1:1"
36+
: request.meta.orientation === ClapMediaOrientation.PORTRAIT
37+
? "9:16"
38+
: "16:9"
39+
3140
if (
3241
request.settings.imageGenerationWorkflow.data === 'fofr/pulid-lightning'
3342
) {
3443
params = {
3544
...params,
3645
face_image: request.prompts.image.identity,
3746
}
47+
} else if (
48+
request.settings.imageGenerationWorkflow.data === 'lucataco/flux-dev-lora'
49+
) {
50+
51+
// note: this isn't the right place to do this, because maybe the LoRAs are dynamic
52+
const loraModel = getWorkflowLora(request.settings.imageGenerationWorkflow)
53+
54+
params = {
55+
// for some reason this model doesn't support arbitrary width and height,
56+
// at least not at the time of writing..
57+
aspect_ratio: aspectRatio,
58+
59+
hf_lora: workflowValues['hf_lora'] || '',
60+
61+
prompt: [
62+
loraModel?.trigger,
63+
request.prompts.image.positive
64+
].filter(x => x).join(' '),
65+
66+
disable_safety_checker: !request.settings.censorNotForAllAudiencesContent,
67+
}
68+
3869
} else if (
3970
request.settings.imageGenerationWorkflow.data === 'zsxkib/pulid'
4071
) {
@@ -43,32 +74,44 @@ export async function resolveSegment(
4374
main_face_image: request.prompts.image.identity,
4475
}
4576
}
77+
78+
/*
79+
console.log("debug:", {
80+
model: request.settings.imageGenerationWorkflow.data,
81+
params,
82+
})
83+
*/
4684
const response = (await replicate.run(
47-
request.settings.imageGenerationWorkflow as any,
85+
request.settings.imageGenerationWorkflow.data as any,
4886
{ input: params }
4987
)) as any
50-
segment.assetUrl = `${response.output || ''}`
88+
89+
90+
segment.assetUrl = `${response[0] || ''}`
91+
5192
} else if (request.segment.category === ClapSegmentCategory.DIALOGUE) {
5293
const response = (await replicate.run(
5394
request.settings.voiceGenerationWorkflow.data as any,
5495
{
5596
input: {
5697
text: request.prompts.voice.positive,
5798
audio: request.prompts.voice.identity,
99+
disable_safety_checker: !request.settings.censorNotForAllAudiencesContent,
58100
},
59101
}
60102
)) as any
61-
segment.assetUrl = `${response.output || ''}`
103+
segment.assetUrl = `${response[0] || ''}`
62104
} else if (request.segment.category === ClapSegmentCategory.VIDEO) {
63105
const response = (await replicate.run(
64106
request.settings.videoGenerationWorkflow.data as any,
65107
{
66108
input: {
67109
image: request.prompts.video.image,
110+
disable_safety_checker: !request.settings.censorNotForAllAudiencesContent,
68111
},
69112
}
70113
)) as any
71-
segment.assetUrl = `${response.output || ''}`
114+
segment.assetUrl = `${response[0] || ''}`
72115
} else {
73116
throw new Error(
74117
`Clapper doesn't support ${request.segment.category} generation for provider "Replicate". Please open a pull request with (working code) to solve this!`

‎packages/app/src/app/api/resolve/route.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,9 @@ export async function POST(req: NextRequest) {
132132
segment.outputType === ClapOutputType.AUDIO ||
133133
segment.outputType === ClapOutputType.VIDEO
134134
) {
135+
136+
137+
// TODO this should be down in the browser side, so that we can scale better
135138
const { durationInMs, hasAudio } = await getMediaInfo(segment.assetUrl)
136139
segment.assetDurationInMs = durationInMs
137140

‎packages/app/src/components/toolbars/top-menu/lists/AssistantWorkflows.tsx

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -21,18 +21,20 @@ import {
2121
ClapWorkflowProviderLogo,
2222
ClapWorkflowProviderName,
2323
} from '@/components/core/providers'
24+
import { parseWorkflow } from '@/services/settings/workflows/parseWorkflow'
25+
26+
const category = ClapWorkflowCategory.ASSISTANT
2427

2528
export function AssistantWorkflows() {
26-
const workflowId = useSettings((s) => s.assistantWorkflow)
27-
const setWorkflowId = useSettings((s) => s.setAssistantWorkflow)
29+
const assistantWorkflow = useSettings((s) => s.assistantWorkflow)
30+
const setAssistantWorkflow = useSettings((s) => s.setAssistantWorkflow)
2831
const availableWorkflows = useWorkflowEditor((s) => s.availableWorkflows)
2932

30-
const { workflows, providers, nbProviders } = findWorkflows(
31-
availableWorkflows,
32-
{ category: ClapWorkflowCategory.ASSISTANT }
33-
)
33+
const { providers, nbProviders } = findWorkflows(availableWorkflows, {
34+
category,
35+
})
3436

35-
const { workflow } = findWorkflows(workflows, { workflowId })
37+
const workflow = parseWorkflow(assistantWorkflow, category)
3638

3739
if (!nbProviders) {
3840
return null
@@ -65,15 +67,15 @@ export function AssistantWorkflows() {
6567
{workflows?.map((w) => (
6668
<MenubarCheckboxItem
6769
key={w.id}
68-
checked={workflowId === w.id}
70+
checked={workflow.id === w.id}
6971
disabled={hasNoPublicAPI(w)}
7072
onClick={(e) => {
7173
if (hasNoPublicAPI(w)) {
7274
e.stopPropagation()
7375
e.preventDefault()
7476
return false
7577
}
76-
setWorkflowId(w.id)
78+
setAssistantWorkflow(w)
7779
e.stopPropagation()
7880
e.preventDefault()
7981
return false

‎packages/app/src/components/toolbars/top-menu/lists/ImageDepthWorkflows.tsx

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -21,18 +21,20 @@ import {
2121
ClapWorkflowProviderLogo,
2222
ClapWorkflowProviderName,
2323
} from '@/components/core/providers'
24+
import { parseWorkflow } from '@/services/settings/workflows/parseWorkflow'
25+
26+
const category = ClapWorkflowCategory.IMAGE_DEPTH_MAPPING
2427

2528
export function ImageDepthWorkflows() {
26-
const workflowId = useSettings((s) => s.imageDepthWorkflow)
27-
const setWorkflowId = useSettings((s) => s.setImageDepthWorkflow)
29+
const imageDepthWorkflow = useSettings((s) => s.imageDepthWorkflow)
30+
const setImageDepthWorkflow = useSettings((s) => s.setImageDepthWorkflow)
2831
const availableWorkflows = useWorkflowEditor((s) => s.availableWorkflows)
2932

30-
const { workflows, providers, nbProviders } = findWorkflows(
31-
availableWorkflows,
32-
{ category: ClapWorkflowCategory.IMAGE_DEPTH_MAPPING }
33-
)
33+
const { providers, nbProviders } = findWorkflows(availableWorkflows, {
34+
category,
35+
})
3436

35-
const { workflow } = findWorkflows(workflows, { workflowId })
37+
const workflow = parseWorkflow(imageDepthWorkflow, category)
3638

3739
if (!nbProviders) {
3840
return null
@@ -65,15 +67,15 @@ export function ImageDepthWorkflows() {
6567
{workflows?.map((w) => (
6668
<MenubarCheckboxItem
6769
key={w.id}
68-
checked={workflowId === w.id}
70+
checked={workflow.id === w.id}
6971
disabled={hasNoPublicAPI(w)}
7072
onClick={(e) => {
7173
if (hasNoPublicAPI(w)) {
7274
e.stopPropagation()
7375
e.preventDefault()
7476
return false
7577
}
76-
setWorkflowId(w.id)
78+
setImageDepthWorkflow(w)
7779
e.stopPropagation()
7880
e.preventDefault()
7981
return false

0 commit comments

Comments
 (0)