Skip to content

Commit a8ff2d1

Browse files
jbilcke-hfjbilcke-hf
committed
refactoring to use workflows
1 parent 8722988 commit a8ff2d1

File tree

70 files changed

+1768
-2608
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

70 files changed

+1768
-2608
lines changed

‎package-lock.json

Lines changed: 113 additions & 724 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -36,15 +36,15 @@
3636
"electron:make": "npm run build && electron-forge make"
3737
},
3838
"dependencies": {
39-
"@aitube/broadway": "0.1.3-1",
40-
"@aitube/clap": "0.1.2",
41-
"@aitube/clapper-services": "0.1.6",
42-
"@aitube/engine": "0.1.2",
43-
"@aitube/timeline": "0.1.3",
39+
"@aitube/broadway": "0.2.0",
40+
"@aitube/clap": "0.2.0",
41+
"@aitube/clapper-services": "0.2.0-2",
42+
"@aitube/engine": "0.2.0",
43+
"@aitube/timeline": "0.2.0",
4444
"@fal-ai/serverless-client": "^0.13.0",
4545
"@ffmpeg/ffmpeg": "^0.12.10",
4646
"@ffmpeg/util": "^0.12.1",
47-
"@gradio/client": "^1.4.0",
47+
"@gradio/client": "^1.5.0",
4848
"@huggingface/hub": "^0.15.1",
4949
"@huggingface/inference": "^2.8.0",
5050
"@langchain/anthropic": "^0.2.12",

‎src/app/api/assistant/askAnyAssistant.ts

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
'use server'
22

3-
import { ClapSegmentCategory } from '@aitube/clap'
3+
import { ClapWorkflowProvider } from '@aitube/clap'
44
import { RunnableLike } from '@langchain/core/runnables'
55
import { ChatPromptValueInterface } from '@langchain/core/prompt_values'
66
import {
@@ -28,7 +28,6 @@ import {
2828
AssistantRequest,
2929
AssistantSceneSegment,
3030
AssistantStoryBlock,
31-
ComputeProvider,
3231
ChatEventVisibility,
3332
} from '@aitube/clapper-services'
3433

@@ -65,7 +64,9 @@ export async function askAnyAssistant({
6564

6665
history = [],
6766
}: AssistantRequest): Promise<AssistantMessage> {
68-
const provider = settings.assistantProvider
67+
const workflow = settings.assistantWorkflow
68+
const provider = workflow.provider
69+
const modelName = workflow.data
6970

7071
if (!provider) {
7172
throw new Error(`Missing assistant provider`)
@@ -74,40 +75,40 @@ export async function askAnyAssistant({
7475
let coerceable:
7576
| undefined
7677
| RunnableLike<ChatPromptValueInterface, AIMessageChunk> =
77-
provider === ComputeProvider.GROQ
78+
provider === ClapWorkflowProvider.GROQ
7879
? new ChatGroq({
7980
apiKey: settings.groqApiKey,
80-
modelName: settings.assistantModel,
81+
modelName,
8182
// temperature: 0.7,
8283
})
83-
: provider === ComputeProvider.OPENAI
84+
: provider === ClapWorkflowProvider.OPENAI
8485
? new ChatOpenAI({
8586
openAIApiKey: settings.openaiApiKey,
86-
modelName: settings.assistantModel,
87+
modelName,
8788
// temperature: 0.7,
8889
})
89-
: provider === ComputeProvider.ANTHROPIC
90+
: provider === ClapWorkflowProvider.ANTHROPIC
9091
? new ChatAnthropic({
9192
anthropicApiKey: settings.anthropicApiKey,
92-
modelName: settings.assistantModel,
93+
modelName,
9394
// temperature: 0.7,
9495
})
95-
: provider === ComputeProvider.COHERE
96+
: provider === ClapWorkflowProvider.COHERE
9697
? new ChatCohere({
9798
apiKey: settings.cohereApiKey,
98-
model: settings.assistantModel,
99+
model: modelName,
99100
// temperature: 0.7,
100101
})
101-
: provider === ComputeProvider.MISTRALAI
102+
: provider === ClapWorkflowProvider.MISTRALAI
102103
? new ChatMistralAI({
103104
apiKey: settings.mistralAiApiKey,
104-
modelName: settings.assistantModel,
105+
modelName,
105106
// temperature: 0.7,
106107
})
107-
: provider === ComputeProvider.GOOGLE
108+
: provider === ClapWorkflowProvider.GOOGLE
108109
? new ChatVertexAI({
109110
apiKey: settings.googleApiKey,
110-
modelName: settings.assistantModel,
111+
modelName,
111112
// temperature: 0.7,
112113
})
113114
: undefined

‎src/app/api/resolve/providers/falai/index.ts

Lines changed: 23 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,14 @@ export async function resolveSegment(
2222

2323
const segment: TimelineSegment = request.segment
2424

25+
let model = request.settings.imageGenerationWorkflow.data || ''
26+
2527
// for doc see:
2628
// https://fal.ai/models/fal-ai/fast-sdxl/api
2729

2830
if (request.segment.category === ClapSegmentCategory.STORYBOARD) {
31+
model = request.settings.imageGenerationWorkflow.data || ''
32+
2933
if (!request.prompts.image.positive) {
3034
console.error(
3135
`resolveSegment: cannot resolve a storyboard with an empty prompt`
@@ -42,18 +46,18 @@ export async function resolveSegment(
4246

4347
let result: FalAiImageResponse | undefined = undefined
4448

45-
if (request.settings.imageGenerationModel === 'fal-ai/pulid') {
49+
if (model === 'fal-ai/pulid') {
4650
if (!request.prompts.image.identity) {
4751
// throw new Error(`you selected model ${request.settings.falAiModelForImage}, but no character was found, so skipping`)
4852
// console.log(`warning: user selected model ${request.settings.falAiModelForImage}, but no character was found. Falling back to fal-ai/flux-pro`)
4953

5054
// dirty fix to fallback to a non-face model
51-
request.settings.imageGenerationModel = 'fal-ai/flux-pro'
55+
model = 'fal-ai/flux-pro'
5256
}
5357
}
5458

55-
if (request.settings.imageGenerationModel === 'fal-ai/pulid') {
56-
result = (await fal.run(request.settings.imageGenerationModel, {
59+
if (model === 'fal-ai/pulid') {
60+
result = (await fal.run(model, {
5761
input: {
5862
reference_images: [
5963
{
@@ -68,16 +72,13 @@ export async function resolveSegment(
6872
},
6973
})) as FalAiImageResponse
7074
} else {
71-
result = (await fal.run(request.settings.imageGenerationModel, {
75+
result = (await fal.run(model, {
7276
input: {
7377
prompt: request.prompts.image.positive,
7478
image_size: imageSize,
7579
sync_mode: true,
7680
num_inference_steps:
77-
request.settings.imageGenerationModel ===
78-
'fal-ai/stable-diffusion-v3-medium'
79-
? 40
80-
: 25,
81+
model === 'fal-ai/stable-diffusion-v3-medium' ? 40 : 25,
8182
num_images: 1,
8283
enable_safety_checker:
8384
request.settings.censorNotForAllAudiencesContent,
@@ -95,8 +96,10 @@ export async function resolveSegment(
9596

9697
segment.assetUrl = result.images[0]?.url || ''
9798
} else if (request.segment.category === ClapSegmentCategory.VIDEO) {
99+
model = request.settings.videoGenerationWorkflow.data || ''
100+
98101
// console.log(`request.settings.falAiModelForVideo = `, request.settings.falAiModelForVideo)
99-
if (request.settings.videoGenerationModel !== 'fal-ai/stable-video') {
102+
if (model !== 'fal-ai/stable-video') {
100103
throw new Error(
101104
`only "fal-ai/stable-video" is supported by Clapper for the moment`
102105
)
@@ -110,7 +113,7 @@ export async function resolveSegment(
110113
`cannot generate a video without a storyboard (the concept of Clapper is to use storyboards)`
111114
)
112115
}
113-
const result = (await fal.run(request.settings.videoGenerationModel, {
116+
const result = (await fal.run(model, {
114117
input: {
115118
image_url: storyboard.assetUrl,
116119

@@ -141,7 +144,12 @@ export async function resolveSegment(
141144
request.segment.category === ClapSegmentCategory.SOUND ||
142145
request.segment.category === ClapSegmentCategory.MUSIC
143146
) {
144-
const result = (await fal.run(request.settings.soundGenerationModel, {
147+
model =
148+
request.segment.category === ClapSegmentCategory.MUSIC
149+
? request.settings.musicGenerationWorkflow.data
150+
: request.settings.soundGenerationWorkflow.data
151+
152+
const result = (await fal.run(model, {
145153
input: {
146154
// note how we use the *segment* prompt for music or sound
147155
prompt: request.segment.prompt,
@@ -153,7 +161,9 @@ export async function resolveSegment(
153161

154162
segment.assetUrl = result?.audio_file?.url || ''
155163
} else if (request.segment.category === ClapSegmentCategory.DIALOGUE) {
156-
const result = (await fal.run(request.settings.voiceGenerationModel, {
164+
model = request.settings.voiceGenerationWorkflow.data || ''
165+
166+
const result = (await fal.run(model, {
157167
input: {
158168
text: request.segment.prompt,
159169

‎src/app/api/resolve/providers/huggingface/generateImage.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,9 @@ import { decodeOutput } from '@/lib/utils/decodeOutput'
44
import { ResolveRequest } from '@aitube/clapper-services'
55

66
export async function generateImage(request: ResolveRequest): Promise<string> {
7-
if (!request.settings.imageGenerationModel) {
7+
if (!request.settings.imageGenerationWorkflow.data) {
88
throw new Error(
9-
`HuggingFace.generateImage: cannot generate without a valid imageGenerationModel`
9+
`HuggingFace.generateImage: cannot generate without a valid imageGenerationWorkflow`
1010
)
1111
}
1212

@@ -27,7 +27,7 @@ export async function generateImage(request: ResolveRequest): Promise<string> {
2727
)
2828

2929
const blob: Blob = await hf.textToImage({
30-
model: request.settings.imageGenerationModel,
30+
model: request.settings.imageGenerationWorkflow.data,
3131
inputs: request.prompts.image.positive,
3232
parameters: {
3333
height: request.meta.height,

‎src/app/api/resolve/providers/huggingface/generateVideo.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@ import { ResolveRequest } from '@aitube/clapper-services'
22
import { callGradioApi } from '@/lib/hf/callGradioApi'
33

44
export async function generateVideo(request: ResolveRequest): Promise<string> {
5-
if (!request.settings.videoGenerationModel) {
5+
if (!request.settings.videoGenerationWorkflow.data) {
66
throw new Error(
7-
`HuggingFace.generateVideo: cannot generate without a valid videoGenerationModel`
7+
`HuggingFace.generateVideo: cannot generate without a valid videoGenerationWorkflow.data`
88
)
99
}
1010

@@ -22,7 +22,7 @@ export async function generateVideo(request: ResolveRequest): Promise<string> {
2222

2323
// TODO pass a type to the template function
2424
const assetUrl = await callGradioApi<string>({
25-
url: request.settings.videoGenerationModel,
25+
url: request.settings.videoGenerationWorkflow.data,
2626
inputs: request.prompts.video,
2727
apiKey: request.settings.huggingFaceApiKey,
2828
})

‎src/app/api/resolve/providers/huggingface/generateVoice.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,9 @@ import { HfInference, HfInferenceEndpoint } from '@huggingface/inference'
33
import { ResolveRequest } from '@aitube/clapper-services'
44

55
export async function generateVoice(request: ResolveRequest): Promise<string> {
6-
if (!request.settings.voiceGenerationModel) {
6+
if (!request.settings.voiceGenerationWorkflow.data) {
77
throw new Error(
8-
`HuggingFace.generateVoice: cannot generate without a valid voiceGenerationModel`
8+
`HuggingFace.generateVoice: cannot generate without a valid voiceGenerationWorkflow`
99
)
1010
}
1111

@@ -26,7 +26,7 @@ export async function generateVoice(request: ResolveRequest): Promise<string> {
2626
)
2727

2828
const blob: Blob = await hf.textToSpeech({
29-
model: request.settings.voiceGenerationModel,
29+
model: request.settings.voiceGenerationWorkflow.data,
3030
inputs: request.prompts.voice.positive,
3131
})
3232

‎src/app/api/resolve/providers/huggingface/index.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ export async function resolveSegment(
3131
segment.assetUrl = await generateVideo(request)
3232
} else {
3333
throw new Error(
34-
`Clapper doesn't support ${request.segment.category} generation for provider "Hugging Face" with model (or space) "${request.settings.videoGenerationModel}". Please open a pull request with (working code) to solve this!`
34+
`Clapper doesn't support ${request.segment.category} generation for provider "Hugging Face" with model (or space) "${request.settings.videoGenerationWorkflow}". Please open a pull request with (working code) to solve this!`
3535
)
3636
}
3737
return segment

‎src/app/api/resolve/providers/replicate/index.ts

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,16 @@ export async function resolveSegment(
2424
// like we are doing for Hugging Face (match the fields etc)
2525
if (request.segment.category === ClapSegmentCategory.STORYBOARD) {
2626
let params: object = {}
27-
if (request.settings.imageGenerationModel === 'fofr/pulid-lightning') {
27+
if (
28+
request.settings.imageGenerationWorkflow.data === 'fofr/pulid-lightning'
29+
) {
2830
params = {
2931
prompt: request.prompts.image.positive,
3032
face_image: request.prompts.image.identity,
3133
}
32-
} else if (request.settings.imageGenerationModel === 'zsxkib/pulid') {
34+
} else if (
35+
request.settings.imageGenerationWorkflow.data === 'zsxkib/pulid'
36+
) {
3337
params = {
3438
prompt: request.prompts.image.positive,
3539
main_face_image: request.prompts.image.identity,
@@ -40,13 +44,13 @@ export async function resolveSegment(
4044
}
4145
}
4246
const response = (await replicate.run(
43-
request.settings.imageGenerationModel as any,
47+
request.settings.imageGenerationWorkflow as any,
4448
{ input: params }
4549
)) as any
4650
segment.assetUrl = `${response.output || ''}`
4751
} else if (request.segment.category === ClapSegmentCategory.DIALOGUE) {
4852
const response = (await replicate.run(
49-
request.settings.voiceGenerationModel as any,
53+
request.settings.voiceGenerationWorkflow.data as any,
5054
{
5155
input: {
5256
text: request.prompts.voice.positive,
@@ -57,7 +61,7 @@ export async function resolveSegment(
5761
segment.assetUrl = `${response.output || ''}`
5862
} else if (request.segment.category === ClapSegmentCategory.VIDEO) {
5963
const response = (await replicate.run(
60-
request.settings.videoGenerationModel as any,
64+
request.settings.videoGenerationWorkflow.data as any,
6165
{
6266
input: {
6367
image: request.prompts.video.image,

‎src/app/api/resolve/providers/stabilityai/generateImage.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ export async function generateImage(request: ResolveRequest): Promise<string> {
99
)
1010
}
1111

12-
if (!request.settings.imageGenerationModel) {
12+
if (!request.settings.imageGenerationWorkflow.data) {
1313
throw new Error(
1414
`StabilityAI.generateImage: cannot generate without a valid stabilityAiModelForImage`
1515
)
@@ -45,7 +45,7 @@ export async function generateImage(request: ResolveRequest): Promise<string> {
4545
body.set('aspect_ratio', `${aspectRatio || ''}`)
4646

4747
const response = await fetch(
48-
`https://api.stability.ai/v2beta/${request.settings.imageGenerationModel}`,
48+
`https://api.stability.ai/v2beta/${request.settings.imageGenerationWorkflow.data}`,
4949
{
5050
method: 'POST',
5151
headers: {

0 commit comments

Comments
 (0)