diff --git a/README.md b/README.md index 44fcdfda..0b42020b 100644 --- a/README.md +++ b/README.md @@ -86,12 +86,20 @@ If you find that Clapper is working with a more recent (stable) version of Node, ### Installing and running the app +Install the dependencies. + +`--include=optional` is to make sure optional dependencies are installed (pre-build native modules compatible with your system) + ```bash npm i --include=optional +``` + +Then run the actual app, the first time you go to localhost:3000 after typing this command, the app will compile, which can take a minute (like, literally: `Compiled / in 52.6s (6372 modules)`) + +```bash npm run dev ``` -`--include=optional` is to make sure optional dependencies are installed (pre-build native modules compatible with your system) ### Building the app diff --git a/package-lock.json b/package-lock.json index 008187a9..876048df 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,8 +11,8 @@ "dependencies": { "@aitube/broadway": "0.2.0", "@aitube/clap": "0.2.0", - "@aitube/clapper-services": "0.2.0-2", - "@aitube/engine": "0.2.0", + "@aitube/clapper-services": "0.2.0-3", + "@aitube/engine": "0.2.0-1", "@aitube/timeline": "0.2.0", "@fal-ai/serverless-client": "^0.13.0", "@ffmpeg/ffmpeg": "^0.12.10", @@ -188,9 +188,9 @@ } }, "node_modules/@aitube/clapper-services": { - "version": "0.2.0-2", - "resolved": "https://registry.npmjs.org/@aitube/clapper-services/-/clapper-services-0.2.0-2.tgz", - "integrity": "sha512-lCXK8tP9Pj3kz/JCs1m/fN9Brx5c6WgZl6Lfo2d3yw9YvbguGDXWs/Q85LKtaL3o6oAhRQz1NUBsL3RHeubKaQ==", + "version": "0.2.0-3", + "resolved": "https://registry.npmjs.org/@aitube/clapper-services/-/clapper-services-0.2.0-3.tgz", + "integrity": "sha512-vKXxslpNqSjP1+nbHS+yiHd/QtRdHnr8DYUGyw8IwviF0+jx8HqM0khOSptvW3ipxrgeHXXqpMIJq5eY1SF+eA==", "peerDependencies": { "@aitube/clap": "0.2.0", "@aitube/timeline": "0.2.0", @@ -211,9 +211,9 @@ } }, "node_modules/@aitube/engine": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/@aitube/engine/-/engine-0.2.0.tgz", - "integrity": "sha512-sAEjX2QoRdI/HE7ApkrLwBuCTlMh9g+hpNvkrTzpV1A/AMInAWkF0iiCs7PS9OaPT6F1U9Qr+yGqq4JFAgp32Q==", + "version": "0.2.0-1", + "resolved": "https://registry.npmjs.org/@aitube/engine/-/engine-0.2.0-1.tgz", + "integrity": "sha512-q+70dt7a5ZNM/yyGVrTcyNSW86FHEQYOqvOKRdy27IPVdi4UwY5OlL3Gemr04xmh3LzMSRjfK3/hTnKRgCGiLQ==", "peerDependencies": { "@aitube/clap": "0.2.0" } diff --git a/package.json b/package.json index 72cd1958..69827820 100644 --- a/package.json +++ b/package.json @@ -38,8 +38,8 @@ "dependencies": { "@aitube/broadway": "0.2.0", "@aitube/clap": "0.2.0", - "@aitube/clapper-services": "0.2.0-2", - "@aitube/engine": "0.2.0", + "@aitube/clapper-services": "0.2.0-3", + "@aitube/engine": "0.2.0-1", "@aitube/timeline": "0.2.0", "@fal-ai/serverless-client": "^0.13.0", "@ffmpeg/ffmpeg": "^0.12.10", diff --git a/public/carriers/README.md b/public/carriers/README.md new file mode 100644 index 00000000..6721d435 --- /dev/null +++ b/public/carriers/README.md @@ -0,0 +1,3 @@ +# Carriers + +Put carrier videos here (for Live Portrait and other similar tech) \ No newline at end of file diff --git a/public/images/providers/comfyicu.png b/public/images/providers/comfyicu.png new file mode 100644 index 00000000..88fa34ae --- /dev/null +++ b/public/images/providers/comfyicu.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad12a017d9bdb6aed396e305cb43207247eadeada5f959b3e3b700a98bc9e120 +size 4014 diff --git a/public/voices/README.md b/public/voices/README.md new file mode 100644 index 00000000..9ed5eba3 --- /dev/null +++ b/public/voices/README.md @@ -0,0 +1 @@ +Voice files \ No newline at end of file diff --git a/src/app/api/resolve/providers/comfy-comfyicu/index.ts b/src/app/api/resolve/providers/comfy-comfyicu/index.ts index a7665019..1b1f06da 100644 --- a/src/app/api/resolve/providers/comfy-comfyicu/index.ts +++ b/src/app/api/resolve/providers/comfy-comfyicu/index.ts @@ -5,6 +5,8 @@ import { getClapAssetSourceType, } from '@aitube/clap' import { TimelineSegment } from '@aitube/timeline' +import { getWorkflowInputValues } from '../getWorkflowInputValues' +import { ComfyIcuApiRequestRunWorkflow } from './types' export async function resolveSegment( request: ResolveRequest @@ -12,7 +14,65 @@ export async function resolveSegment( if (!request.settings.comfyIcuApiKey) { throw new Error(`Missing API key for "Comfy.icu"`) } - if (request.segment.category !== ClapSegmentCategory.STORYBOARD) { + + if (request.segment.category === ClapSegmentCategory.STORYBOARD) { + + const workflowId = request.settings.imageGenerationWorkflow.id.split('://').pop() || '' + + if (!workflowId) { + throw new Error(`The ComfyICU workflow ID is missing`) + } + + const inputFields = request.settings.imageGenerationWorkflow.inputFields || [] + + // since this is a random "wild" workflow, it is possible + // that the field name is a bit different + // we try to look into the workflow input fields + // to find the best match + const promptFields = [ + inputFields.find(f => f.id === 'prompt'),// exactMatch, + inputFields.find(f => f.id.includes('prompt')), // similarName, + inputFields.find(f => f.type === 'string') // similarType + ].filter(x => typeof x !== 'undefined') + + const promptField = promptFields[0] + if (!promptField) { + throw new Error(`this workflow doesn't seem to have a parameter called "prompt"`) + } + + // TODO: modify the serialized workflow payload + // to inject our params: + // ...getWorkflowInputValues(request.settings.imageGenerationWorkflow), + // [promptField.id]: request.prompts.image.positive, + + const payload: ComfyIcuApiRequestRunWorkflow = { + workflow_id: workflowId, + prompt: request.settings.imageGenerationWorkflow.data, + files: {}, + } + + + const rawResponse = await fetch(`https://comfy.icu/api/v1/workflows/${workflowId}/runs`, { + headers: { + accept: "application/json", + "content-type": "application/json", + authorization: `Bearer ${request.settings.comfyIcuApiKey}`, + }, + body: JSON.stringify(payload), + method: "POST", + }); + + const response = await rawResponse.json() + + if (response.status === "error") { + throw new Error(response.message) + } + + console.log('response:', response) + + // TODO use the RUN ID to regularly check for status + // see https://comfy.icu/docs/api + throw new Error( `Clapper doesn't support ${request.segment.category} generation for provider "Comfy.icu". Please open a pull request with (working code) to solve this!` ) @@ -20,14 +80,5 @@ export async function resolveSegment( const segment: TimelineSegment = { ...request.segment } - try { - throw new Error(`Not Implemented!`) - } catch (err) { - console.error(`failed to call Comfy.icu: `, err) - segment.assetUrl = '' - segment.assetSourceType = getClapAssetSourceType(segment.assetUrl) - segment.status = ClapSegmentStatus.TO_GENERATE - } - return segment } diff --git a/src/app/api/resolve/providers/comfy-comfyicu/types.ts b/src/app/api/resolve/providers/comfy-comfyicu/types.ts new file mode 100644 index 00000000..cbf30888 --- /dev/null +++ b/src/app/api/resolve/providers/comfy-comfyicu/types.ts @@ -0,0 +1,24 @@ +export type ComfyIcuApiRequestRunWorkflow = { + workflow_id: string + prompt: string + files: Record +} + +export type ComfyIcuApiResponseWorkflowStatus = { + id: string + run_time?: number + status: string + name?: string + created_at: string + output?: ComfyIcuWorkflowOutput[] + project_id: string + api_key_id: any + device?: string +} + +export type ComfyIcuWorkflowOutput = { + filename: string + url: string + thumbnail_url: string +} + diff --git a/src/app/api/resolve/providers/falai/index.ts b/src/app/api/resolve/providers/falai/index.ts index 18bde33d..72f15ef2 100644 --- a/src/app/api/resolve/providers/falai/index.ts +++ b/src/app/api/resolve/providers/falai/index.ts @@ -8,6 +8,7 @@ import { FalAiSpeechResponse, FalAiVideoResponse, } from './types' +import { getWorkflowInputValues } from '../getWorkflowInputValues' export async function resolveSegment( request: ResolveRequest @@ -37,13 +38,6 @@ export async function resolveSegment( return segment } - const imageSize = - request.meta.orientation === ClapMediaOrientation.SQUARE - ? FalAiImageSize.SQUARE_HD - : request.meta.orientation === ClapMediaOrientation.PORTRAIT - ? FalAiImageSize.PORTRAIT_16_9 - : FalAiImageSize.LANDSCAPE_16_9 - let result: FalAiImageResponse | undefined = undefined if (model === 'fal-ai/pulid') { @@ -56,6 +50,25 @@ export async function resolveSegment( } } + const { + workflowDefaultValues, + workflowValues + } = getWorkflowInputValues(request.settings.imageGenerationWorkflow) + + // for the moment let's use FAL's predefined sizes + const imageSize = + request.meta.orientation === ClapMediaOrientation.SQUARE + ? FalAiImageSize.SQUARE_HD + : request.meta.orientation === ClapMediaOrientation.PORTRAIT + ? FalAiImageSize.PORTRAIT_16_9 + : FalAiImageSize.LANDSCAPE_16_9 + + // for the moment let's use FAL's predefined sizes + delete workflowDefaultValues.width + delete workflowDefaultValues.height + delete workflowValues.width + delete workflowValues.height + if (model === 'fal-ai/pulid') { result = (await fal.run(model, { input: { @@ -74,11 +87,13 @@ export async function resolveSegment( } else { result = (await fal.run(model, { input: { + ...workflowDefaultValues, + ...workflowValues, + prompt: request.prompts.image.positive, + image_size: imageSize, sync_mode: true, - num_inference_steps: - model === 'fal-ai/stable-diffusion-v3-medium' ? 40 : 25, num_images: 1, enable_safety_checker: request.settings.censorNotForAllAudiencesContent, @@ -113,18 +128,12 @@ export async function resolveSegment( `cannot generate a video without a storyboard (the concept of Clapper is to use storyboards)` ) } + const result = (await fal.run(model, { input: { - image_url: storyboard.assetUrl, + ...getWorkflowInputValues(request.settings.videoGenerationWorkflow), - motion_bucket_id: 55, - - // The conditoning augmentation determines the amount of noise that - // will be added to the conditioning frame. The higher the number, - // the more noise there will be, and the less the video will look - // like the initial image. Increase it for more motion. - // Default value: 0.02 - cond_aug: 0.02, + image_url: storyboard.assetUrl, sync_mode: true, enable_safety_checker: request.settings.censorNotForAllAudiencesContent, @@ -140,17 +149,29 @@ export async function resolveSegment( } segment.assetUrl = result?.video?.url || '' - } else if ( - request.segment.category === ClapSegmentCategory.SOUND || - request.segment.category === ClapSegmentCategory.MUSIC - ) { - model = - request.segment.category === ClapSegmentCategory.MUSIC - ? request.settings.musicGenerationWorkflow.data - : request.settings.soundGenerationWorkflow.data + } else if (request.segment.category === ClapSegmentCategory.SOUND) { + model = request.settings.musicGenerationWorkflow.data + + const result = (await fal.run(model, { + input: { + ...getWorkflowInputValues(request.settings.soundGenerationWorkflow), + + // note how we use the *segment* prompt for music or sound + prompt: request.segment.prompt, + + sync_mode: true, + enable_safety_checker: request.settings.censorNotForAllAudiencesContent, + }, + })) as FalAiAudioResponse + + segment.assetUrl = result?.audio_file?.url || '' + } else if (request.segment.category === ClapSegmentCategory.MUSIC) { + model = request.settings.musicGenerationWorkflow.data const result = (await fal.run(model, { input: { + ...getWorkflowInputValues(request.settings.soundGenerationWorkflow), + // note how we use the *segment* prompt for music or sound prompt: request.segment.prompt, @@ -163,12 +184,19 @@ export async function resolveSegment( } else if (request.segment.category === ClapSegmentCategory.DIALOGUE) { model = request.settings.voiceGenerationWorkflow.data || '' + + let voiceIdentity = + request.prompts.voice.identity || + // TODO for the default we should use one of our own voices instea + // PS: are you implementing this task? please do a search in the code for speakers/bria.mp3 + 'https://cdn.themetavoice.xyz/speakers/bria.mp3' + const result = (await fal.run(model, { input: { - text: request.segment.prompt, + ...getWorkflowInputValues(request.settings.voiceGenerationWorkflow), - // todo use the entty audio id, if available - audio_url: 'https://cdn.themetavoice.xyz/speakers/bria.mp3', + text: request.segment.prompt, // <-- we are using the segment prompt + audio_url: voiceIdentity, sync_mode: true, enable_safety_checker: request.settings.censorNotForAllAudiencesContent, diff --git a/src/app/api/resolve/providers/getWorkflowInputValues.ts b/src/app/api/resolve/providers/getWorkflowInputValues.ts new file mode 100644 index 00000000..c579c917 --- /dev/null +++ b/src/app/api/resolve/providers/getWorkflowInputValues.ts @@ -0,0 +1,23 @@ +import { ClapInputValues, ClapWorkflow } from "@aitube/clap" + +export function getWorkflowInputValues(workflow: ClapWorkflow): { + workflowDefaultValues: ClapInputValues + workflowValues: ClapInputValues +} { + const workflowDefaultValues = + workflow.inputFields.reduce( + (acc, field) => ({ + ...acc, + [field.id]: field.defaultValue, + }), + {} as ClapInputValues + ) + +const workflowValues = workflow + .inputValues as ClapInputValues + + return { + workflowDefaultValues, + workflowValues + } +} \ No newline at end of file diff --git a/src/app/api/resolve/providers/huggingface/generateImage.ts b/src/app/api/resolve/providers/huggingface/generateImage.ts index 948a9bd4..cbe11372 100644 --- a/src/app/api/resolve/providers/huggingface/generateImage.ts +++ b/src/app/api/resolve/providers/huggingface/generateImage.ts @@ -2,6 +2,8 @@ import { HfInference, HfInferenceEndpoint } from '@huggingface/inference' import { decodeOutput } from '@/lib/utils/decodeOutput' import { ResolveRequest } from '@aitube/clapper-services' +import { blobToBase64DataUri } from '@/lib/utils' +import { addBase64Header } from '@/lib/utils/addBase64Header' export async function generateImage(request: ResolveRequest): Promise { if (!request.settings.imageGenerationWorkflow.data) { @@ -32,6 +34,11 @@ export async function generateImage(request: ResolveRequest): Promise { parameters: { height: request.meta.height, width: request.meta.width, + + // this triggers the following exception: + // Error: __call__() got an unexpected keyword argument 'negative_prompt' + // negative_prompt: request.prompts.image.negative || '', + /** * The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. */ @@ -43,7 +50,9 @@ export async function generateImage(request: ResolveRequest): Promise { }, }) - console.log('output from Hugging Face Inference API:', blob) + // console.log('output from Hugging Face Inference API:', blob) + + const buffer = Buffer.from(await blob.arrayBuffer()) - throw new Error(`finish me`) + return `data:${blob.type || 'image/jpeg'};base64,${buffer.toString('base64')}` } diff --git a/src/app/api/resolve/providers/huggingface/generateMusic.ts b/src/app/api/resolve/providers/huggingface/generateMusic.ts new file mode 100644 index 00000000..f2a1111e --- /dev/null +++ b/src/app/api/resolve/providers/huggingface/generateMusic.ts @@ -0,0 +1,40 @@ +import { HfInference, HfInferenceEndpoint } from '@huggingface/inference' + +import { ResolveRequest } from '@aitube/clapper-services' + +export async function generateMusic(request: ResolveRequest): Promise { + if (!request.settings.musicGenerationWorkflow.data) { + throw new Error( + `HuggingFace.generateMusic: cannot generate without a valid musicGenerationWorkflow` + ) + } + + if (!request.prompts.music.positive) { + throw new Error( + `HuggingFace.generateMusic: cannot generate without a valid music prompt` + ) + } + + if (!request.settings.huggingFaceApiKey) { + throw new Error( + `HuggingFace.generateMusic: cannot generate without a valid huggingFaceApiKey` + ) + } + + const hf: HfInferenceEndpoint = new HfInference( + request.settings.huggingFaceApiKey + ) + + /* + hf.textToMusic doesn't exist yet! + + const blob: Blob = await hf.textToMusic({ + model: request.settings.musicGenerationWorkflow.data, + inputs: request.prompts.music.positive, + }) + + console.log('output from Hugging Face Inference API:', blob) + */ + + throw new Error(`finish me`) +} diff --git a/src/app/api/resolve/providers/huggingface/index.ts b/src/app/api/resolve/providers/huggingface/index.ts index 0f867716..3b8eaf53 100644 --- a/src/app/api/resolve/providers/huggingface/index.ts +++ b/src/app/api/resolve/providers/huggingface/index.ts @@ -23,15 +23,13 @@ export async function resolveSegment( if (request.segment.category === ClapSegmentCategory.STORYBOARD) { segment.assetUrl = await generateImage(request) - } - if (request.segment.category === ClapSegmentCategory.DIALOGUE) { + } else if (request.segment.category === ClapSegmentCategory.DIALOGUE) { segment.assetUrl = await generateVoice(request) - } - if (request.segment.category === ClapSegmentCategory.VIDEO) { + } else if (request.segment.category === ClapSegmentCategory.VIDEO) { segment.assetUrl = await generateVideo(request) } else { throw new Error( - `Clapper doesn't support ${request.segment.category} generation for provider "Hugging Face" with model (or space) "${request.settings.videoGenerationWorkflow}". Please open a pull request with (working code) to solve this!` + `Clapper doesn't support ${request.segment.category} generation for provider "Hugging Face". Please open a pull request with (working code) to solve this!` ) } return segment diff --git a/src/app/api/resolve/route.ts b/src/app/api/resolve/route.ts index aa5a02c1..dca51bb6 100644 --- a/src/app/api/resolve/route.ts +++ b/src/app/api/resolve/route.ts @@ -6,6 +6,7 @@ import { getClapAssetSourceType, ClapWorkflowProvider, ClapWorkflow, + ClapAssetSource, } from '@aitube/clap' import { @@ -118,7 +119,7 @@ export async function POST(req: NextRequest) { } catch (err) { console.error(`failed to generate a segment: ${err}`) segment.assetUrl = '' - segment.assetSourceType = getClapAssetSourceType(segment.assetUrl) + segment.assetSourceType = ClapAssetSource.EMPTY segment.assetDurationInMs = 0 segment.outputGain = 0 segment.status = ClapSegmentStatus.TO_GENERATE diff --git a/src/components/core/providers/ClapWorkflowProviderLogo.tsx b/src/components/core/providers/ClapWorkflowProviderLogo.tsx index 60bb3d7b..89832708 100644 --- a/src/components/core/providers/ClapWorkflowProviderLogo.tsx +++ b/src/components/core/providers/ClapWorkflowProviderLogo.tsx @@ -21,6 +21,7 @@ export function ClapWorkflowProviderLogo({ return ( {formatProvider(provider)} = [ClapWorkflowProvider.CUSTOM]: none, [ClapWorkflowProvider.COMFY_HUGGINGFACE]: huggingface, [ClapWorkflowProvider.COMFY_REPLICATE]: replicate, - [ClapWorkflowProvider.COMFY_COMFYICU]: none, + [ClapWorkflowProvider.COMFY_COMFYICU]: comfyicu, [ClapWorkflowProvider.KUAISHOU]: kuaishou, [ClapWorkflowProvider.LEONARDOAI]: leonardoai, [ClapWorkflowProvider.EVERARTAI]: everartai, diff --git a/src/components/editors/WorkflowEditor/WorkflowViewer/ReactFlowCanvas/formats/comfyui/types.ts b/src/components/editors/WorkflowEditor/WorkflowViewer/ReactFlowCanvas/formats/comfyui/types.ts index 5104bfbc..4d8e107c 100644 --- a/src/components/editors/WorkflowEditor/WorkflowViewer/ReactFlowCanvas/formats/comfyui/types.ts +++ b/src/components/editors/WorkflowEditor/WorkflowViewer/ReactFlowCanvas/formats/comfyui/types.ts @@ -16,11 +16,14 @@ export type ComfyuiWorkflow = { export type ComfyuiWorkflowNode = { id: number - pos: number[] + title?: string | null + pos: Record | number[] mode: number size: { '0': number; '1': number } type: string color?: string | null + bgcolor?: string | null + locked?: boolean flags: { collapsed?: boolean | null } @@ -33,6 +36,7 @@ export type ComfyuiWorkflowNode = { export type ComfyuiWorkflowNodeInput = { link?: number | null + dir?: number | null name: string type: string label?: string | null @@ -47,7 +51,7 @@ export type ComfyuiWorkflowNodeOutput = { type: string label?: string | null links?: number[] | null - shape: number + shape?: number | null slot_index?: number | null } diff --git a/src/components/toolbars/top-menu/lists/getWorkflowProviders.ts b/src/components/toolbars/top-menu/lists/getWorkflowProviders.ts index 331c55ec..6c0d896c 100644 --- a/src/components/toolbars/top-menu/lists/getWorkflowProviders.ts +++ b/src/components/toolbars/top-menu/lists/getWorkflowProviders.ts @@ -54,7 +54,10 @@ export function findWorkflows( } workflows.push(workflow) workflowIds[workflow.id] = workflow - providers[workflow.provider] = workflows + if (!Array.isArray(providers[workflow.provider])) { + providers[workflow.provider] = [] + } + providers[workflow.provider]?.push(workflow) } return { diff --git a/src/lib/utils/addBase64Header.ts b/src/lib/utils/addBase64Header.ts new file mode 100644 index 00000000..1a0a54d8 --- /dev/null +++ b/src/lib/utils/addBase64Header.ts @@ -0,0 +1,53 @@ +export function addBase64Header( + image?: string, + format?: + | 'jpeg' + | 'jpg' + | 'png' + | 'webp' + | 'heic' + | 'mp3' + | 'wav' + | 'mp4' + | 'webm' + | string +) { + if (!image || typeof image !== 'string' || image.length < 60) { + return '' + } + + const ext = (`${format || ''}`.split('.').pop() || '').toLowerCase().trim() + + let mime = '' + if (ext === 'jpeg' || ext === 'jpg') { + mime = 'image/jpeg' + } else if (ext === 'webp') { + mime = 'image/webp' + } else if (ext === 'png') { + mime = 'image/png' + } else if (ext === 'heic') { + mime = 'image/heic' + } else if (ext === 'mp3') { + mime = 'audio/mp3' + } else if (ext === 'mp4') { + mime = 'video/mp4' + } else if (ext === 'webm') { + mime = 'video/webm' + } else if (ext === 'wav') { + mime = 'audio/wav' + } else { + throw new Error(`addBase64Header failed (unsupported format: ${format})`) + } + + if (image.startsWith('data:')) { + if (image.startsWith(`data:${mime};base64,`)) { + return image + } else { + throw new Error( + `addBase64Header failed (input string is NOT a ${mime} image)` + ) + } + } else { + return `data:${mime};base64,${image}` + } +} diff --git a/src/services/editors/workflow-editor/workflows/_documentation_/demo_examples.ts b/src/services/editors/workflow-editor/workflows/_documentation_/demo_examples.ts new file mode 100644 index 00000000..e677d289 --- /dev/null +++ b/src/services/editors/workflow-editor/workflows/_documentation_/demo_examples.ts @@ -0,0 +1,101 @@ +/* + + + This file is just a documented example, so that you can understand + the purpose of the workflow format. + + +*/ + +// a workflow is a core concept within the OpenClap format itself +// (a .clap file can contain a workflow) +import { + ClapWorkflow, + ClapWorkflowCategory, + ClapWorkflowEngine, + ClapWorkflowProvider, +} from '@aitube/clap' + +// some parameters keep reappering from one workflow to another +// for this purpose I've put some common params in defaultValues.ts +import { genericPrompt } from '../common/defaultValues' + +// always use ClapWorkflow to validate your types, +// it will save you headaches +const justAnExample_DoNotUseThis: ClapWorkflow[] = [ + { + // a workflow ID must be unique across ALL the workflow database + // if you want to be safe, you can generate a UUID, but this has to be + // done once and for all (ie. you cannot do `id: generateUUID()` or else + // the user will lose their settings each time they restart the app) + id: '', + + // this is a human-readable label, shown in the UI + label: 'My Workflow', + + // this is a longer description. We don't show it yet, + // but it could be in a tooltip eg. when hover a small (i) icon + description: 'It does this and that', + + // tags useable to search and filter. + // this will be used once we pull workflows from a communit database + tags: ['image', 'character', 'realistic'], + + // the author(s) - might be long if there is the original research team, + // the person who adapted the model etc + author: 'Weyland Yutani', + + // a screenshot preview of the thumbnail + thumbnailUrl: '', + + // the engine used for this workflow + engine: ClapWorkflowEngine.COMFYUI_WORKFLOW, + + // platform and/or cloud service provider + provider: ClapWorkflowProvider.COMFY_COMFYICU, + + // type of workflow (this is important) + category: ClapWorkflowCategory.IMAGE_GENERATION, + + // if this is a REST_API engine this can be any kind of identifier, + // like an ID or a URL path eg. "fal-ai/something" + // + // if this is a workflow engine this will typically be a JSON + // containing the workflow, serialized so it can fit in a string + data: '{"5":{"inputs":{"width":1024,"height":1024,"batch_size":1},"class_type":"EmptyLatentImage"},"6":{"inputs":{"clip":["11",0],"text":"detailed cinematic dof render of an old dusty detailed CRT monitor on a wooden desk in a dim room with items around, messy dirty room. On the screen are the letters "FLUX Schnell" glowing softly. High detail hard surface render"},"class_type":"CLIPTextEncode"},"8":{"inputs":{"vae":["10",0],"samples":["13",0]},"class_type":"VAEDecode"},"9":{"inputs":{"images":["8",0],"filename_prefix":"ComfyUI"},"class_type":"SaveImage"},"10":{"inputs":{"vae_name":"flux1-ae.safetensors"},"class_type":"VAELoader"},"11":{"inputs":{"type":"flux","clip_name1":"t5xxl_fp16.safetensors","clip_name2":"clip_l.safetensors"},"class_type":"DualCLIPLoader"},"12":{"inputs":{"unet_name":"flux1-schnell.safetensors","weight_dtype":"default"},"class_type":"UNETLoader"},"13":{"inputs":{"noise":["25",0],"guider":["22",0],"sigmas":["17",0],"sampler":["16",0],"latent_image":["5",0]},"class_type":"SamplerCustomAdvanced"},"16":{"inputs":{"sampler_name":"euler"},"class_type":"KSamplerSelect"},"17":{"inputs":{"model":["12",0],"steps":4,"denoise":1,"scheduler":"simple"},"class_type":"BasicScheduler"},"22":{"inputs":{"model":["12",0],"conditioning":["6",0]},"class_type":"BasicGuider"},"25":{"inputs":{"noise_seed":24016052484185},"class_type":"RandomNoise"}}', + + // this describe the parameters of the workflow + // this should be as close as possible to the actual parameters + // of the workflow (eg. this could come from some kind of registry, community website API etc) + inputFields: [ + genericPrompt, + { + id: 'custom_field', + label: 'Custom field', + description: 'A field unique to this workflow', + type: 'number', + minValue: 0, + maxValue: 1, + + // this is the original default value, as recommended by the author + // it is not supposed to be changed (ideally, it would come from an API etc) + // so please do not touch this as there is a dedicated way to override values + // (see below, in "inputValues: {}") + defaultValue: 0.03, + }, + ], + + // this is where we can get creative and override the recommende default values, + // using values suggested by the Clapper teap + inputValues: { + // if you are fine with the original default value, + // you don't actually need to do this + // + // if you see me doing this in the code, that is just + // as a placeholder, in case we want to tweak the value + // [genericPrompt.id]: genericPrompt.defaultValue, + + custom_field: 0.042, // <-- a value curated/suggested by Clapper + }, + }, +] diff --git a/src/services/editors/workflow-editor/workflows/comfyicu/index.ts b/src/services/editors/workflow-editor/workflows/comfyicu/index.ts index bdb923a1..71ac0ee8 100644 --- a/src/services/editors/workflow-editor/workflows/comfyicu/index.ts +++ b/src/services/editors/workflow-editor/workflows/comfyicu/index.ts @@ -1,3 +1,49 @@ -import { ClapWorkflow } from '@aitube/clap' +import { + ClapWorkflow, + ClapWorkflowCategory, + ClapWorkflowEngine, + ClapWorkflowProvider, +} from '@aitube/clap' +import { + genericHeight2048, + genericPrompt, + genericWidth2048, +} from '../common/defaultValues' -export const comfyicuWorkflows: ClapWorkflow[] = [] +export const comfyicuWorkflows: ClapWorkflow[] = [ + + /* + + Unfortunately Comfy.icu doesn't support: + - programmatic run of a new arbitrary workflow (like on Replicate) + - OR (as a fallback) programmatic execution of an existing workflow created by someone else + - OR (as a fallback) programmatic creation of a fork/duplicate of a workflow created by someone else + - OR (as a fallback) programmatic creation of a new arbitrary workflow + + so basically.. we can do nothing with it for Clapper 🤷 + + + { + id: 'comfyicu://SGeMzu2XVDPvQNU-7jhIv', + label: 'FLUX.1 [schnell]', + description: '', + tags: ['flux'], + author: 'BFL (https://BlackForestLabs.ai)', + thumbnailUrl: '', + engine: ClapWorkflowEngine.COMFYUI_WORKFLOW, + provider: ClapWorkflowProvider.COMFY_COMFYICU, + category: ClapWorkflowCategory.IMAGE_GENERATION, + + // note: ComfyICU uses "API Workflows", which are simplified ComfyUI workflows + + data: '{"5":{"inputs":{"width":1024,"height":1024,"batch_size":1},"class_type":"EmptyLatentImage"},"6":{"inputs":{"clip":["11",0],"text":"detailed cinematic dof render of an old dusty detailed CRT monitor on a wooden desk in a dim room with items around, messy dirty room. On the screen are the letters "FLUX Schnell" glowing softly. High detail hard surface render"},"class_type":"CLIPTextEncode"},"8":{"inputs":{"vae":["10",0],"samples":["13",0]},"class_type":"VAEDecode"},"9":{"inputs":{"images":["8",0],"filename_prefix":"ComfyUI"},"class_type":"SaveImage"},"10":{"inputs":{"vae_name":"flux1-ae.safetensors"},"class_type":"VAELoader"},"11":{"inputs":{"type":"flux","clip_name1":"t5xxl_fp16.safetensors","clip_name2":"clip_l.safetensors"},"class_type":"DualCLIPLoader"},"12":{"inputs":{"unet_name":"flux1-schnell.safetensors","weight_dtype":"default"},"class_type":"UNETLoader"},"13":{"inputs":{"noise":["25",0],"guider":["22",0],"sigmas":["17",0],"sampler":["16",0],"latent_image":["5",0]},"class_type":"SamplerCustomAdvanced"},"16":{"inputs":{"sampler_name":"euler"},"class_type":"KSamplerSelect"},"17":{"inputs":{"model":["12",0],"steps":4,"denoise":1,"scheduler":"simple"},"class_type":"BasicScheduler"},"22":{"inputs":{"model":["12",0],"conditioning":["6",0]},"class_type":"BasicGuider"},"25":{"inputs":{"noise_seed":24016052484185},"class_type":"RandomNoise"}}', + inputFields: [genericPrompt, genericWidth2048, genericHeight2048], + inputValues: { + [genericPrompt.id]: genericPrompt.defaultValue, + [genericWidth2048.id]: genericWidth2048.defaultValue, + [genericHeight2048.id]: genericHeight2048.defaultValue, + }, + }, + + */ +] diff --git a/src/services/editors/workflow-editor/workflows/common/comfyui/flux_plus_ultimatesdupscale.ts b/src/services/editors/workflow-editor/workflows/common/comfyui/flux_plus_ultimatesdupscale.ts new file mode 100644 index 00000000..f850a11d --- /dev/null +++ b/src/services/editors/workflow-editor/workflows/common/comfyui/flux_plus_ultimatesdupscale.ts @@ -0,0 +1,853 @@ +import { ComfyuiWorkflow } from '@/components/editors/WorkflowEditor/WorkflowViewer/ReactFlowCanvas/formats/comfyui/types' + +export const fluxUltimateSDUpscale: ComfyuiWorkflow = { + last_node_id: 58, + last_link_id: 97, + nodes: [ + { + id: 37, + type: 'Reroute', + pos: [390, 120], + size: [75, 26], + flags: {}, + order: 6, + mode: 0, + inputs: [ + { + name: '', + type: '*', + link: 58, + }, + ], + outputs: [ + { + name: '', + type: 'MODEL', + links: [60], + slot_index: 0, + }, + ], + properties: { + showOutputText: false, + horizontal: false, + }, + }, + { + id: 39, + type: 'Reroute', + pos: [390, 80], + size: [75, 26], + flags: {}, + order: 9, + mode: 0, + inputs: [ + { + name: '', + type: '*', + link: 62, + }, + ], + outputs: [ + { + name: '', + type: 'VAE', + links: [64], + slot_index: 0, + }, + ], + properties: { + showOutputText: false, + horizontal: false, + }, + }, + { + id: 30, + type: 'CheckpointLoaderSimple', + pos: [40, 190], + size: { + '0': 320, + '1': 100, + }, + flags: {}, + order: 0, + mode: 0, + outputs: [ + { + name: 'MODEL', + type: 'MODEL', + links: [58], + shape: 3, + slot_index: 0, + }, + { + name: 'CLIP', + type: 'CLIP', + links: [45, 54], + shape: 3, + slot_index: 1, + }, + { + name: 'VAE', + type: 'VAE', + links: [62], + shape: 3, + slot_index: 2, + }, + ], + properties: { + 'Node name for S&R': 'CheckpointLoaderSimple', + }, + widgets_values: ['flux\\flux1-dev-fp8.safetensors'], + }, + { + id: 9, + type: 'SaveImage', + pos: [1660, 190], + size: { + '0': 985.3012084960938, + '1': 1060.3828125, + }, + flags: {}, + order: 18, + mode: 0, + inputs: [ + { + name: 'images', + type: 'IMAGE', + link: 9, + }, + ], + properties: { + 'Node name for S&R': 'SaveImage', + }, + widgets_values: ['flux/img_'], + }, + { + id: 27, + type: 'EmptySD3LatentImage', + pos: [820, 610], + size: { + '0': 210, + '1': 100, + }, + flags: {}, + order: 11, + mode: 0, + inputs: [ + { + name: 'width', + type: 'INT', + link: 67, + widget: { + name: 'width', + }, + }, + { + name: 'height', + type: 'INT', + link: 68, + widget: { + name: 'height', + }, + }, + ], + outputs: [ + { + name: 'LATENT', + type: 'LATENT', + links: [69], + shape: 3, + slot_index: 0, + }, + ], + properties: { + 'Node name for S&R': 'EmptySD3LatentImage', + }, + widgets_values: [1024, 1024, 1], + color: '#323', + bgcolor: '#535', + }, + { + id: 34, + type: 'Note', + pos: [1100, 720], + size: { + '0': 320, + '1': 130, + }, + flags: {}, + order: 1, + mode: 0, + properties: { + text: '', + }, + widgets_values: [ + 'Note that Flux dev and schnell do not have any negative prompt so CFG should be set to 1.0. Setting CFG to 1.0 means the negative prompt is ignored.', + ], + color: '#432', + bgcolor: '#653', + }, + { + id: 44, + type: 'Note', + pos: [40, 350], + size: { + '0': 320, + '1': 130, + }, + flags: {}, + order: 2, + mode: 0, + properties: { + text: '', + }, + widgets_values: [ + 'https://huggingface.co/Comfy-Org/flux1-dev/blob/main/flux1-dev-fp8.safetensors\n(ComfyUI\\models\\checkpoints\\flux)', + ], + color: '#432', + bgcolor: '#653', + }, + { + id: 38, + type: 'Reroute', + pos: [960, 120], + size: [75, 26], + flags: {}, + order: 13, + mode: 0, + inputs: [ + { + name: '', + type: '*', + link: 60, + }, + ], + outputs: [ + { + name: '', + type: 'MODEL', + links: [61, 71], + slot_index: 0, + }, + ], + properties: { + showOutputText: false, + horizontal: false, + }, + }, + { + id: 35, + type: 'FluxGuidance', + pos: [830, 190], + size: { + '0': 211.60000610351562, + '1': 58, + }, + flags: {}, + order: 14, + mode: 0, + inputs: [ + { + name: 'conditioning', + type: 'CONDITIONING', + link: 56, + }, + ], + outputs: [ + { + name: 'CONDITIONING', + type: 'CONDITIONING', + links: [57, 72], + shape: 3, + slot_index: 0, + }, + ], + properties: { + 'Node name for S&R': 'FluxGuidance', + }, + widgets_values: [3.5], + }, + { + id: 33, + type: 'CLIPTextEncode', + pos: [390, 400], + size: [420, 160], + flags: { + collapsed: false, + }, + order: 8, + mode: 0, + inputs: [ + { + name: 'clip', + type: 'CLIP', + link: 54, + slot_index: 0, + }, + ], + outputs: [ + { + name: 'CONDITIONING', + type: 'CONDITIONING', + links: [55, 73], + slot_index: 0, + }, + ], + title: 'CLIP Text Encode (Negative Prompt)', + properties: { + 'Node name for S&R': 'CLIPTextEncode', + }, + widgets_values: [''], + color: '#322', + bgcolor: '#533', + locked: true, + }, + { + id: 40, + type: 'Reroute', + pos: [1340, 80], + size: [75, 26], + flags: {}, + order: 15, + mode: 0, + inputs: [ + { + name: '', + type: '*', + link: 64, + }, + ], + outputs: [ + { + name: '', + type: 'VAE', + links: [65, 74], + slot_index: 0, + }, + ], + properties: { + showOutputText: false, + horizontal: false, + }, + }, + { + id: 46, + type: 'UpscaleModelLoader', + pos: [820, 1060], + size: { + '0': 210, + '1': 60, + }, + flags: {}, + order: 3, + mode: 0, + outputs: [ + { + name: 'UPSCALE_MODEL', + type: 'UPSCALE_MODEL', + links: [75], + shape: 3, + }, + ], + properties: { + 'Node name for S&R': 'UpscaleModelLoader', + }, + widgets_values: ['4x-UltraSharp.pth'], + }, + { + id: 48, + type: 'SaveImage', + pos: [2650, 190], + size: { + '0': 985.3012084960938, + '1': 1060.3828125, + }, + flags: {}, + order: 20, + mode: 0, + inputs: [ + { + name: 'images', + type: 'IMAGE', + link: 77, + }, + ], + properties: { + 'Node name for S&R': 'SaveImage', + }, + widgets_values: ['flux/img_'], + }, + { + id: 8, + type: 'VAEDecode', + pos: [1440, 190], + size: { + '0': 210, + '1': 50, + }, + flags: {}, + order: 17, + mode: 0, + inputs: [ + { + name: 'samples', + type: 'LATENT', + link: 52, + }, + { + name: 'vae', + type: 'VAE', + link: 65, + }, + ], + outputs: [ + { + name: 'IMAGE', + type: 'IMAGE', + links: [9, 70, 88], + slot_index: 0, + }, + ], + properties: { + 'Node name for S&R': 'VAEDecode', + }, + }, + { + id: 54, + type: 'Image Comparer (rgthree)', + pos: { + '0': 3660, + '1': 190, + '2': 0, + '3': 0, + '4': 0, + '5': 0, + '6': 0, + '7': 0, + '8': 0, + '9': 0, + }, + size: [790, 1060], + flags: {}, + order: 21, + mode: 0, + inputs: [ + { + name: 'image_a', + type: 'IMAGE', + link: 88, + dir: 3, + }, + { + name: 'image_b', + type: 'IMAGE', + link: 89, + dir: 3, + }, + ], + outputs: [], + properties: { + comparer_mode: 'Slide', + }, + widgets_values: [ + [ + { + name: 'A', + selected: true, + url: '/view?filename=rgthree.compare._temp_bslpf_00013_.png&type=temp&subfolder=&rand=0.6440769801357371', + }, + { + name: 'B', + selected: true, + url: '/view?filename=rgthree.compare._temp_bslpf_00014_.png&type=temp&subfolder=&rand=0.08215616275940874', + }, + ], + ], + }, + { + id: 45, + type: 'UltimateSDUpscale', + pos: [1100, 960], + size: [320, 830], + flags: {}, + order: 19, + mode: 0, + inputs: [ + { + name: 'image', + type: 'IMAGE', + link: 70, + }, + { + name: 'model', + type: 'MODEL', + link: 71, + }, + { + name: 'positive', + type: 'CONDITIONING', + link: 72, + }, + { + name: 'negative', + type: 'CONDITIONING', + link: 73, + }, + { + name: 'vae', + type: 'VAE', + link: 74, + }, + { + name: 'upscale_model', + type: 'UPSCALE_MODEL', + link: 75, + slot_index: 5, + }, + { + name: 'upscale_by', + type: 'FLOAT', + link: 91, + widget: { + name: 'upscale_by', + }, + slot_index: 6, + }, + { + name: 'tile_width', + type: 'INT', + link: 94, + widget: { + name: 'tile_width', + }, + }, + { + name: 'tile_height', + type: 'INT', + link: 97, + widget: { + name: 'tile_height', + }, + }, + ], + outputs: [ + { + name: 'IMAGE', + type: 'IMAGE', + links: [77, 89], + shape: 3, + slot_index: 0, + }, + ], + properties: { + 'Node name for S&R': 'UltimateSDUpscale', + }, + widgets_values: [ + 2, + 1000122408342527, + 'fixed', + 20, + 1, + 'euler', + 'simple', + 0.25, + 'Linear', + 1024, + 1024, + 16, + 32, + 'None', + 0.25, + 64, + 16, + 16, + true, + false, + ], + }, + { + id: 57, + type: 'MathExpression|pysssss', + pos: [760, 1290], + size: [270, 116.00003433227539], + flags: {}, + order: 10, + mode: 0, + inputs: [ + { + name: 'a', + type: 'INT,FLOAT,IMAGE,LATENT', + link: 92, + }, + { + name: 'b', + type: 'INT,FLOAT,IMAGE,LATENT', + link: 93, + }, + { + name: 'c', + type: 'INT,FLOAT,IMAGE,LATENT', + link: null, + }, + ], + outputs: [ + { + name: 'INT', + type: 'INT', + links: [94], + shape: 3, + slot_index: 0, + }, + { + name: 'FLOAT', + type: 'FLOAT', + links: null, + shape: 3, + }, + ], + title: 'tile height (Math Expression 🐍)', + properties: { + 'Node name for S&R': 'MathExpression|pysssss', + }, + widgets_values: ['a * b / 2 + 32'], + color: '#432', + bgcolor: '#653', + }, + { + id: 56, + type: 'easy float', + pos: [820, 1170], + size: [210, 60], + flags: {}, + order: 4, + mode: 0, + outputs: [ + { + name: 'float', + type: 'FLOAT', + links: [91, 92, 95], + shape: 3, + slot_index: 0, + }, + ], + title: 'upscale by', + properties: { + 'Node name for S&R': 'easy float', + }, + widgets_values: [2], + }, + { + id: 58, + type: 'MathExpression|pysssss', + pos: [760, 1460], + size: { + '0': 270, + '1': 116.00003051757812, + }, + flags: {}, + order: 12, + mode: 0, + inputs: [ + { + name: 'a', + type: 'INT,FLOAT,IMAGE,LATENT', + link: 95, + }, + { + name: 'b', + type: 'INT,FLOAT,IMAGE,LATENT', + link: 96, + }, + { + name: 'c', + type: 'INT,FLOAT,IMAGE,LATENT', + link: null, + }, + ], + outputs: [ + { + name: 'INT', + type: 'INT', + links: [97], + shape: 3, + slot_index: 0, + }, + { + name: 'FLOAT', + type: 'FLOAT', + links: null, + shape: 3, + }, + ], + title: 'tile height (Math Expression 🐍)', + properties: { + 'Node name for S&R': 'MathExpression|pysssss', + }, + widgets_values: ['a * b / 2 + 32'], + color: '#432', + bgcolor: '#653', + }, + { + id: 43, + type: 'SDXLAspectRatioSelector', + pos: [390, 610], + size: { + '0': 420, + '1': 100, + }, + flags: {}, + order: 5, + mode: 0, + outputs: [ + { + name: 'ratio', + type: 'STRING', + links: null, + shape: 3, + }, + { + name: 'width', + type: 'INT', + links: [67, 93], + shape: 3, + slot_index: 1, + }, + { + name: 'height', + type: 'INT', + links: [68, 96], + shape: 3, + slot_index: 2, + }, + ], + properties: { + 'Node name for S&R': 'SDXLAspectRatioSelector', + }, + widgets_values: ['2:3'], + }, + { + id: 31, + type: 'KSampler', + pos: [1100, 190], + size: [320, 470], + flags: {}, + order: 16, + mode: 0, + inputs: [ + { + name: 'model', + type: 'MODEL', + link: 61, + }, + { + name: 'positive', + type: 'CONDITIONING', + link: 57, + }, + { + name: 'negative', + type: 'CONDITIONING', + link: 55, + }, + { + name: 'latent_image', + type: 'LATENT', + link: 69, + slot_index: 3, + }, + ], + outputs: [ + { + name: 'LATENT', + type: 'LATENT', + links: [52], + shape: 3, + slot_index: 0, + }, + ], + properties: { + 'Node name for S&R': 'KSampler', + }, + widgets_values: [540311436836572, 'fixed', 20, 1, 'euler', 'simple', 1], + }, + { + id: 6, + type: 'CLIPTextEncode', + pos: [390, 190], + size: { + '0': 430, + '1': 160, + }, + flags: {}, + order: 7, + mode: 0, + inputs: [ + { + name: 'clip', + type: 'CLIP', + link: 45, + }, + ], + outputs: [ + { + name: 'CONDITIONING', + type: 'CONDITIONING', + links: [56], + slot_index: 0, + }, + ], + title: 'CLIP Text Encode (Positive Prompt)', + properties: { + 'Node name for S&R': 'CLIPTextEncode', + }, + widgets_values: [ + 'VitWaterStyle, 1girl, beautiful, white hair, long hair, bangs, fair skin, big breasts, leather armor, sword, forest, dappled sunlight, upper body, looking at viewer', + ], + color: '#232', + bgcolor: '#353', + }, + ], + links: [ + [9, 8, 0, 9, 0, 'IMAGE'], + [45, 30, 1, 6, 0, 'CLIP'], + [52, 31, 0, 8, 0, 'LATENT'], + [54, 30, 1, 33, 0, 'CLIP'], + [55, 33, 0, 31, 2, 'CONDITIONING'], + [56, 6, 0, 35, 0, 'CONDITIONING'], + [57, 35, 0, 31, 1, 'CONDITIONING'], + [58, 30, 0, 37, 0, '*'], + [60, 37, 0, 38, 0, '*'], + [61, 38, 0, 31, 0, 'MODEL'], + [62, 30, 2, 39, 0, '*'], + [64, 39, 0, 40, 0, '*'], + [65, 40, 0, 8, 1, 'VAE'], + [67, 43, 1, 27, 0, 'INT'], + [68, 43, 2, 27, 1, 'INT'], + [69, 27, 0, 31, 3, 'LATENT'], + [70, 8, 0, 45, 0, 'IMAGE'], + [71, 38, 0, 45, 1, 'MODEL'], + [72, 35, 0, 45, 2, 'CONDITIONING'], + [73, 33, 0, 45, 3, 'CONDITIONING'], + [74, 40, 0, 45, 4, 'VAE'], + [75, 46, 0, 45, 5, 'UPSCALE_MODEL'], + [77, 45, 0, 48, 0, 'IMAGE'], + [88, 8, 0, 54, 0, 'IMAGE'], + [89, 45, 0, 54, 1, 'IMAGE'], + [91, 56, 0, 45, 6, 'FLOAT'], + [92, 56, 0, 57, 0, 'INT,FLOAT,IMAGE,LATENT'], + [93, 43, 1, 57, 1, 'INT,FLOAT,IMAGE,LATENT'], + [94, 57, 0, 45, 7, 'INT'], + [95, 56, 0, 58, 0, 'INT,FLOAT,IMAGE,LATENT'], + [96, 43, 2, 58, 1, 'INT,FLOAT,IMAGE,LATENT'], + [97, 58, 0, 45, 8, 'INT'], + ], + groups: [], + config: {}, + extra: { + ds: { + scale: 0.6830134553650705, + offset: [-1420.2171836433743, 117.31047235297616], + }, + }, + version: 0.4, +} diff --git a/src/services/editors/workflow-editor/workflows/common/comfyui/index.ts b/src/services/editors/workflow-editor/workflows/common/comfyui/index.ts new file mode 100644 index 00000000..86427110 --- /dev/null +++ b/src/services/editors/workflow-editor/workflows/common/comfyui/index.ts @@ -0,0 +1 @@ +// a list of generic comfyui workflows diff --git a/src/services/editors/workflow-editor/workflows/common/defaultValues.ts b/src/services/editors/workflow-editor/workflows/common/defaultValues.ts index 809881c1..d041e375 100644 --- a/src/services/editors/workflow-editor/workflows/common/defaultValues.ts +++ b/src/services/editors/workflow-editor/workflows/common/defaultValues.ts @@ -5,7 +5,25 @@ import { ClapInputField } from '@aitube/clap' // // instead you should extend/overwrite them in your own workflow -export const genericTextPrompt: ClapInputField = { +export const genericInput: ClapInputField = { + id: 'input', + label: 'Input', + description: 'Input', + type: 'string', + allowedValues: [], + defaultValue: '', +} + +export const genericText: ClapInputField = { + id: 'text', + label: 'Text', + description: 'Text', + type: 'string', + allowedValues: [], + defaultValue: '', +} + +export const genericPrompt: ClapInputField = { id: 'prompt', label: 'Prompt', description: 'Prompt', @@ -14,6 +32,107 @@ export const genericTextPrompt: ClapInputField = { defaultValue: '', } +export const genericSeed: ClapInputField = { + id: 'seed', + label: 'Seed', + description: 'Seed', + type: 'number', + minValue: 0, + maxValue: Math.pow(2, 31), + defaultValue: 0, +} + +export const genericImage: ClapInputField = { + id: 'image', + label: 'Image', + description: 'Image', + type: 'string', + allowedValues: [], + defaultValue: '', +} + +export const genericVideo: ClapInputField = { + id: 'video', + label: 'Video', + description: 'Video', + type: 'string', + allowedValues: [], + defaultValue: '', +} + +export const genericVoice: ClapInputField = { + id: 'voice', + label: 'Voice', + description: 'Voice', + type: 'string', + allowedValues: [], + defaultValue: '', +} + +export const genericAudio: ClapInputField = { + id: 'audio', + label: 'Audio', + description: 'Audio', + type: 'string', + allowedValues: [], + defaultValue: '', +} + +export const genericInferenceSteps: ClapInputField = { + id: 'num_inference_steps', + label: 'Inference steps', + description: 'Number of inference steps', + type: 'number', + minValue: 1, + maxValue: 50, + defaultValue: 28, +} + +export const genericUpscalingFactor: ClapInputField = { + id: 'upscaling_factor', + label: 'Upscaling Factor', + description: 'Upscaling Factor (2, 3, 4..)', + // <-- TODO: we should be able to have type: 'integer' + // that is not a big issue, however (the implementation can do the rounding) + type: 'number', + minValue: 2, + maxValue: 4, + defaultValue: 2, +} + +export const genericOverlappingTiles: ClapInputField = { + id: 'overlapping_tiles', + label: 'Overlapping Tiles', + description: + 'Overlapping tiles should reduce visible seams, but doubles the inference time.', + // <-- TODO: we should be able to have type: 'integer' + // that is not a big issue, however (the implementation can do the rounding) + type: 'boolean', + defaultValue: true, +} + +export const genericMotionBucketId: ClapInputField = { + id: 'motion_bucket_id', + label: 'Motion Bucket ID', + description: + 'The motion bucket ID determines the motion of the generated video. The higher the number, the more motion there will be.', + type: 'number', + minValue: 0, + maxValue: 255, + defaultValue: 127, +} + +export const genericConditioningAugmentation: ClapInputField = { + id: 'conditioning_augmentation', + label: 'Conditioning Augmentation', + description: + 'The conditoning augmentation determines the amount of noise that will be added to the conditioning frame. The higher the number, the more noise there will be, and the less the video will look like the initial image. Increase it for more motion.', + type: 'number', + minValue: 0, + maxValue: 1, + defaultValue: 0.02, +} + export const genericWidth1024: ClapInputField = { id: 'width', label: 'Width', diff --git a/src/services/editors/workflow-editor/workflows/elevenlabs/index.ts b/src/services/editors/workflow-editor/workflows/elevenlabs/index.ts new file mode 100644 index 00000000..42227e35 --- /dev/null +++ b/src/services/editors/workflow-editor/workflows/elevenlabs/index.ts @@ -0,0 +1,52 @@ +import { + ClapWorkflow, + ClapWorkflowEngine, + ClapWorkflowCategory, + ClapWorkflowProvider, +} from '@aitube/clap' +import { + genericHeight2048, + genericPrompt, + genericWidth2048, +} from '../common/defaultValues' + +export const elevenlabsWorkflows: ClapWorkflow[] = [ + { + id: 'elevenlabs://v1/text-to-speech', + label: 'Text-to-Speech V1', + description: '', + tags: ['TTS'], + author: 'ElevenLabs', + thumbnailUrl: '', + engine: ClapWorkflowEngine.REST_API, + provider: ClapWorkflowProvider.ELEVENLABS, + category: ClapWorkflowCategory.VOICE_GENERATION, + data: 'v1/text-to-speech', + /** + * Inputs of the workflow (this is used to build an UI for the automatically) + */ + inputFields: [genericPrompt], + inputValues: { + prompt: genericPrompt.defaultValue, + }, + }, + { + id: 'elevenlabs://v1/sound-generation', + label: 'Sound Generation V1', + description: '', + tags: ['sound'], + author: 'ElevenLabs', + thumbnailUrl: '', + engine: ClapWorkflowEngine.REST_API, + provider: ClapWorkflowProvider.ELEVENLABS, + category: ClapWorkflowCategory.SOUND_GENERATION, + data: 'v1/sound-generation', + /** + * Inputs of the workflow (this is used to build an UI for the automatically) + */ + inputFields: [genericPrompt], + inputValues: { + [genericPrompt.id]: genericPrompt.defaultValue, + }, + }, +] diff --git a/src/services/editors/workflow-editor/workflows/falai/defaultWorkflows.ts b/src/services/editors/workflow-editor/workflows/falai/defaultWorkflows.ts index cc2a60a9..793aa9f8 100644 --- a/src/services/editors/workflow-editor/workflows/falai/defaultWorkflows.ts +++ b/src/services/editors/workflow-editor/workflows/falai/defaultWorkflows.ts @@ -6,11 +6,21 @@ import { } from '@aitube/clap' import { + genericAudio, genericHeight1024, genericHeight2048, - genericTextPrompt, + genericImage, + genericInput, + genericSeed, + genericPrompt, + genericVideo, genericWidth1024, genericWidth2048, + genericMotionBucketId, + genericConditioningAugmentation, + genericUpscalingFactor, + genericOverlappingTiles, + genericInferenceSteps, } from '../common/defaultValues' /* @@ -25,6 +35,62 @@ TODO: add those as well */ export const defaultWorkflows: ClapWorkflow[] = [ + { + id: 'falai://fal-ai/stable-video', + label: 'Stable Video Diffusion', + description: '', + tags: ['video'], + author: '', + thumbnailUrl: '', + engine: ClapWorkflowEngine.REST_API, + provider: ClapWorkflowProvider.FALAI, + category: ClapWorkflowCategory.VIDEO_GENERATION, + data: 'fal-ai/stable-video', + inputFields: [ + { + ...genericImage, + id: 'image_url', + }, + genericSeed, + genericMotionBucketId, + { + ...genericConditioningAugmentation, + id: 'cond_aug', + }, + ], + inputValues: { + image_url: genericImage.defaultValue, + [genericMotionBucketId.id]: 55, + cond_aug: 0.02, + }, + }, + { + id: 'falai://fal-ai/flux-realism', + label: 'Flux Realism LoRA', + description: '', + tags: ['flux', 'LoRA', 'realism'], + author: '', + thumbnailUrl: '', + engine: ClapWorkflowEngine.REST_API, + provider: ClapWorkflowProvider.FALAI, + category: ClapWorkflowCategory.IMAGE_GENERATION, + data: 'fal-ai/flux-realism', + /** + * Inputs of the workflow (this is used to build an UI for the automatically) + */ + inputFields: [ + genericPrompt, + genericWidth2048, + genericHeight2048, + genericInferenceSteps, + ], + inputValues: { + [genericPrompt.id]: genericPrompt.defaultValue, + [genericWidth2048.id]: genericWidth2048.defaultValue, + [genericHeight2048.id]: genericHeight2048.defaultValue, + [genericInferenceSteps.id]: genericInferenceSteps.defaultValue, + }, + }, { id: 'falai://fal-ai/flux-pro', label: 'FLUX.1 [pro]', @@ -40,16 +106,16 @@ export const defaultWorkflows: ClapWorkflow[] = [ * Inputs of the workflow (this is used to build an UI for the automatically) */ inputFields: [ - genericTextPrompt, + genericPrompt, genericWidth2048, genericHeight2048, // TODO: add guidance scale and number of steps ], inputValues: { - prompt: genericTextPrompt.defaultValue, - width: genericWidth2048.defaultValue, - height: genericHeight2048.defaultValue, + [genericPrompt.id]: genericPrompt.defaultValue, + [genericWidth2048.id]: genericWidth2048.defaultValue, + [genericHeight2048.id]: genericHeight2048.defaultValue, // TODO: add guidance scale and number of steps }, @@ -65,9 +131,9 @@ export const defaultWorkflows: ClapWorkflow[] = [ provider: ClapWorkflowProvider.FALAI, category: ClapWorkflowCategory.IMAGE_GENERATION, data: 'fal-ai/flux-schnell', - inputFields: [genericTextPrompt, genericWidth2048, genericHeight2048], + inputFields: [genericPrompt, genericWidth2048, genericHeight2048], inputValues: { - prompt: genericTextPrompt.defaultValue, + prompt: genericPrompt.defaultValue, width: genericWidth2048.defaultValue, height: genericHeight2048.defaultValue, }, @@ -87,16 +153,16 @@ export const defaultWorkflows: ClapWorkflow[] = [ * Inputs of the workflow (this is used to build an UI for the automatically) */ inputFields: [ - genericTextPrompt, + genericPrompt, genericWidth2048, genericHeight2048, // TODO: add guidance scale and number of steps ], inputValues: { - prompt: genericTextPrompt.defaultValue, - width: genericWidth2048.defaultValue, - height: genericHeight2048.defaultValue, + [genericPrompt.id]: genericPrompt.defaultValue, + [genericWidth2048.id]: genericWidth2048.defaultValue, + [genericHeight2048.id]: genericHeight2048.defaultValue, // TODO: add guidance scale and number of steps }, @@ -116,16 +182,16 @@ export const defaultWorkflows: ClapWorkflow[] = [ * Inputs of the workflow (this is used to build an UI for the automatically) */ inputFields: [ - genericTextPrompt, + genericPrompt, genericWidth1024, genericHeight1024, // TODO: add guidance scale and number of steps ], inputValues: { - prompt: genericTextPrompt.defaultValue, - width: genericWidth1024.defaultValue, - height: genericHeight1024.defaultValue, + [genericPrompt.id]: genericPrompt.defaultValue, + [genericWidth1024.id]: genericWidth1024.defaultValue, + [genericHeight1024.id]: genericHeight1024.defaultValue, // TODO: add guidance scale and number of steps }, @@ -145,18 +211,98 @@ export const defaultWorkflows: ClapWorkflow[] = [ * Inputs of the workflow (this is used to build an UI for the automatically) */ inputFields: [ - genericTextPrompt, + genericPrompt, genericWidth1024, genericHeight1024, // TODO: add guidance scale and number of steps ], inputValues: { - prompt: genericTextPrompt.defaultValue, - width: genericWidth1024.defaultValue, - height: genericHeight1024.defaultValue, + [genericPrompt.id]: genericPrompt.defaultValue, + [genericWidth1024.id]: genericWidth1024.defaultValue, + [genericHeight1024.id]: genericHeight1024.defaultValue, // TODO: add guidance scale and number of steps }, }, + { + id: 'falai://fal-ai/aura-sr', + label: 'MetaVoice V1', + description: '', + tags: ['upscaling'], + author: 'AuraSR', + thumbnailUrl: '', + engine: ClapWorkflowEngine.REST_API, + provider: ClapWorkflowProvider.FALAI, + category: ClapWorkflowCategory.IMAGE_UPSCALING, + data: 'fal-ai/aura-sr', + inputFields: [ + { + ...genericImage, + id: 'image_url', + }, + genericUpscalingFactor, + genericOverlappingTiles, + ], + inputValues: { + image_url: genericImage.defaultValue, + [genericUpscalingFactor.id]: genericUpscalingFactor.defaultValue, + [genericOverlappingTiles.id]: genericOverlappingTiles.defaultValue, + }, + }, + { + id: 'falai://fal-ai/metavoice-v1', + label: 'MetaVoice V1', + description: '', + tags: ['TTS'], + author: 'MetaVoice (themetavoice.xyz)', + thumbnailUrl: '', + engine: ClapWorkflowEngine.REST_API, + provider: ClapWorkflowProvider.FALAI, + category: ClapWorkflowCategory.VOICE_GENERATION, + data: 'fal-ai/metavoice-v1', + /** + * Inputs of the workflow (this is used to build an UI for the automatically) + */ + inputFields: [genericPrompt, { ...genericAudio, id: 'audio_url' }], + inputValues: { + [genericPrompt.id]: genericPrompt.defaultValue, + + // TODO: this should be our own voice instead + // PS: are you implementing this task? please do a search in the code for speakers/bria.mp3 + audio_url: 'https://cdn.themetavoice.xyz/speakers/bria.mp3', + }, + }, + { + id: 'falai://fal-ai/stable-audio', + label: 'Stable Audio', + description: '', + tags: ['audio', 'sound'], + author: 'Stability AI', + thumbnailUrl: '', + engine: ClapWorkflowEngine.REST_API, + provider: ClapWorkflowProvider.FALAI, + category: ClapWorkflowCategory.SOUND_GENERATION, + data: 'fal-ai/stable-audio', + inputFields: [genericPrompt], + inputValues: { + [genericPrompt.id]: genericPrompt.defaultValue, + }, + }, + { + id: 'falai://fal-ai/stable-audio', + label: 'Stable Audio', + description: '', + tags: ['audio', 'music'], + author: 'Stability AI', + thumbnailUrl: '', + engine: ClapWorkflowEngine.REST_API, + provider: ClapWorkflowProvider.FALAI, + category: ClapWorkflowCategory.MUSIC_GENERATION, + data: 'fal-ai/stable-audio', + inputFields: [genericPrompt], + inputValues: { + [genericPrompt.id]: genericPrompt.defaultValue, + }, + }, ] diff --git a/src/services/editors/workflow-editor/workflows/fireworksai/index.ts b/src/services/editors/workflow-editor/workflows/fireworksai/index.ts index 48ef8971..08327e38 100644 --- a/src/services/editors/workflow-editor/workflows/fireworksai/index.ts +++ b/src/services/editors/workflow-editor/workflows/fireworksai/index.ts @@ -6,7 +6,7 @@ import { } from '@aitube/clap' import { genericHeight1024, - genericTextPrompt, + genericPrompt, genericWidth1024, } from '../common/defaultValues' @@ -26,14 +26,14 @@ export const fireworksaiWorkflows: ClapWorkflow[] = [ * Inputs of the workflow (this is used to build an UI for the automatically) */ inputFields: [ - genericTextPrompt, + genericPrompt, genericWidth1024, genericHeight1024, // TODO: add guidance scale and number of steps ], inputValues: { - prompt: genericTextPrompt.defaultValue, + prompt: genericPrompt.defaultValue, width: genericWidth1024.defaultValue, height: genericHeight1024.defaultValue, @@ -55,14 +55,14 @@ export const fireworksaiWorkflows: ClapWorkflow[] = [ * Inputs of the workflow (this is used to build an UI for the automatically) */ inputFields: [ - genericTextPrompt, + genericPrompt, genericWidth1024, genericHeight1024, // TODO: add guidance scale and number of steps ], inputValues: { - prompt: genericTextPrompt.defaultValue, + prompt: genericPrompt.defaultValue, width: genericWidth1024.defaultValue, height: genericHeight1024.defaultValue, @@ -84,14 +84,14 @@ export const fireworksaiWorkflows: ClapWorkflow[] = [ * Inputs of the workflow (this is used to build an UI for the automatically) */ inputFields: [ - genericTextPrompt, + genericPrompt, genericWidth1024, genericHeight1024, // TODO: add guidance scale and number of steps ], inputValues: { - prompt: genericTextPrompt.defaultValue, + prompt: genericPrompt.defaultValue, width: genericWidth1024.defaultValue, height: genericHeight1024.defaultValue, @@ -113,14 +113,14 @@ export const fireworksaiWorkflows: ClapWorkflow[] = [ * Inputs of the workflow (this is used to build an UI for the automatically) */ inputFields: [ - genericTextPrompt, + genericPrompt, genericWidth1024, genericHeight1024, // TODO: add guidance scale and number of steps ], inputValues: { - prompt: genericTextPrompt.defaultValue, + prompt: genericPrompt.defaultValue, width: genericWidth1024.defaultValue, height: genericHeight1024.defaultValue, diff --git a/src/services/editors/workflow-editor/workflows/huggingface/index.ts b/src/services/editors/workflow-editor/workflows/huggingface/index.ts index b9fcd97f..c9f4718b 100644 --- a/src/services/editors/workflow-editor/workflows/huggingface/index.ts +++ b/src/services/editors/workflow-editor/workflows/huggingface/index.ts @@ -4,5 +4,212 @@ import { ClapWorkflowCategory, ClapWorkflowProvider, } from '@aitube/clap' +import { + genericHeight2048, + genericPrompt, + genericWidth2048, +} from '../common/defaultValues' + +export const huggingfaceWorkflows: ClapWorkflow[] = [ + { + id: 'huggingface://black-forest-labs/FLUX.1-schnell', + label: 'FLUX.1 [schnell]', + description: '', + tags: ['flux'], + author: 'BFL (https://BlackForestLabs.ai)', + thumbnailUrl: '', + engine: ClapWorkflowEngine.REST_API, + provider: ClapWorkflowProvider.HUGGINGFACE, + category: ClapWorkflowCategory.IMAGE_GENERATION, + data: 'black-forest-labs/FLUX.1-schnell', + inputFields: [genericPrompt, genericWidth2048, genericHeight2048], + inputValues: { + prompt: genericPrompt.defaultValue, + width: genericWidth2048.defaultValue, + height: genericHeight2048.defaultValue, + }, + }, + { + id: 'huggingface://black-forest-labs/FLUX.1-dev', + label: 'FLUX.1 [dev]', + description: '', + tags: ['flux'], + author: 'BFL (https://BlackForestLabs.ai)', + thumbnailUrl: '', + engine: ClapWorkflowEngine.REST_API, + provider: ClapWorkflowProvider.HUGGINGFACE, + category: ClapWorkflowCategory.IMAGE_GENERATION, + data: 'black-forest-labs/FLUX.1-dev', + /** + * Inputs of the workflow (this is used to build an UI for the automatically) + */ + inputFields: [ + genericPrompt, + genericWidth2048, + genericHeight2048, + + // TODO: add guidance scale and number of steps + ], + inputValues: { + prompt: genericPrompt.defaultValue, + width: genericWidth2048.defaultValue, + height: genericHeight2048.defaultValue, -export const huggingfaceWorkflows: ClapWorkflow[] = [] + // TODO: add guidance scale and number of steps + }, + }, + { + id: 'huggingface://coqui/XTTS-v2', + label: 'Coqui XTTS-v2', + description: '', + tags: ['TTS'], + author: 'Coqui', + thumbnailUrl: '', + engine: ClapWorkflowEngine.REST_API, + provider: ClapWorkflowProvider.HUGGINGFACE, + category: ClapWorkflowCategory.VOICE_GENERATION, + data: 'coqui/XTTS-v2', + /** + * Inputs of the workflow (this is used to build an UI for the automatically) + */ + inputFields: [genericPrompt], + inputValues: { + prompt: genericPrompt.defaultValue, + }, + }, + { + id: 'huggingface://myshell-ai/OpenVoiceV2', + label: 'MyShell.ai OpenVoiceV2', + description: '', + tags: ['TTS'], + author: 'MyShell.ai', + thumbnailUrl: '', + engine: ClapWorkflowEngine.REST_API, + provider: ClapWorkflowProvider.HUGGINGFACE, + category: ClapWorkflowCategory.VOICE_GENERATION, + data: 'myshell-ai/OpenVoiceV2', + /** + * Inputs of the workflow (this is used to build an UI for the automatically) + */ + inputFields: [genericPrompt], + inputValues: { + prompt: genericPrompt.defaultValue, + }, + }, + { + id: 'huggingface://myshell-ai/OpenVoice', + label: 'MyShell.ai OpenVoice', + description: '', + tags: ['TTS'], + author: 'MyShell.ai', + thumbnailUrl: '', + engine: ClapWorkflowEngine.REST_API, + provider: ClapWorkflowProvider.HUGGINGFACE, + category: ClapWorkflowCategory.VOICE_GENERATION, + data: 'myshell-ai/OpenVoice', + /** + * Inputs of the workflow (this is used to build an UI for the automatically) + */ + inputFields: [genericPrompt], + inputValues: { + prompt: genericPrompt.defaultValue, + }, + }, + { + id: 'huggingface://WhisperSpeech/WhisperSpeech', + label: 'WhisperSpeech', + description: '', + tags: ['TTS'], + author: 'WhisperSpeech', + thumbnailUrl: '', + engine: ClapWorkflowEngine.REST_API, + provider: ClapWorkflowProvider.HUGGINGFACE, + category: ClapWorkflowCategory.VOICE_GENERATION, + data: 'WhisperSpeech/WhisperSpeech', + /** + * Inputs of the workflow (this is used to build an UI for the automatically) + */ + inputFields: [genericPrompt], + inputValues: { + prompt: genericPrompt.defaultValue, + }, + }, + { + id: 'huggingface://metavoiceio/metavoice-1B-v0.1', + label: 'MetaVoice 1B v0.1', + description: '', + tags: ['TTS'], + author: 'MetaVoice (themetavoice.xyz)', + thumbnailUrl: '', + engine: ClapWorkflowEngine.REST_API, + provider: ClapWorkflowProvider.HUGGINGFACE, + category: ClapWorkflowCategory.VOICE_GENERATION, + data: 'metavoiceio/metavoice-1B-v0.1', + /** + * Inputs of the workflow (this is used to build an UI for the automatically) + */ + inputFields: [genericPrompt], + inputValues: { + prompt: genericPrompt.defaultValue, + }, + }, + { + id: 'huggingface://parler-tts/parler_tts_mini_v0.1', + label: 'ParlerTTS Mini v0.1', + description: '', + tags: ['TTS'], + author: 'ParlerTTS', + thumbnailUrl: '', + engine: ClapWorkflowEngine.REST_API, + provider: ClapWorkflowProvider.HUGGINGFACE, + category: ClapWorkflowCategory.VOICE_GENERATION, + data: 'parler-tts/parler_tts_mini_v0.1', + /** + * Inputs of the workflow (this is used to build an UI for the automatically) + */ + inputFields: [genericPrompt], + inputValues: { + prompt: genericPrompt.defaultValue, + }, + }, + { + id: 'huggingface://parler-tts/parler-tts-mini-expresso', + label: 'ParlerTTS Mini Expresso v0.1', + description: '', + tags: ['TTS'], + author: 'ParlerTTS', + thumbnailUrl: '', + engine: ClapWorkflowEngine.REST_API, + provider: ClapWorkflowProvider.HUGGINGFACE, + category: ClapWorkflowCategory.VOICE_GENERATION, + data: 'parler-tts/parler-tts-mini-expresso', + /** + * Inputs of the workflow (this is used to build an UI for the automatically) + */ + inputFields: [genericPrompt], + inputValues: { + prompt: genericPrompt.defaultValue, + }, + }, + /* + { + id: 'huggingface://cvssp/audioldm2-music', + label: 'CVSSP AudioLDM2 Music', + description: '', + tags: ['music'], + author: 'CVSSP', + thumbnailUrl: '', + engine: ClapWorkflowEngine.REST_API, + provider: ClapWorkflowProvider.HUGGINGFACE, + category: ClapWorkflowCategory.MUSIC_GENERATION, + data: 'cvssp/audioldm2-music', + // Inputs of the workflow (this is used to build an UI for the automatically) + inputFields: [ + genericPrompt, + ], + inputValues: { + prompt: genericPrompt.defaultValue, + }, + }, + */ +] diff --git a/src/services/editors/workflow-editor/workflows/index.ts b/src/services/editors/workflow-editor/workflows/index.ts index 72aa8feb..7c3db5ba 100644 --- a/src/services/editors/workflow-editor/workflows/index.ts +++ b/src/services/editors/workflow-editor/workflows/index.ts @@ -1,17 +1,23 @@ import { ClapWorkflow } from '@aitube/clap' import { falaiWorkflows } from './falai' +import { elevenlabsWorkflows } from './elevenlabs' import { stabilityaiWorkflows } from './stabilityai' import { replicateWorkflows } from './replicate' import { fireworksaiWorkflows } from './fireworksai' import { huggingfaceWorkflows } from './huggingface' import { comfyicuWorkflows } from './comfyicu' +// I haven't ported all the workflows yet, there are still some here +// (eg. utilities for segmentation etc) +// https://github.com/jbilcke-hf/clapper/blob/872298838ea3721f9945140fb00f0239b253b172/src/components/settings/constants.ts#L329 + export const workflows: ClapWorkflow[] = [ + ...comfyicuWorkflows, + ...elevenlabsWorkflows, ...falaiWorkflows, - ...stabilityaiWorkflows, - ...replicateWorkflows, ...fireworksaiWorkflows, ...huggingfaceWorkflows, - ...comfyicuWorkflows, + ...replicateWorkflows, + ...stabilityaiWorkflows, ] diff --git a/src/services/editors/workflow-editor/workflows/openart/samples/openart_workflow.json b/src/services/editors/workflow-editor/workflows/openart/samples/openart_workflow.json new file mode 100644 index 00000000..f3463396 --- /dev/null +++ b/src/services/editors/workflow-editor/workflows/openart/samples/openart_workflow.json @@ -0,0 +1,77 @@ +{ + "id": "kfpq7gwn6oE7ygaJTrOX", + "creator": { + "uid": "0y3EaraLmvz2vAhd90LE", + "name": "LIOR ROITER", + "bio": "", + "avatar": "https://lh3.googleusercontent.com/a/ACg8ocIrkLPmFolGbYnDOYrUJ9jRACooqDRlGQb-u_ilE91G=s96-c", + "username": "elephant_gleaming_9" + }, + "updated_at": { + "_seconds": 1723063308, + "_nanoseconds": 705000000 + }, + "cloud_app_id": "wf-17fef4f5716d481983cfd4a8dd87d0c4", + "stats": { + "num_shares": 0, + "rating": 5, + "num_reviews": 0, + "num_runs": 0, + "num_bookmarks": 0, + "num_likes": 4, + "num_comments": 4, + "num_views": 2101, + "num_downloads": 719 + }, + "nodes_index": [ + "Note", + "Reroute", + "FluxGuidance", + "Image Comparer (rgthree)", + "EmptySD3LatentImage", + ".", + "ComfyUI", + "SaveImage", + "UpscaleModelLoader", + "VAEDecode", + "CLIPTextEncode", + "UNETLoader", + "KSampler", + "VAELoader", + "DualCLIPLoader", + ",", + "pythongosssss/ComfyUI-Custom-Scripts", + "MathExpression|pysssss", + ",", + "ComfyUI Easy Use", + "easy float", + ",", + "comfyui-art-venture", + "SDXLAspectRatioSelector", + ",", + "UltimateSDUpscale", + "UltimateSDUpscale", + "," + ], + "name": "Flux + UltimateSDUpscale - fix workflow (from author datou)", + "is_public": true, + "description": "

workflow use flux (flux1-dev-fp8.safetensors) for Image sharpening and upscale.

i didnt build this workflow. It belongs to the legendary creator datou.

But his flow didn't work and returned errors due to incorrect settings.

So I fixed it, and here it is working again


you need to use:

flux1-dev-fp8.safetensors

https://huggingface.co/Comfy-Org/flux1-dev/blob/main/flux1-dev-fp8.safetensors

t5xxl_fp16.safetensors

https://huggingface.co/comfyanonymous/flux_text_encoders/blob/main/t5xxl_fp16.safetensors

clip_l.safetensors

https://huggingface.co/comfyanonymous/flux_text_encoders/blob/main/clip_l.safetensors

ae.sft (rename ae.safetensors to ae.sft)

https://huggingface.co/black-forest-labs/FLUX.1-schnell/tree/main




", + "created_at": { + "_seconds": 1723063308, + "_nanoseconds": 705000000 + }, + "categories": ["Upscale", "SDXL"], + "thumbnails": [ + { + "width": 1664, + "url": "https://cdn.openart.ai/workflow_thumbnails/0y3EaraLmvz2vAhd90LE/image_AJJZt677_1723063305485_raw.jpg", + "height": 2432 + }, + { + "width": 832, + "url": "https://cdn.openart.ai/workflow_thumbnails/0y3EaraLmvz2vAhd90LE/image_uduKFf9T_1723063305136_raw.jpg", + "height": 1216 + } + ], + "key": "flux-ultimatesdupscale---fix-workflow-from-author-datou" +} diff --git a/src/services/editors/workflow-editor/workflows/replicate/defaultWorkflows.ts b/src/services/editors/workflow-editor/workflows/replicate/defaultWorkflows.ts index 579bb275..bfcde08e 100644 --- a/src/services/editors/workflow-editor/workflows/replicate/defaultWorkflows.ts +++ b/src/services/editors/workflow-editor/workflows/replicate/defaultWorkflows.ts @@ -8,7 +8,9 @@ import { import { genericHeight1024, genericHeight2048, - genericTextPrompt, + genericImage, + genericPrompt, + genericVideo, genericWidth1024, genericWidth2048, } from '../common/defaultValues' @@ -29,14 +31,14 @@ export const defaultWorkflows: ClapWorkflow[] = [ * Inputs of the workflow (this is used to build an UI for the automatically) */ inputFields: [ - genericTextPrompt, + genericPrompt, genericWidth2048, genericHeight2048, // TODO: add guidance scale and number of steps ], inputValues: { - prompt: genericTextPrompt.defaultValue, + prompt: genericPrompt.defaultValue, width: genericWidth2048.defaultValue, height: genericHeight2048.defaultValue, @@ -57,9 +59,9 @@ export const defaultWorkflows: ClapWorkflow[] = [ /** * Inputs of the workflow (this is used to build an UI for the automatically) */ - inputFields: [genericTextPrompt, genericWidth2048, genericHeight2048], + inputFields: [genericPrompt, genericWidth2048, genericHeight2048], inputValues: { - prompt: genericTextPrompt.defaultValue, + prompt: genericPrompt.defaultValue, width: genericWidth2048.defaultValue, height: genericHeight2048.defaultValue, }, @@ -79,14 +81,14 @@ export const defaultWorkflows: ClapWorkflow[] = [ * Inputs of the workflow (this is used to build an UI for the automatically) */ inputFields: [ - genericTextPrompt, + genericPrompt, genericWidth2048, genericHeight2048, // TODO: add guidance scale and number of steps ], inputValues: { - prompt: genericTextPrompt.defaultValue, + prompt: genericPrompt.defaultValue, width: genericWidth2048.defaultValue, height: genericHeight2048.defaultValue, @@ -108,18 +110,74 @@ export const defaultWorkflows: ClapWorkflow[] = [ * Inputs of the workflow (this is used to build an UI for the automatically) */ inputFields: [ - genericTextPrompt, + genericPrompt, genericWidth1024, genericHeight1024, // TODO: add guidance scale and number of steps ], inputValues: { - prompt: genericTextPrompt.defaultValue, + prompt: genericPrompt.defaultValue, width: genericWidth1024.defaultValue, height: genericHeight1024.defaultValue, // TODO: add guidance scale and number of steps }, }, + { + id: 'replicate://chenxwh/openvoice', + label: 'OpenVoice V2', + description: '', + tags: ['OpenVoice'], + author: '@chenxwh', + thumbnailUrl: '', + engine: ClapWorkflowEngine.REST_API, + provider: ClapWorkflowProvider.REPLICATE, + category: ClapWorkflowCategory.VOICE_GENERATION, + data: 'chenxwh/openvoice', + /** + * Inputs of the workflow (this is used to build an UI for the automatically) + */ + inputFields: [genericPrompt], + inputValues: { + prompt: genericPrompt.defaultValue, + }, + }, + { + id: 'replicate://lucataco/real-esrgan-video', + label: 'Real-ESRGAN', + description: '', + tags: ['video', 'upscaling'], + author: '', + thumbnailUrl: '', + engine: ClapWorkflowEngine.REST_API, + provider: ClapWorkflowProvider.REPLICATE, + category: ClapWorkflowCategory.VIDEO_UPSCALING, + data: 'lucataco/real-esrgan-video', + inputFields: [ + { + ...genericVideo, + id: 'video_path', + }, + ], + inputValues: { + video_path: genericVideo.defaultValue, + }, + }, + { + id: 'replicate://nightmareai/real-esrgan', + label: 'Real-ESRGAN', + description: '', + tags: ['image', 'upscaling'], + author: '', + thumbnailUrl: '', + engine: ClapWorkflowEngine.REST_API, + provider: ClapWorkflowProvider.REPLICATE, + category: ClapWorkflowCategory.IMAGE_UPSCALING, + data: 'nightmareai/real-esrgan', + inputFields: [genericImage], + inputValues: { + prompt: genericImage.defaultValue, + }, + }, ] diff --git a/src/services/editors/workflow-editor/workflows/stabilityai/index.ts b/src/services/editors/workflow-editor/workflows/stabilityai/index.ts index 0fc5db4d..358f9e99 100644 --- a/src/services/editors/workflow-editor/workflows/stabilityai/index.ts +++ b/src/services/editors/workflow-editor/workflows/stabilityai/index.ts @@ -7,12 +7,32 @@ import { import { genericHeight1024, - genericTextPrompt, + genericImage, + genericPrompt, genericWidth1024, genericWidth2048, } from '../common/defaultValues' export const stabilityaiWorkflows: ClapWorkflow[] = [ + { + id: 'stabilityai://image-to-video', + label: 'Image To Video', + description: '', + tags: ['SVD'], + author: 'Stability AI', + thumbnailUrl: '', + engine: ClapWorkflowEngine.REST_API, + provider: ClapWorkflowProvider.STABILITYAI, + category: ClapWorkflowCategory.VIDEO_GENERATION, + data: 'image-to-video', + /** + * Inputs of the workflow (this is used to build an UI for the automatically) + */ + inputFields: [genericImage], + inputValues: { + image: genericPrompt.defaultValue, + }, + }, { id: 'stabilityai://stable-image/generate/ultra', label: 'Stable Image Ultra', @@ -21,21 +41,21 @@ export const stabilityaiWorkflows: ClapWorkflow[] = [ author: 'Stability AI', thumbnailUrl: '', engine: ClapWorkflowEngine.REST_API, - provider: ClapWorkflowProvider.FALAI, + provider: ClapWorkflowProvider.STABILITYAI, category: ClapWorkflowCategory.IMAGE_GENERATION, data: 'stable-image/generate/ultra', /** * Inputs of the workflow (this is used to build an UI for the automatically) */ inputFields: [ - genericTextPrompt, + genericPrompt, genericWidth1024, genericHeight1024, // TODO: add guidance scale and number of steps ], inputValues: { - prompt: genericTextPrompt.defaultValue, + prompt: genericPrompt.defaultValue, width: genericWidth1024.defaultValue, height: genericHeight1024.defaultValue, @@ -50,21 +70,21 @@ export const stabilityaiWorkflows: ClapWorkflow[] = [ author: 'Stability AI', thumbnailUrl: '', engine: ClapWorkflowEngine.REST_API, - provider: ClapWorkflowProvider.FALAI, + provider: ClapWorkflowProvider.STABILITYAI, category: ClapWorkflowCategory.IMAGE_GENERATION, data: 'stable-image/generate/core', /** * Inputs of the workflow (this is used to build an UI for the automatically) */ inputFields: [ - genericTextPrompt, + genericPrompt, genericWidth1024, genericHeight1024, // TODO: add guidance scale and number of steps ], inputValues: { - prompt: genericTextPrompt.defaultValue, + prompt: genericPrompt.defaultValue, width: genericWidth1024.defaultValue, height: genericHeight1024.defaultValue, @@ -79,21 +99,21 @@ export const stabilityaiWorkflows: ClapWorkflow[] = [ author: 'Stability AI', thumbnailUrl: '', engine: ClapWorkflowEngine.REST_API, - provider: ClapWorkflowProvider.FALAI, + provider: ClapWorkflowProvider.STABILITYAI, category: ClapWorkflowCategory.IMAGE_GENERATION, data: 'stable-image/generate/sd3', /** * Inputs of the workflow (this is used to build an UI for the automatically) */ inputFields: [ - genericTextPrompt, + genericPrompt, genericWidth1024, genericHeight1024, // TODO: add guidance scale and number of steps ], inputValues: { - prompt: genericTextPrompt.defaultValue, + prompt: genericPrompt.defaultValue, width: genericWidth1024.defaultValue, height: genericHeight1024.defaultValue, diff --git a/src/services/resolver/useResolver.ts b/src/services/resolver/useResolver.ts index 04d6e64b..6ac5683d 100644 --- a/src/services/resolver/useResolver.ts +++ b/src/services/resolver/useResolver.ts @@ -18,7 +18,13 @@ import { segmentVisibilityPriority, TimelineSegment, } from '@aitube/timeline' -import { getVideoPrompt } from '@aitube/engine' +import { + getBackgroundAudioPrompt, + getSoundPrompt, + getMusicPrompt, + getSpeechForegroundAudioPrompt, + getVideoPrompt, +} from '@aitube/engine' import { ResolverStore } from '@aitube/clapper-services' import { getDefaultResolverState } from './getDefaultResolverState' @@ -522,6 +528,13 @@ export const useResolver = create((set, get) => ({ .filter((x) => x) .join(', ') + const positiveVoicePrompt = getSpeechForegroundAudioPrompt(segments) + const positiveAudioPrompt = getSoundPrompt(segments) + const positiveMusicPrompt = getMusicPrompt(segments) + + // we can also create a background audio ambiance by calling + // getBackgroundAudioPrompt() + // note: not all AI models will support those parameters. // in 2024, even the "best" proprietary video models like Sora, Veo, Kling, Gen-3, Dream Machine etc.. // don't support voice input for lip syncing, for instance. @@ -541,7 +554,15 @@ export const useResolver = create((set, get) => ({ }, voice: { identity: `${mainCharacterEntity?.audioId || ''}`, - positive: '', + positive: positiveVoicePrompt, + negative: '', + }, + audio: { + positive: positiveAudioPrompt, + negative: '', + }, + music: { + positive: positiveMusicPrompt, negative: '', }, }