diff --git a/bun.lockb b/bun.lockb index b283d947..1d4fb537 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/package.json b/package.json index f775ffc5..4949d9eb 100644 --- a/package.json +++ b/package.json @@ -1,22 +1,12 @@ { "name": "@aitube/clapper-monorepo", "version": "0.2.4", - "private": true, "description": "A monorepo for the Clapper project. Individual packages are in the packages directory.", - "workspaces": [ - "packages/clap", - "packages/timeline", - "packages/api-client", - "packages/io", - "packages/colors", - "packages/engine", - "packages/broadway", - "packages/clapper-services", - "packages/app" - ], "engines": { "bun": ">=1.0.0" }, + "packageManager": "bun@1.0.25", + "private": true, "scripts": { "dev": "bun run --cwd packages/app dev", "start": "bun run --cwd packages/app start", @@ -36,10 +26,20 @@ "test:all": "bun run --cwd packages/clap test && bun run --cwd packages/timeline test && bun run --cwd packages/api-client test && bun run --cwd packages/io test && bun run --cwd packages/colors test && bun run --cwd packages/engine test && bun run --cwd packages/broadway test && bun run --cwd packages/clapper-services test && bun run --cwd packages/app test", "format": "bun run --cwd packages/app format" }, - "packageManager": "bun@1.0.25", "trustedDependencies": [ "@aitube/clapper", "onnxruntime-node", "protobufjs" + ], + "workspaces": [ + "packages/clap", + "packages/timeline", + "packages/api-client", + "packages/io", + "packages/colors", + "packages/engine", + "packages/broadway", + "packages/clapper-services", + "packages/app" ] } \ No newline at end of file diff --git a/packages/api-client/package.json b/packages/api-client/package.json index 67462a2f..bb4bb01b 100644 --- a/packages/api-client/package.json +++ b/packages/api-client/package.json @@ -39,6 +39,8 @@ "dist/**/*.d.ts" ], "dependencies": { + "@aitube/clap": "workspace:*", + "@types/bun": "latest", "query-string": "^9.0.0" } } diff --git a/packages/app/package.json b/packages/app/package.json index f58653b1..f10d3c01 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -41,7 +41,7 @@ "@aitube/clapper-services": "workspace:*", "@aitube/engine": "workspace:*", "@aitube/timeline": "workspace:*", - "@fal-ai/serverless-client": "^0.13.0", + "@fal-ai/serverless-client": "^0.14.2", "@ffmpeg/ffmpeg": "^0.12.10", "@ffmpeg/util": "^0.12.1", "@gradio/client": "^1.5.0", diff --git a/packages/app/src/app/api/resolve/providers/replicate/index.ts b/packages/app/src/app/api/resolve/providers/replicate/index.ts index d3cfe32e..ce5b72ca 100644 --- a/packages/app/src/app/api/resolve/providers/replicate/index.ts +++ b/packages/app/src/app/api/resolve/providers/replicate/index.ts @@ -1,8 +1,11 @@ import Replicate from 'replicate' -import { ClapSegmentCategory } from '@aitube/clap' +import { ClapMediaOrientation, ClapSegmentCategory } from '@aitube/clap' import { ResolveRequest } from '@aitube/clapper-services' import { TimelineSegment } from '@aitube/timeline' +import { getWorkflowInputValues } from '../getWorkflowInputValues' +import { defaultLoraModels } from '@/services/editors/workflow-editor/workflows/common/loras' +import { getWorkflowLora } from '@/services/editors/workflow-editor/workflows/common/loras/getWorkflowLora' export async function resolveSegment( request: ResolveRequest @@ -12,22 +15,28 @@ export async function resolveSegment( } const replicate = new Replicate({ auth: request.settings.replicateApiKey }) - if (request.segment.category !== ClapSegmentCategory.STORYBOARD) { - throw new Error( - `Clapper doesn't support ${request.segment.category} generation for provider "Replicate". Please open a pull request with (working code) to solve this!` - ) - } - const segment = request.segment - // this mapping isn't great, we should use something auto-adapting - // like we are doing for Hugging Face (match the fields etc) - if (request.segment.category === ClapSegmentCategory.STORYBOARD) { + if (request.segment.category == ClapSegmentCategory.STORYBOARD) { + + const { workflowValues } = getWorkflowInputValues( + request.settings.imageGenerationWorkflow + ) + let params: object = { prompt: request.prompts.image.positive, width: request.meta.width, height: request.meta.height, + disable_safety_checker: !request.settings.censorNotForAllAudiencesContent, } + + const aspectRatio = + request.meta.orientation === ClapMediaOrientation.SQUARE + ? "1:1" + : request.meta.orientation === ClapMediaOrientation.PORTRAIT + ? "9:16" + : "16:9" + if ( request.settings.imageGenerationWorkflow.data === 'fofr/pulid-lightning' ) { @@ -35,6 +44,28 @@ export async function resolveSegment( ...params, face_image: request.prompts.image.identity, } + } else if ( + request.settings.imageGenerationWorkflow.data === 'lucataco/flux-dev-lora' + ) { + + // note: this isn't the right place to do this, because maybe the LoRAs are dynamic + const loraModel = getWorkflowLora(request.settings.imageGenerationWorkflow) + + params = { + // for some reason this model doesn't support arbitrary width and height, + // at least not at the time of writing.. + aspect_ratio: aspectRatio, + + hf_lora: workflowValues['hf_lora'] || '', + + prompt: [ + loraModel?.trigger, + request.prompts.image.positive + ].filter(x => x).join(' '), + + disable_safety_checker: !request.settings.censorNotForAllAudiencesContent, + } + } else if ( request.settings.imageGenerationWorkflow.data === 'zsxkib/pulid' ) { @@ -43,11 +74,21 @@ export async function resolveSegment( main_face_image: request.prompts.image.identity, } } + + /* + console.log("debug:", { + model: request.settings.imageGenerationWorkflow.data, + params, + }) + */ const response = (await replicate.run( - request.settings.imageGenerationWorkflow as any, + request.settings.imageGenerationWorkflow.data as any, { input: params } )) as any - segment.assetUrl = `${response.output || ''}` + + + segment.assetUrl = `${response[0] || ''}` + } else if (request.segment.category === ClapSegmentCategory.DIALOGUE) { const response = (await replicate.run( request.settings.voiceGenerationWorkflow.data as any, @@ -55,20 +96,22 @@ export async function resolveSegment( input: { text: request.prompts.voice.positive, audio: request.prompts.voice.identity, + disable_safety_checker: !request.settings.censorNotForAllAudiencesContent, }, } )) as any - segment.assetUrl = `${response.output || ''}` + segment.assetUrl = `${response[0] || ''}` } else if (request.segment.category === ClapSegmentCategory.VIDEO) { const response = (await replicate.run( request.settings.videoGenerationWorkflow.data as any, { input: { image: request.prompts.video.image, + disable_safety_checker: !request.settings.censorNotForAllAudiencesContent, }, } )) as any - segment.assetUrl = `${response.output || ''}` + segment.assetUrl = `${response[0] || ''}` } else { throw new Error( `Clapper doesn't support ${request.segment.category} generation for provider "Replicate". Please open a pull request with (working code) to solve this!` diff --git a/packages/app/src/app/api/resolve/route.ts b/packages/app/src/app/api/resolve/route.ts index 24ec2b15..7983abed 100644 --- a/packages/app/src/app/api/resolve/route.ts +++ b/packages/app/src/app/api/resolve/route.ts @@ -132,6 +132,9 @@ export async function POST(req: NextRequest) { segment.outputType === ClapOutputType.AUDIO || segment.outputType === ClapOutputType.VIDEO ) { + + + // TODO this should be down in the browser side, so that we can scale better const { durationInMs, hasAudio } = await getMediaInfo(segment.assetUrl) segment.assetDurationInMs = durationInMs diff --git a/packages/app/src/components/toolbars/top-menu/lists/AssistantWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/AssistantWorkflows.tsx index d34915cc..72b72075 100644 --- a/packages/app/src/components/toolbars/top-menu/lists/AssistantWorkflows.tsx +++ b/packages/app/src/components/toolbars/top-menu/lists/AssistantWorkflows.tsx @@ -21,18 +21,20 @@ import { ClapWorkflowProviderLogo, ClapWorkflowProviderName, } from '@/components/core/providers' +import { parseWorkflow } from '@/services/settings/workflows/parseWorkflow' + +const category = ClapWorkflowCategory.ASSISTANT export function AssistantWorkflows() { - const workflowId = useSettings((s) => s.assistantWorkflow) - const setWorkflowId = useSettings((s) => s.setAssistantWorkflow) + const assistantWorkflow = useSettings((s) => s.assistantWorkflow) + const setAssistantWorkflow = useSettings((s) => s.setAssistantWorkflow) const availableWorkflows = useWorkflowEditor((s) => s.availableWorkflows) - const { workflows, providers, nbProviders } = findWorkflows( - availableWorkflows, - { category: ClapWorkflowCategory.ASSISTANT } - ) + const { providers, nbProviders } = findWorkflows(availableWorkflows, { + category, + }) - const { workflow } = findWorkflows(workflows, { workflowId }) + const workflow = parseWorkflow(assistantWorkflow, category) if (!nbProviders) { return null @@ -65,7 +67,7 @@ export function AssistantWorkflows() { {workflows?.map((w) => ( { if (hasNoPublicAPI(w)) { @@ -73,7 +75,7 @@ export function AssistantWorkflows() { e.preventDefault() return false } - setWorkflowId(w.id) + setAssistantWorkflow(w) e.stopPropagation() e.preventDefault() return false diff --git a/packages/app/src/components/toolbars/top-menu/lists/ImageDepthWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/ImageDepthWorkflows.tsx index 44948c78..7dde58fc 100644 --- a/packages/app/src/components/toolbars/top-menu/lists/ImageDepthWorkflows.tsx +++ b/packages/app/src/components/toolbars/top-menu/lists/ImageDepthWorkflows.tsx @@ -21,18 +21,20 @@ import { ClapWorkflowProviderLogo, ClapWorkflowProviderName, } from '@/components/core/providers' +import { parseWorkflow } from '@/services/settings/workflows/parseWorkflow' + +const category = ClapWorkflowCategory.IMAGE_DEPTH_MAPPING export function ImageDepthWorkflows() { - const workflowId = useSettings((s) => s.imageDepthWorkflow) - const setWorkflowId = useSettings((s) => s.setImageDepthWorkflow) + const imageDepthWorkflow = useSettings((s) => s.imageDepthWorkflow) + const setImageDepthWorkflow = useSettings((s) => s.setImageDepthWorkflow) const availableWorkflows = useWorkflowEditor((s) => s.availableWorkflows) - const { workflows, providers, nbProviders } = findWorkflows( - availableWorkflows, - { category: ClapWorkflowCategory.IMAGE_DEPTH_MAPPING } - ) + const { providers, nbProviders } = findWorkflows(availableWorkflows, { + category, + }) - const { workflow } = findWorkflows(workflows, { workflowId }) + const workflow = parseWorkflow(imageDepthWorkflow, category) if (!nbProviders) { return null @@ -65,7 +67,7 @@ export function ImageDepthWorkflows() { {workflows?.map((w) => ( { if (hasNoPublicAPI(w)) { @@ -73,7 +75,7 @@ export function ImageDepthWorkflows() { e.preventDefault() return false } - setWorkflowId(w.id) + setImageDepthWorkflow(w) e.stopPropagation() e.preventDefault() return false diff --git a/packages/app/src/components/toolbars/top-menu/lists/ImageGenerationWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/ImageGenerationWorkflows.tsx index e851e87d..005cf11f 100644 --- a/packages/app/src/components/toolbars/top-menu/lists/ImageGenerationWorkflows.tsx +++ b/packages/app/src/components/toolbars/top-menu/lists/ImageGenerationWorkflows.tsx @@ -1,6 +1,10 @@ 'use client' -import { ClapWorkflowCategory, ClapWorkflowProvider } from '@aitube/clap' +import { + ClapInputCategory, + ClapWorkflowCategory, + ClapWorkflowProvider, +} from '@aitube/clap' import { MenubarCheckboxItem, @@ -21,23 +25,33 @@ import { ClapWorkflowProviderLogo, ClapWorkflowProviderName, } from '@/components/core/providers' +import { parseWorkflow } from '@/services/settings/workflows/parseWorkflow' +import { Lora } from '@/services/editors/workflow-editor/workflows/common/types' +import { getWorkflowLora } from '@/services/editors/workflow-editor/workflows/common/loras/getWorkflowLora' +import { getWorkflowInputField } from '@/services/editors/workflow-editor/workflows/common/loras/getWorkflowInputField' -export function ImageGenerationWorkflows() { - const workflowId = useSettings((s) => s.imageGenerationWorkflow) - const setWorkflowId = useSettings((s) => s.setImageGenerationWorkflow) - const availableWorkflows = useWorkflowEditor((s) => s.availableWorkflows) +import { LoraModelList } from './LoraModelList' - const { workflows, providers, nbProviders } = findWorkflows( - availableWorkflows, - { category: ClapWorkflowCategory.IMAGE_GENERATION } +const category = ClapWorkflowCategory.IMAGE_GENERATION + +export function ImageGenerationWorkflows() { + const imageGenerationWorkflow = useSettings((s) => s.imageGenerationWorkflow) + const setImageGenerationWorkflow = useSettings( + (s) => s.setImageGenerationWorkflow ) + const availableWorkflows = useWorkflowEditor((s) => s.availableWorkflows) - const { workflow } = findWorkflows(workflows, { workflowId }) + const { providers, nbProviders } = findWorkflows(availableWorkflows, { + category, + }) if (!nbProviders) { return null } + const workflow = parseWorkflow(imageGenerationWorkflow, category) + const workflowLora = getWorkflowLora(workflow) + return ( @@ -62,26 +76,66 @@ export function ImageGenerationWorkflows() { - {workflows?.map((w) => ( - { - if (hasNoPublicAPI(w)) { + {workflows?.map((w) => { + // if this workflow has at least one field of type lora + const loraFieldName = getWorkflowInputField( + w, + ClapInputCategory.LORA + )?.id + if (loraFieldName) { + return ( + { + + console.log(`onChange:`, { + w, + newLora, + loraFieldName, + repoUrl: newLora?.repoOrUrl, + newWorkflowValue: { + ...w, + inputValues: { + ...w.inputValues, + [loraFieldName]: newLora?.repoOrUrl || '', + }, + }, + }) + setImageGenerationWorkflow({ + ...w, + inputValues: { + ...w.inputValues, + [loraFieldName]: newLora?.repoOrUrl || '', + }, + }) + }} + /> + ) + } + + return ( + { + if (hasNoPublicAPI(w)) { + e.stopPropagation() + e.preventDefault() + return false + } + setImageGenerationWorkflow(w) e.stopPropagation() e.preventDefault() return false - } - setWorkflowId(w.id) - e.stopPropagation() - e.preventDefault() - return false - }} - > - {w.label} - - ))} + }} + > + {w.label} + + ) + })} ))} diff --git a/packages/app/src/components/toolbars/top-menu/lists/ImageSegmentationWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/ImageSegmentationWorkflows.tsx index 6e217d26..f3d794c0 100644 --- a/packages/app/src/components/toolbars/top-menu/lists/ImageSegmentationWorkflows.tsx +++ b/packages/app/src/components/toolbars/top-menu/lists/ImageSegmentationWorkflows.tsx @@ -21,18 +21,24 @@ import { ClapWorkflowProviderLogo, ClapWorkflowProviderName, } from '@/components/core/providers' +import { parseWorkflow } from '@/services/settings/workflows/parseWorkflow' + +const category = ClapWorkflowCategory.IMAGE_SEGMENTATION export function ImageSegmentationWorkflows() { - const workflowId = useSettings((s) => s.imageSegmentationWorkflow) - const setWorkflowId = useSettings((s) => s.setImageSegmentationWorkflow) + const imageSegmentationWorkflow = useSettings( + (s) => s.imageSegmentationWorkflow + ) + const setImageSegmentationWorkflow = useSettings( + (s) => s.setImageSegmentationWorkflow + ) const availableWorkflows = useWorkflowEditor((s) => s.availableWorkflows) - const { workflows, providers, nbProviders } = findWorkflows( - availableWorkflows, - { category: ClapWorkflowCategory.IMAGE_SEGMENTATION } - ) + const { providers, nbProviders } = findWorkflows(availableWorkflows, { + category, + }) - const { workflow } = findWorkflows(workflows, { workflowId }) + const workflow = parseWorkflow(imageSegmentationWorkflow, category) if (!nbProviders) { return null @@ -65,7 +71,7 @@ export function ImageSegmentationWorkflows() { {workflows?.map((w) => ( { if (hasNoPublicAPI(w)) { @@ -73,7 +79,7 @@ export function ImageSegmentationWorkflows() { e.preventDefault() return false } - setWorkflowId(w.id) + setImageSegmentationWorkflow(w) e.stopPropagation() e.preventDefault() return false diff --git a/packages/app/src/components/toolbars/top-menu/lists/ImageUpscalingWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/ImageUpscalingWorkflows.tsx index fc8e46df..5dc329c7 100644 --- a/packages/app/src/components/toolbars/top-menu/lists/ImageUpscalingWorkflows.tsx +++ b/packages/app/src/components/toolbars/top-menu/lists/ImageUpscalingWorkflows.tsx @@ -21,18 +21,22 @@ import { ClapWorkflowProviderLogo, ClapWorkflowProviderName, } from '@/components/core/providers' +import { parseWorkflow } from '@/services/settings/workflows/parseWorkflow' + +const category = ClapWorkflowCategory.IMAGE_UPSCALING export function ImageUpscalingWorkflows() { - const workflowId = useSettings((s) => s.imageUpscalingWorkflow) - const setWorkflowId = useSettings((s) => s.setImageUpscalingWorkflow) + const imageUpscalingWorkflow = useSettings((s) => s.imageUpscalingWorkflow) + const setImageUpscalingWorkflow = useSettings( + (s) => s.setImageUpscalingWorkflow + ) const availableWorkflows = useWorkflowEditor((s) => s.availableWorkflows) - const { workflows, providers, nbProviders } = findWorkflows( - availableWorkflows, - { category: ClapWorkflowCategory.IMAGE_UPSCALING } - ) + const { providers, nbProviders } = findWorkflows(availableWorkflows, { + category, + }) - const { workflow } = findWorkflows(workflows, { workflowId }) + const workflow = parseWorkflow(imageUpscalingWorkflow, category) if (!nbProviders) { return null @@ -65,7 +69,7 @@ export function ImageUpscalingWorkflows() { {workflows?.map((w) => ( { if (hasNoPublicAPI(w)) { @@ -73,7 +77,7 @@ export function ImageUpscalingWorkflows() { e.preventDefault() return false } - setWorkflowId(w.id) + setImageUpscalingWorkflow(w) e.stopPropagation() e.preventDefault() return false diff --git a/packages/app/src/components/toolbars/top-menu/lists/LoraModelList.tsx b/packages/app/src/components/toolbars/top-menu/lists/LoraModelList.tsx new file mode 100644 index 00000000..93b1ad03 --- /dev/null +++ b/packages/app/src/components/toolbars/top-menu/lists/LoraModelList.tsx @@ -0,0 +1,66 @@ +'use client' + +import { ClapWorkflow } from '@aitube/clap' + +import { + MenubarCheckboxItem, + MenubarContent, + MenubarItem, + MenubarMenu, + MenubarSeparator, + MenubarSub, + MenubarSubContent, + MenubarSubTrigger, +} from '@/components/ui/menubar' + +import { defaultLoraModels } from '@/services/editors/workflow-editor/workflows/common/loras' +import { Lora } from '@/services/editors/workflow-editor/workflows/common/types' + +export function LoraModelList({ + workflow, + currentLora, + onChange, +}: { + workflow: ClapWorkflow + currentLora?: Lora + onChange: (lora?: Lora) => void +}) { + return ( + + +
+ {workflow.label} + {currentLora ? `(${currentLora.label})` : `(no lora selected)`} +
+ + + { + onChange(undefined) + e.stopPropagation() + e.preventDefault() + return false + }} + > + No LoRA + + {defaultLoraModels.map((lora: Lora) => ( + { + onChange(lora) + e.stopPropagation() + e.preventDefault() + return false + }} + > + {lora.label} + + ))} + + + ) +} diff --git a/packages/app/src/components/toolbars/top-menu/lists/MusicGenerationWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/MusicGenerationWorkflows.tsx index 26afac98..85b4806b 100644 --- a/packages/app/src/components/toolbars/top-menu/lists/MusicGenerationWorkflows.tsx +++ b/packages/app/src/components/toolbars/top-menu/lists/MusicGenerationWorkflows.tsx @@ -21,18 +21,22 @@ import { ClapWorkflowProviderLogo, ClapWorkflowProviderName, } from '@/components/core/providers' +import { parseWorkflow } from '@/services/settings/workflows/parseWorkflow' + +const category = ClapWorkflowCategory.MUSIC_GENERATION export function MusicGenerationWorkflows() { - const workflowId = useSettings((s) => s.musicGenerationWorkflow) - const setWorkflowId = useSettings((s) => s.setMusicGenerationWorkflow) + const musicGenerationWorkflow = useSettings((s) => s.musicGenerationWorkflow) + const setMusicGenerationWorkflow = useSettings( + (s) => s.setMusicGenerationWorkflow + ) const availableWorkflows = useWorkflowEditor((s) => s.availableWorkflows) - const { workflows, providers, nbProviders } = findWorkflows( - availableWorkflows, - { category: ClapWorkflowCategory.MUSIC_GENERATION } - ) + const { providers, nbProviders } = findWorkflows(availableWorkflows, { + category, + }) - const { workflow } = findWorkflows(workflows, { workflowId }) + const workflow = parseWorkflow(musicGenerationWorkflow, category) if (!nbProviders) { return null @@ -65,7 +69,7 @@ export function MusicGenerationWorkflows() { {workflows?.map((w) => ( { if (hasNoPublicAPI(w)) { @@ -73,7 +77,7 @@ export function MusicGenerationWorkflows() { e.preventDefault() return false } - setWorkflowId(w.id) + setMusicGenerationWorkflow(w) e.stopPropagation() e.preventDefault() return false diff --git a/packages/app/src/components/toolbars/top-menu/lists/SoundGenerationWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/SoundGenerationWorkflows.tsx index 78c5edc4..ad22d9bd 100644 --- a/packages/app/src/components/toolbars/top-menu/lists/SoundGenerationWorkflows.tsx +++ b/packages/app/src/components/toolbars/top-menu/lists/SoundGenerationWorkflows.tsx @@ -21,18 +21,22 @@ import { ClapWorkflowProviderLogo, ClapWorkflowProviderName, } from '@/components/core/providers' +import { parseWorkflow } from '@/services/settings/workflows/parseWorkflow' + +const category = ClapWorkflowCategory.SOUND_GENERATION export function SoundGenerationWorkflows() { - const workflowId = useSettings((s) => s.soundGenerationWorkflow) - const setWorkflowId = useSettings((s) => s.setSoundGenerationWorkflow) + const soundGenerationWorkflow = useSettings((s) => s.soundGenerationWorkflow) + const setSoundGenerationWorkflow = useSettings( + (s) => s.setSoundGenerationWorkflow + ) const availableWorkflows = useWorkflowEditor((s) => s.availableWorkflows) - const { workflows, providers, nbProviders } = findWorkflows( - availableWorkflows, - { category: ClapWorkflowCategory.SOUND_GENERATION } - ) + const { providers, nbProviders } = findWorkflows(availableWorkflows, { + category, + }) - const { workflow } = findWorkflows(workflows, { workflowId }) + const workflow = parseWorkflow(soundGenerationWorkflow, category) if (!nbProviders) { return null @@ -65,7 +69,7 @@ export function SoundGenerationWorkflows() { {workflows?.map((w) => ( { if (hasNoPublicAPI(w)) { @@ -73,7 +77,7 @@ export function SoundGenerationWorkflows() { e.preventDefault() return false } - setWorkflowId(w.id) + setSoundGenerationWorkflow(w) e.stopPropagation() e.preventDefault() return false diff --git a/packages/app/src/components/toolbars/top-menu/lists/VideoDepthWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/VideoDepthWorkflows.tsx index e81a9c0a..db00363c 100644 --- a/packages/app/src/components/toolbars/top-menu/lists/VideoDepthWorkflows.tsx +++ b/packages/app/src/components/toolbars/top-menu/lists/VideoDepthWorkflows.tsx @@ -21,18 +21,20 @@ import { ClapWorkflowProviderLogo, ClapWorkflowProviderName, } from '@/components/core/providers' +import { parseWorkflow } from '@/services/settings/workflows/parseWorkflow' + +const category = ClapWorkflowCategory.VIDEO_DEPTH_MAPPING export function VideoDepthWorkflows() { - const workflowId = useSettings((s) => s.videoDepthWorkflow) - const setWorkflowId = useSettings((s) => s.setVideoDepthWorkflow) + const videoDepthWorkflow = useSettings((s) => s.videoDepthWorkflow) + const setVideoDepthWorkflow = useSettings((s) => s.setVideoDepthWorkflow) const availableWorkflows = useWorkflowEditor((s) => s.availableWorkflows) - const { workflows, providers, nbProviders } = findWorkflows( - availableWorkflows, - { category: ClapWorkflowCategory.VIDEO_DEPTH_MAPPING } - ) + const { providers, nbProviders } = findWorkflows(availableWorkflows, { + category, + }) - const { workflow } = findWorkflows(workflows, { workflowId }) + const workflow = parseWorkflow(videoDepthWorkflow, category) if (!nbProviders) { return null @@ -65,7 +67,7 @@ export function VideoDepthWorkflows() { {workflows?.map((w) => ( { if (hasNoPublicAPI(w)) { @@ -73,7 +75,7 @@ export function VideoDepthWorkflows() { e.preventDefault() return false } - setWorkflowId(w.id) + setVideoDepthWorkflow(w) e.stopPropagation() e.preventDefault() return false diff --git a/packages/app/src/components/toolbars/top-menu/lists/VideoGenerationWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/VideoGenerationWorkflows.tsx index 16b81af4..94e90b06 100644 --- a/packages/app/src/components/toolbars/top-menu/lists/VideoGenerationWorkflows.tsx +++ b/packages/app/src/components/toolbars/top-menu/lists/VideoGenerationWorkflows.tsx @@ -21,18 +21,22 @@ import { ClapWorkflowProviderLogo, ClapWorkflowProviderName, } from '@/components/core/providers' +import { parseWorkflow } from '@/services/settings/workflows/parseWorkflow' + +const category = ClapWorkflowCategory.VIDEO_GENERATION export function VideoGenerationWorkflows() { - const workflowId = useSettings((s) => s.videoGenerationWorkflow) - const setWorkflowId = useSettings((s) => s.setVideoGenerationWorkflow) + const videoGenerationWorkflow = useSettings((s) => s.videoGenerationWorkflow) + const setVideoGenerationWorkflow = useSettings( + (s) => s.setVideoGenerationWorkflow + ) const availableWorkflows = useWorkflowEditor((s) => s.availableWorkflows) - const { workflows, providers, nbProviders } = findWorkflows( - availableWorkflows, - { category: ClapWorkflowCategory.VIDEO_GENERATION } - ) + const { providers, nbProviders } = findWorkflows(availableWorkflows, { + category, + }) - const { workflow } = findWorkflows(workflows, { workflowId }) + const workflow = parseWorkflow(videoGenerationWorkflow, category) if (!nbProviders) { return null @@ -65,7 +69,7 @@ export function VideoGenerationWorkflows() { {workflows?.map((w) => ( { if (hasNoPublicAPI(w)) { @@ -73,7 +77,7 @@ export function VideoGenerationWorkflows() { e.preventDefault() return false } - setWorkflowId(w.id) + setVideoGenerationWorkflow(w) e.stopPropagation() e.preventDefault() return false diff --git a/packages/app/src/components/toolbars/top-menu/lists/VideoSegmentationWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/VideoSegmentationWorkflows.tsx index e0eab94e..8415219a 100644 --- a/packages/app/src/components/toolbars/top-menu/lists/VideoSegmentationWorkflows.tsx +++ b/packages/app/src/components/toolbars/top-menu/lists/VideoSegmentationWorkflows.tsx @@ -21,18 +21,24 @@ import { ClapWorkflowProviderLogo, ClapWorkflowProviderName, } from '@/components/core/providers' +import { parseWorkflow } from '@/services/settings/workflows/parseWorkflow' + +const category = ClapWorkflowCategory.VIDEO_SEGMENTATION export function VideoSegmentationWorkflows() { - const workflowId = useSettings((s) => s.videoSegmentationWorkflow) - const setWorkflowId = useSettings((s) => s.setVideoSegmentationWorkflow) + const videoSegmentationWorkflow = useSettings( + (s) => s.videoSegmentationWorkflow + ) + const setVideoSegmentationWorkflow = useSettings( + (s) => s.setVideoSegmentationWorkflow + ) const availableWorkflows = useWorkflowEditor((s) => s.availableWorkflows) - const { workflows, providers, nbProviders } = findWorkflows( - availableWorkflows, - { category: ClapWorkflowCategory.VIDEO_SEGMENTATION } - ) + const { providers, nbProviders } = findWorkflows(availableWorkflows, { + category: ClapWorkflowCategory.VIDEO_SEGMENTATION, + }) - const { workflow } = findWorkflows(workflows, { workflowId }) + const workflow = parseWorkflow(videoSegmentationWorkflow, category) if (!nbProviders) { return null @@ -65,7 +71,7 @@ export function VideoSegmentationWorkflows() { {workflows?.map((w) => ( { if (hasNoPublicAPI(w)) { @@ -73,7 +79,7 @@ export function VideoSegmentationWorkflows() { e.preventDefault() return false } - setWorkflowId(w.id) + setVideoSegmentationWorkflow(w) e.stopPropagation() e.preventDefault() return false diff --git a/packages/app/src/components/toolbars/top-menu/lists/VideoUpscalingWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/VideoUpscalingWorkflows.tsx index ebc055df..c47023e5 100644 --- a/packages/app/src/components/toolbars/top-menu/lists/VideoUpscalingWorkflows.tsx +++ b/packages/app/src/components/toolbars/top-menu/lists/VideoUpscalingWorkflows.tsx @@ -21,18 +21,22 @@ import { ClapWorkflowProviderLogo, ClapWorkflowProviderName, } from '@/components/core/providers' +import { parseWorkflow } from '@/services/settings/workflows/parseWorkflow' + +const category = ClapWorkflowCategory.VIDEO_UPSCALING export function VideoUpscalingWorkflows() { - const workflowId = useSettings((s) => s.videoUpscalingWorkflow) - const setWorkflowId = useSettings((s) => s.setVideoUpscalingWorkflow) + const videoUpscalingWorkflow = useSettings((s) => s.videoUpscalingWorkflow) + const setVideoUpscalingWorkflow = useSettings( + (s) => s.setVideoUpscalingWorkflow + ) const availableWorkflows = useWorkflowEditor((s) => s.availableWorkflows) - const { workflows, providers, nbProviders } = findWorkflows( - availableWorkflows, - { category: ClapWorkflowCategory.VIDEO_UPSCALING } - ) + const { providers, nbProviders } = findWorkflows(availableWorkflows, { + category, + }) - const { workflow } = findWorkflows(workflows, { workflowId }) + const workflow = parseWorkflow(videoUpscalingWorkflow, category) if (!nbProviders) { return null @@ -65,7 +69,7 @@ export function VideoUpscalingWorkflows() { {workflows?.map((w) => ( { if (hasNoPublicAPI(w)) { @@ -73,7 +77,7 @@ export function VideoUpscalingWorkflows() { e.preventDefault() return false } - setWorkflowId(w.id) + setVideoUpscalingWorkflow(w) e.stopPropagation() e.preventDefault() return false diff --git a/packages/app/src/components/toolbars/top-menu/lists/VoiceGenerationWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/VoiceGenerationWorkflows.tsx index 5e03d992..9085e480 100644 --- a/packages/app/src/components/toolbars/top-menu/lists/VoiceGenerationWorkflows.tsx +++ b/packages/app/src/components/toolbars/top-menu/lists/VoiceGenerationWorkflows.tsx @@ -21,18 +21,22 @@ import { ClapWorkflowProviderLogo, ClapWorkflowProviderName, } from '@/components/core/providers' +import { parseWorkflow } from '@/services/settings/workflows/parseWorkflow' + +const category = ClapWorkflowCategory.VOICE_GENERATION export function VoiceGenerationWorkflows() { - const workflowId = useSettings((s) => s.voiceGenerationWorkflow) - const setWorkflowId = useSettings((s) => s.setVoiceGenerationWorkflow) + const voiceGenerationWorkflow = useSettings((s) => s.voiceGenerationWorkflow) + const setVoiceGenerationWorkflow = useSettings( + (s) => s.setVoiceGenerationWorkflow + ) const availableWorkflows = useWorkflowEditor((s) => s.availableWorkflows) - const { workflows, providers, nbProviders } = findWorkflows( - availableWorkflows, - { category: ClapWorkflowCategory.VOICE_GENERATION } - ) + const { providers, nbProviders } = findWorkflows(availableWorkflows, { + category, + }) - const { workflow } = findWorkflows(workflows, { workflowId }) + const workflow = parseWorkflow(voiceGenerationWorkflow, category) if (!nbProviders) { return null @@ -65,7 +69,7 @@ export function VoiceGenerationWorkflows() { {workflows?.map((w) => ( { if (hasNoPublicAPI(w)) { @@ -73,7 +77,7 @@ export function VoiceGenerationWorkflows() { e.preventDefault() return false } - setWorkflowId(w.id) + setVoiceGenerationWorkflow(w) e.stopPropagation() e.preventDefault() return false diff --git a/packages/app/src/components/toolbars/top-menu/lists/getWorkflowProviders.ts b/packages/app/src/components/toolbars/top-menu/lists/getWorkflowProviders.ts index 6c0d896c..9894310b 100644 --- a/packages/app/src/components/toolbars/top-menu/lists/getWorkflowProviders.ts +++ b/packages/app/src/components/toolbars/top-menu/lists/getWorkflowProviders.ts @@ -5,6 +5,15 @@ import { ClapWorkflowProvider, } from '@aitube/clap' +export type WorkflowSearchResults = { + workflow?: ClapWorkflow + workflows: ClapWorkflow[] + nbWorkflows: number + providers: Partial> + nbProviders: number + workflowIds: Record +} + /** * Helper to find workflows by id, category, provider or engine * @@ -21,14 +30,7 @@ export function findWorkflows( provider?: ClapWorkflowProvider engine?: ClapWorkflowEngine } -): { - workflow?: ClapWorkflow - workflows: ClapWorkflow[] - nbWorkflows: number - providers: Partial> - nbProviders: number - workflowIds: Record -} { +): WorkflowSearchResults { const workflows: ClapWorkflow[] = [] const providers: Partial> = {} const workflowIds: Record = {} diff --git a/packages/app/src/lib/utils/decodeOutput.ts b/packages/app/src/lib/utils/decodeOutput.ts index 346bbf57..ae078d6e 100644 --- a/packages/app/src/lib/utils/decodeOutput.ts +++ b/packages/app/src/lib/utils/decodeOutput.ts @@ -14,6 +14,7 @@ export async function decodeOutput(input: any): Promise { ? urlOrBase64 : await fetchContentToBase64(urlOrBase64) + if (base64Url.startsWith('data:image/')) { if ( base64Url.startsWith('data:image/jpeg') || diff --git a/packages/app/src/lib/utils/fetchContentToBase64.ts b/packages/app/src/lib/utils/fetchContentToBase64.ts index 1bf93e71..8e687a1a 100644 --- a/packages/app/src/lib/utils/fetchContentToBase64.ts +++ b/packages/app/src/lib/utils/fetchContentToBase64.ts @@ -1,4 +1,7 @@ export async function fetchContentToBase64(url: string) { + + const predictedFormat = url.split(".").pop()?.trim().toLowerCase() + const res = await fetch(url, { method: 'GET', headers: { @@ -11,5 +14,19 @@ export async function fetchContentToBase64(url: string) { const blob = await res.blob() const buffer = Buffer.from(await blob.arrayBuffer()) - return 'data:' + blob.type + ';base64,' + buffer.toString('base64') + // some providers such as Replicate return a generic octet-stream type in the headers + const type = blob.type === "application/octet-stream" + ? (predictedFormat === "webp" ? "image/webp" : + predictedFormat === "jpeg" ? "image/jpeg" : + predictedFormat === "jpg" ? "image/jpeg" : + predictedFormat === "png" ? "image/png" : + predictedFormat === "avif" ? "image/avif" : + predictedFormat === "heic" ? "image/heic" : + predictedFormat === "mp4" ? "video/mp4" : + predictedFormat === "mp3" ? "audio/mp3" : + predictedFormat === "wav" ? "audio/wav" : + "application/octet-stream" + ) : blob.type + + return 'data:' + type + ';base64,' + buffer.toString('base64') } diff --git a/packages/app/src/services/editors/filter-editor/filters/analogLens.ts b/packages/app/src/services/editors/filter-editor/filters/analogLens.ts index 9221752b..c314f1be 100644 --- a/packages/app/src/services/editors/filter-editor/filters/analogLens.ts +++ b/packages/app/src/services/editors/filter-editor/filters/analogLens.ts @@ -1,3 +1,4 @@ +import { ClapInputCategory } from '@aitube/clap' import { Filter } from '@aitube/clapper-services' export const analogLensSimulator: Filter = { @@ -8,6 +9,7 @@ export const analogLensSimulator: Filter = { id: 'chromaticAberration', label: 'Chromatic aberration', description: 'Chromatic aberration strength', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 0.05, @@ -17,6 +19,7 @@ export const analogLensSimulator: Filter = { id: 'vignetteStrength', label: 'Vignette strength', description: 'Vignette strength', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 1, @@ -26,6 +29,7 @@ export const analogLensSimulator: Filter = { id: 'vignetteRadius', label: 'Vignette radius', description: 'Vignette radius', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 1, @@ -35,6 +39,7 @@ export const analogLensSimulator: Filter = { id: 'distortion', label: 'Distortion', description: 'Lens distortion', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: -1, maxValue: 1, @@ -44,6 +49,7 @@ export const analogLensSimulator: Filter = { id: 'bloomStrength', label: 'Bloom strength', description: 'Bloom strength', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 1, @@ -53,6 +59,7 @@ export const analogLensSimulator: Filter = { id: 'bloomRadius', label: 'Bloom radius', description: 'Bloom radius', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 1, maxValue: 10, @@ -62,6 +69,7 @@ export const analogLensSimulator: Filter = { id: 'dofFocusDistance', label: 'DOF focus distance', description: 'Depth of field focus distance', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 1, @@ -71,6 +79,7 @@ export const analogLensSimulator: Filter = { id: 'dofFocusRange', label: 'DOF focus range', description: 'Depth of field focus range', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0.01, maxValue: 1, @@ -80,6 +89,7 @@ export const analogLensSimulator: Filter = { id: 'dofBlurStrength', label: 'DOF blur strength', description: 'Depth of field blur strength', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 1, diff --git a/packages/app/src/services/editors/filter-editor/filters/cinematic.ts b/packages/app/src/services/editors/filter-editor/filters/cinematic.ts index debfaa68..ecb8f867 100644 --- a/packages/app/src/services/editors/filter-editor/filters/cinematic.ts +++ b/packages/app/src/services/editors/filter-editor/filters/cinematic.ts @@ -1,3 +1,4 @@ +import { ClapInputCategory } from '@aitube/clap' import { Filter } from '@aitube/clapper-services' export const cinematic: Filter = { @@ -8,6 +9,7 @@ export const cinematic: Filter = { id: 'preset', label: 'Preset', description: 'Cinematic color preset', + category: ClapInputCategory.UNKNOWN, type: 'string', allowedValues: [ 'Blade Runner', @@ -22,6 +24,7 @@ export const cinematic: Filter = { id: 'intensity', label: 'Intensity', description: 'Intensity of the cinematic effect', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 1, @@ -31,6 +34,7 @@ export const cinematic: Filter = { id: 'contrast', label: 'Contrast', description: 'Image contrast', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0.5, maxValue: 2, @@ -40,6 +44,7 @@ export const cinematic: Filter = { id: 'grain', label: 'Film Grain', description: 'Intensity of film grain effect', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 1, @@ -49,6 +54,7 @@ export const cinematic: Filter = { id: 'blur', label: 'Blur', description: 'Slight blur effect', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 1, diff --git a/packages/app/src/services/editors/filter-editor/filters/colorMapping.ts b/packages/app/src/services/editors/filter-editor/filters/colorMapping.ts index 1b9333e6..e8d7b981 100644 --- a/packages/app/src/services/editors/filter-editor/filters/colorMapping.ts +++ b/packages/app/src/services/editors/filter-editor/filters/colorMapping.ts @@ -1,3 +1,4 @@ +import { ClapInputCategory } from '@aitube/clap' import { Filter } from '@aitube/clapper-services' export const colorMapping: Filter = { @@ -8,6 +9,7 @@ export const colorMapping: Filter = { id: 'redMultiplier', label: 'Red multiplier', description: 'Red channel multiplier', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 2, @@ -17,6 +19,7 @@ export const colorMapping: Filter = { id: 'greenMultiplier', label: 'Green multiplier', description: 'Green channel multiplier', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 2, @@ -26,6 +29,7 @@ export const colorMapping: Filter = { id: 'blueMultiplier', label: 'Blue multiplier', description: 'Blue channel multiplier', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 2, diff --git a/packages/app/src/services/editors/filter-editor/filters/colorTemperature.ts b/packages/app/src/services/editors/filter-editor/filters/colorTemperature.ts index f61a027c..62824328 100644 --- a/packages/app/src/services/editors/filter-editor/filters/colorTemperature.ts +++ b/packages/app/src/services/editors/filter-editor/filters/colorTemperature.ts @@ -1,3 +1,4 @@ +import { ClapInputCategory } from '@aitube/clap' import { Filter } from '@aitube/clapper-services' export const colorTemperature: Filter = { @@ -9,6 +10,7 @@ export const colorTemperature: Filter = { label: 'Temperature', description: 'Color temperature in Kelvin', type: 'number', + category: ClapInputCategory.UNKNOWN, minValue: 1000, maxValue: 40000, defaultValue: 6500, @@ -17,6 +19,7 @@ export const colorTemperature: Filter = { id: 'tint', label: 'Tint', description: 'Green-Magenta tint', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: -100, maxValue: 100, diff --git a/packages/app/src/services/editors/filter-editor/filters/crossProcessing.ts b/packages/app/src/services/editors/filter-editor/filters/crossProcessing.ts index 8b230a8f..3e1a5000 100644 --- a/packages/app/src/services/editors/filter-editor/filters/crossProcessing.ts +++ b/packages/app/src/services/editors/filter-editor/filters/crossProcessing.ts @@ -1,3 +1,4 @@ +import { ClapInputCategory } from '@aitube/clap' import { Filter } from '@aitube/clapper-services' export const crossProcessing: Filter = { @@ -9,6 +10,7 @@ export const crossProcessing: Filter = { label: 'Intensity', description: 'Intensity of the cross-processing effect', type: 'number', + category: ClapInputCategory.UNKNOWN, minValue: 0, maxValue: 1, defaultValue: 0.5, @@ -17,6 +19,7 @@ export const crossProcessing: Filter = { id: 'contrastBoost', label: 'Contrast boost', description: 'Amount of contrast boost', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 1, @@ -26,6 +29,7 @@ export const crossProcessing: Filter = { id: 'colorShift', label: 'Color shift', description: 'Direction of color shift', + category: ClapInputCategory.UNKNOWN, type: 'string', allowedValues: ['Cool', 'Warm'], defaultValue: 'Cool', diff --git a/packages/app/src/services/editors/filter-editor/filters/filmDegradation.ts b/packages/app/src/services/editors/filter-editor/filters/filmDegradation.ts index caf3a3c0..42cae0a9 100644 --- a/packages/app/src/services/editors/filter-editor/filters/filmDegradation.ts +++ b/packages/app/src/services/editors/filter-editor/filters/filmDegradation.ts @@ -1,3 +1,4 @@ +import { ClapInputCategory } from '@aitube/clap' import { Filter } from '@aitube/clapper-services' export const filmDegradation: Filter = { @@ -8,6 +9,7 @@ export const filmDegradation: Filter = { id: 'scratchesIntensity', label: 'Scratches intensity', description: 'Intensity of film scratches', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 1, @@ -17,6 +19,7 @@ export const filmDegradation: Filter = { id: 'dustIntensity', label: 'Dust intensity', description: 'Intensity of dust and spots', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 1, @@ -26,6 +29,7 @@ export const filmDegradation: Filter = { id: 'grainIntensity', label: 'Grain intensity', description: 'Intensity of film grain', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 1, @@ -35,6 +39,7 @@ export const filmDegradation: Filter = { id: 'colorFading', label: 'Color fading', description: 'Color fading effect', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 1, @@ -44,6 +49,7 @@ export const filmDegradation: Filter = { id: 'vignettingIntensity', label: 'Vignetting intensity', description: 'Intensity of vignetting effect', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 1, @@ -53,6 +59,7 @@ export const filmDegradation: Filter = { id: 'flickerIntensity', label: 'Flicker intensity', description: 'Intensity of light flickering', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 1, @@ -62,6 +69,7 @@ export const filmDegradation: Filter = { id: 'lightLeakIntensity', label: 'Light leak intensity', description: 'Intensity of light leaks', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 1, @@ -71,6 +79,7 @@ export const filmDegradation: Filter = { id: 'filmType', label: 'Film type', description: 'Type of film to simulate', + category: ClapInputCategory.UNKNOWN, type: 'string', allowedValues: ['color', 'blackAndWhite', 'sepia'], defaultValue: 'color', diff --git a/packages/app/src/services/editors/filter-editor/filters/infrared.ts b/packages/app/src/services/editors/filter-editor/filters/infrared.ts index e6a3db00..71356735 100644 --- a/packages/app/src/services/editors/filter-editor/filters/infrared.ts +++ b/packages/app/src/services/editors/filter-editor/filters/infrared.ts @@ -1,3 +1,4 @@ +import { ClapInputCategory } from '@aitube/clap' import { Filter } from '@aitube/clapper-services' export const infraredBlackAndWhite: Filter = { @@ -8,6 +9,7 @@ export const infraredBlackAndWhite: Filter = { id: 'contrast', label: 'Contrast', description: 'Image contrast', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0.5, maxValue: 2.0, @@ -17,6 +19,7 @@ export const infraredBlackAndWhite: Filter = { id: 'grain', label: 'Grain', description: 'Film grain intensity', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 1, @@ -26,6 +29,7 @@ export const infraredBlackAndWhite: Filter = { id: 'glow', label: 'Glow', description: 'Infrared glow effect', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 1, diff --git a/packages/app/src/services/editors/filter-editor/filters/lomography.ts b/packages/app/src/services/editors/filter-editor/filters/lomography.ts index c634b054..1a25a8f6 100644 --- a/packages/app/src/services/editors/filter-editor/filters/lomography.ts +++ b/packages/app/src/services/editors/filter-editor/filters/lomography.ts @@ -1,3 +1,4 @@ +import { ClapInputCategory } from '@aitube/clap' import { Filter } from '@aitube/clapper-services' export const lomography: Filter = { @@ -8,6 +9,7 @@ export const lomography: Filter = { id: 'saturation', label: 'Saturation', description: 'Color saturation', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 2, @@ -17,6 +19,7 @@ export const lomography: Filter = { id: 'contrast', label: 'Contrast', description: 'Image contrast', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0.5, maxValue: 2, @@ -26,6 +29,7 @@ export const lomography: Filter = { id: 'vignetteIntensity', label: 'Vignette intensity', description: 'Intensity of vignette effect', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 1, @@ -35,6 +39,7 @@ export const lomography: Filter = { id: 'lightLeakIntensity', label: 'Light leak intensity', description: 'Intensity of light leak effect', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 1, diff --git a/packages/app/src/services/editors/filter-editor/filters/splitToning.ts b/packages/app/src/services/editors/filter-editor/filters/splitToning.ts index d85dda84..b29887bb 100644 --- a/packages/app/src/services/editors/filter-editor/filters/splitToning.ts +++ b/packages/app/src/services/editors/filter-editor/filters/splitToning.ts @@ -1,3 +1,4 @@ +import { ClapInputCategory } from '@aitube/clap' import { Filter } from '@aitube/clapper-services' export const splitToning: Filter = { @@ -8,6 +9,7 @@ export const splitToning: Filter = { id: 'highlightColor', label: 'Highlight color', description: 'Color for highlights', + category: ClapInputCategory.UNKNOWN, type: 'string', allowedValues: ['Red', 'Green', 'Blue', 'Yellow', 'Cyan', 'Magenta'], defaultValue: 'Yellow', @@ -16,6 +18,7 @@ export const splitToning: Filter = { id: 'shadowColor', label: 'Shadow color', description: 'Color for shadows', + category: ClapInputCategory.UNKNOWN, type: 'string', allowedValues: ['Red', 'Green', 'Blue', 'Yellow', 'Cyan', 'Magenta'], defaultValue: 'Blue', @@ -24,6 +27,7 @@ export const splitToning: Filter = { id: 'balance', label: 'Balance', description: 'Balance between highlights and shadows', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: -1, maxValue: 1, @@ -33,6 +37,7 @@ export const splitToning: Filter = { id: 'intensity', label: 'Intensity', description: 'Intensity of the split toning effect', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 1, diff --git a/packages/app/src/services/editors/filter-editor/filters/toneMapping.ts b/packages/app/src/services/editors/filter-editor/filters/toneMapping.ts index 50f0e6da..036fcb5b 100644 --- a/packages/app/src/services/editors/filter-editor/filters/toneMapping.ts +++ b/packages/app/src/services/editors/filter-editor/filters/toneMapping.ts @@ -1,3 +1,4 @@ +import { ClapInputCategory } from '@aitube/clap' import { Filter } from '@aitube/clapper-services' export const hdrToneMapping: Filter = { @@ -8,6 +9,7 @@ export const hdrToneMapping: Filter = { id: 'exposure', label: 'Exposure', description: 'Exposure adjustment', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: -2, maxValue: 2, @@ -17,6 +19,7 @@ export const hdrToneMapping: Filter = { id: 'contrast', label: 'Contrast', description: 'Contrast adjustment', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0.5, maxValue: 2, @@ -26,6 +29,7 @@ export const hdrToneMapping: Filter = { id: 'saturation', label: 'Saturation', description: 'Color saturation', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 2, @@ -35,6 +39,7 @@ export const hdrToneMapping: Filter = { id: 'highlights', label: 'Highlights', description: 'Highlight adjustment', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: -1, maxValue: 1, @@ -44,6 +49,7 @@ export const hdrToneMapping: Filter = { id: 'shadows', label: 'Shadows', description: 'Shadow adjustment', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: -1, maxValue: 1, diff --git a/packages/app/src/services/editors/filter-editor/filters/vintageFilm.ts b/packages/app/src/services/editors/filter-editor/filters/vintageFilm.ts index ed4c782a..7f5098ae 100644 --- a/packages/app/src/services/editors/filter-editor/filters/vintageFilm.ts +++ b/packages/app/src/services/editors/filter-editor/filters/vintageFilm.ts @@ -1,3 +1,4 @@ +import { ClapInputCategory } from '@aitube/clap' import { Filter } from '@aitube/clapper-services' export const vintageFilm: Filter = { @@ -8,6 +9,7 @@ export const vintageFilm: Filter = { id: 'preset', label: 'Preset', description: 'Vintage film stock preset', + category: ClapInputCategory.UNKNOWN, type: 'string', allowedValues: [ 'Kodachrome 64', @@ -47,6 +49,7 @@ export const vintageFilm: Filter = { id: 'intensity', label: 'Intensity', description: 'Intensity of the film stock effect', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 1, @@ -56,6 +59,7 @@ export const vintageFilm: Filter = { id: 'grain', label: 'Grain', description: 'Film grain intensity', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 1, @@ -65,6 +69,7 @@ export const vintageFilm: Filter = { id: 'ageEffect', label: 'Age effect', description: 'Simulated age of the film', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 1, @@ -74,6 +79,7 @@ export const vintageFilm: Filter = { id: 'colorShift', label: 'Color shift', description: 'Color shift adjustment', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: -1, maxValue: 1, @@ -83,6 +89,7 @@ export const vintageFilm: Filter = { id: 'contrast', label: 'Contrast', description: 'Contrast adjustment', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0.5, maxValue: 2, @@ -92,6 +99,7 @@ export const vintageFilm: Filter = { id: 'saturation', label: 'Saturation', description: 'Saturation adjustment', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 2, diff --git a/packages/app/src/services/editors/workflow-editor/workflows/aitube/index.ts b/packages/app/src/services/editors/workflow-editor/workflows/aitube/index.ts index 1b262784..20bbac01 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/aitube/index.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/aitube/index.ts @@ -25,6 +25,7 @@ export const aitubeWorkflows: ClapWorkflow[] = [ author: 'AiTube.at', thumbnailUrl: '', nonCommercial: false, + canSupportLora: false, engine: ClapWorkflowEngine.OPENCLAP, category: ClapWorkflowCategory.IMAGE_GENERATION, provider: ClapWorkflowProvider.AITUBE, @@ -45,6 +46,7 @@ export const aitubeWorkflows: ClapWorkflow[] = [ author: 'AiTube.at', thumbnailUrl: '', nonCommercial: false, + canSupportLora: false, engine: ClapWorkflowEngine.OPENCLAP, category: ClapWorkflowCategory.MUSIC_GENERATION, provider: ClapWorkflowProvider.AITUBE, diff --git a/packages/app/src/services/editors/workflow-editor/workflows/anthropic/index.ts b/packages/app/src/services/editors/workflow-editor/workflows/anthropic/index.ts index 997f57a8..e2f05b53 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/anthropic/index.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/anthropic/index.ts @@ -25,6 +25,7 @@ export const anthropicWorkflows: ClapWorkflow[] = [ author: 'Anthropic', thumbnailUrl: '', nonCommercial: false, + canSupportLora: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.ANTHROPIC, @@ -43,6 +44,7 @@ export const anthropicWorkflows: ClapWorkflow[] = [ author: 'Anthropic', thumbnailUrl: '', nonCommercial: false, + canSupportLora: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.ANTHROPIC, @@ -61,6 +63,7 @@ export const anthropicWorkflows: ClapWorkflow[] = [ author: 'Anthropic', thumbnailUrl: '', nonCommercial: false, + canSupportLora: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.ANTHROPIC, diff --git a/packages/app/src/services/editors/workflow-editor/workflows/bigmodel/index.ts b/packages/app/src/services/editors/workflow-editor/workflows/bigmodel/index.ts index 0c3c8b90..def1b90b 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/bigmodel/index.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/bigmodel/index.ts @@ -15,6 +15,7 @@ export const bigModelWorkflows: ClapWorkflow[] = [ author: '', thumbnailUrl: '', nonCommercial: false, + canSupportLora: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.BIGMODEL, category: ClapWorkflowCategory.VIDEO_GENERATION, diff --git a/packages/app/src/services/editors/workflow-editor/workflows/cohere/index.ts b/packages/app/src/services/editors/workflow-editor/workflows/cohere/index.ts index 5f2de36b..b3e04148 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/cohere/index.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/cohere/index.ts @@ -56,6 +56,7 @@ export const cohereWorkflows: ClapWorkflow[] = [ author: 'Cohere', thumbnailUrl: '', nonCommercial: false, + canSupportLora: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.COHERE, @@ -74,6 +75,7 @@ export const cohereWorkflows: ClapWorkflow[] = [ author: 'Cohere', thumbnailUrl: '', nonCommercial: false, + canSupportLora: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.COHERE, @@ -92,6 +94,7 @@ export const cohereWorkflows: ClapWorkflow[] = [ author: 'Cohere', thumbnailUrl: '', nonCommercial: false, + canSupportLora: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.COHERE, @@ -110,6 +113,7 @@ export const cohereWorkflows: ClapWorkflow[] = [ author: 'Cohere', thumbnailUrl: '', nonCommercial: false, + canSupportLora: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.COHERE, diff --git a/packages/app/src/services/editors/workflow-editor/workflows/comfyicu/index.ts b/packages/app/src/services/editors/workflow-editor/workflows/comfyicu/index.ts index 9d7158ca..113b8be8 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/comfyicu/index.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/comfyicu/index.ts @@ -30,6 +30,7 @@ export const comfyicuWorkflows: ClapWorkflow[] = [ author: 'BFL (https://BlackForestLabs.ai)', thumbnailUrl: '', nonCommercial: false, + canSupportLora: false, engine: ClapWorkflowEngine.COMFYUI_WORKFLOW, provider: ClapWorkflowProvider.COMFYICU, category: ClapWorkflowCategory.IMAGE_GENERATION, diff --git a/packages/app/src/services/editors/workflow-editor/workflows/comfyui/index.ts b/packages/app/src/services/editors/workflow-editor/workflows/comfyui/index.ts index 1ee4593f..de95c4ed 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/comfyui/index.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/comfyui/index.ts @@ -20,6 +20,7 @@ export const comfyuiWorkflows: ClapWorkflow[] = [ author: '', thumbnailUrl: '', nonCommercial: false, + canSupportLora: false, engine: ClapWorkflowEngine.COMFYUI_WORKFLOW, provider: ClapWorkflowProvider.COMFYUI, category: ClapWorkflowCategory.IMAGE_GENERATION, @@ -52,6 +53,7 @@ export async function getDynamicComfyuiWorkflows(): Promise { author: 'You', thumbnailUrl: '', nonCommercial: false, + canSupportLora: false, engine: ClapWorkflowEngine.COMFYUI_WORKFLOW, provider: ClapWorkflowProvider.COMFYUI, category: ClapWorkflowCategory.IMAGE_GENERATION, @@ -70,6 +72,7 @@ export async function getDynamicComfyuiWorkflows(): Promise { author: 'You', thumbnailUrl: '', nonCommercial: false, + canSupportLora: false, engine: ClapWorkflowEngine.COMFYUI_WORKFLOW, provider: ClapWorkflowProvider.COMFYUI, category: ClapWorkflowCategory.VIDEO_GENERATION, @@ -88,6 +91,7 @@ export async function getDynamicComfyuiWorkflows(): Promise { author: 'You', thumbnailUrl: '', nonCommercial: false, + canSupportLora: false, engine: ClapWorkflowEngine.COMFYUI_WORKFLOW, provider: ClapWorkflowProvider.COMFYUI, category: ClapWorkflowCategory.VOICE_GENERATION, @@ -106,6 +110,7 @@ export async function getDynamicComfyuiWorkflows(): Promise { author: 'You', thumbnailUrl: '', nonCommercial: false, + canSupportLora: false, engine: ClapWorkflowEngine.COMFYUI_WORKFLOW, provider: ClapWorkflowProvider.COMFYUI, category: ClapWorkflowCategory.MUSIC_GENERATION, @@ -124,6 +129,7 @@ export async function getDynamicComfyuiWorkflows(): Promise { author: 'You', thumbnailUrl: '', nonCommercial: false, + canSupportLora: false, engine: ClapWorkflowEngine.COMFYUI_WORKFLOW, provider: ClapWorkflowProvider.COMFYUI, category: ClapWorkflowCategory.SOUND_GENERATION, diff --git a/packages/app/src/services/editors/workflow-editor/workflows/common/defaultValues.ts b/packages/app/src/services/editors/workflow-editor/workflows/common/defaultValues.ts index b380769f..c2f6db16 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/common/defaultValues.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/common/defaultValues.ts @@ -1,4 +1,5 @@ import { ClapInputField } from '@aitube/clap' +import { ClapInputCategory } from '@aitube/clap' // IMPORTANT: do NOT modify those default fields, // otherwise you might break the workflow of someone else! @@ -9,6 +10,7 @@ export const genericInput: ClapInputField = { id: 'input', label: 'Input', description: 'Input', + category: ClapInputCategory.PROMPT, type: 'string', allowedValues: [], defaultValue: '', @@ -18,6 +20,7 @@ export const genericText: ClapInputField = { id: 'text', label: 'Text', description: 'Text', + category: ClapInputCategory.PROMPT, type: 'string', allowedValues: [], defaultValue: '', @@ -27,6 +30,7 @@ export const genericPrompt: ClapInputField = { id: 'prompt', label: 'Prompt', description: 'Prompt', + category: ClapInputCategory.PROMPT, type: 'string', allowedValues: [], defaultValue: '', @@ -36,6 +40,7 @@ export const genericRatio: ClapInputField = { id: 'ratio', label: 'Image ratio', description: 'Image ratio (default to 1:1)', + category: ClapInputCategory.UNKNOWN, type: 'string', allowedValues: ['1:1', '16:9', '9:16'], defaultValue: '1:1', @@ -45,16 +50,38 @@ export const genericSeed: ClapInputField = { id: 'seed', label: 'Seed', description: 'Seed', - type: 'number', + category: ClapInputCategory.SEED, + type: 'number', // <-- TODO: replace by 'integer' (might break stuff) minValue: 0, maxValue: Math.pow(2, 31), defaultValue: 0, } +export const genericLora: ClapInputField = { + id: 'lora', + label: 'Lora URL', + description: 'Lora URL', + category: ClapInputCategory.LORA, + type: 'string', + allowedValues: [], + defaultValue: '', +} + +export const genericLoraUrl: ClapInputField = { + id: 'lora_url', + label: 'Lora URL', + description: 'Lora URL', + category: ClapInputCategory.LORA, + type: 'string', + allowedValues: [], + defaultValue: '', +} + export const genericImage: ClapInputField = { id: 'image', label: 'Image', description: 'Image', + category: ClapInputCategory.IMAGE_URL, type: 'string', allowedValues: [], defaultValue: '', @@ -64,6 +91,7 @@ export const genericImageUrl: ClapInputField = { id: 'image_url', label: 'Image URL', description: 'Image URL', + category: ClapInputCategory.IMAGE_URL, type: 'string', allowedValues: [], defaultValue: '', @@ -73,6 +101,7 @@ export const genericVideo: ClapInputField = { id: 'video', label: 'Video', description: 'Video', + category: ClapInputCategory.VIDEO_URL, type: 'string', allowedValues: [], defaultValue: '', @@ -82,6 +111,7 @@ export const genericVideoUrl: ClapInputField = { id: 'video_url', label: 'Video URL', description: 'Video URL', + category: ClapInputCategory.VIDEO_URL, type: 'string', allowedValues: [], defaultValue: '', @@ -91,6 +121,7 @@ export const genericVoice: ClapInputField = { id: 'voice', label: 'Voice', description: 'Voice', + category: ClapInputCategory.SOUND_URL, type: 'string', allowedValues: [], defaultValue: '', @@ -100,6 +131,7 @@ export const genericAudio: ClapInputField = { id: 'audio', label: 'Audio', description: 'Audio', + category: ClapInputCategory.SOUND_URL, type: 'string', allowedValues: [], defaultValue: '', @@ -109,6 +141,7 @@ export const genericInferenceSteps: ClapInputField = { id: 'num_inference_steps', label: 'Inference steps', description: 'Number of inference steps', + category: ClapInputCategory.INFERENCE_STEPS, type: 'number', minValue: 1, maxValue: 50, @@ -119,9 +152,8 @@ export const genericUpscalingFactor: ClapInputField = { id: 'upscaling_factor', label: 'Upscaling Factor', description: 'Upscaling Factor (2, 3, 4..)', - // <-- TODO: we should be able to have type: 'integer' - // that is not a big issue, however (the implementation can do the rounding) - type: 'number', + category: ClapInputCategory.UPSCALING_FACTOR, + type: 'number', // <-- TODO: replace by 'integer' (might break stuff) minValue: 2, maxValue: 4, defaultValue: 2, @@ -132,8 +164,7 @@ export const genericOverlappingTiles: ClapInputField = { label: 'Overlapping Tiles', description: 'Overlapping tiles should reduce visible seams, but doubles the inference time.', - // <-- TODO: we should be able to have type: 'integer' - // that is not a big issue, however (the implementation can do the rounding) + category: ClapInputCategory.CUSTOM, type: 'boolean', defaultValue: true, } @@ -143,7 +174,8 @@ export const genericMotionBucketId: ClapInputField = { label: 'Motion Bucket ID', description: 'The motion bucket ID determines the motion of the generated video. The higher the number, the more motion there will be.', - type: 'number', + category: ClapInputCategory.CUSTOM, + type: 'number', // <-- TODO: replace by 'integer' (might break stuff) minValue: 0, maxValue: 255, defaultValue: 127, @@ -154,6 +186,7 @@ export const genericConditioningAugmentation: ClapInputField = { label: 'Conditioning Augmentation', description: 'The conditoning augmentation determines the amount of noise that will be added to the conditioning frame. The higher the number, the more noise there will be, and the less the video will look like the initial image. Increase it for more motion.', + category: ClapInputCategory.CUSTOM, type: 'number', minValue: 0, maxValue: 1, @@ -164,6 +197,7 @@ export const genericWidth1024: ClapInputField = { id: 'width', label: 'Width', description: 'Width', + category: ClapInputCategory.WIDTH, type: 'number', minValue: 256, maxValue: 1024, @@ -174,6 +208,7 @@ export const genericWidth2048: ClapInputField = { id: 'width', label: 'Width', description: 'Width', + category: ClapInputCategory.WIDTH, type: 'number', minValue: 256, maxValue: 2048, @@ -184,6 +219,7 @@ export const genericHeight1024: ClapInputField = { id: 'height', label: 'Height', description: 'Height', + category: ClapInputCategory.HEIGHT, type: 'number', minValue: 256, maxValue: 1024, @@ -194,6 +230,7 @@ export const genericHeight2048: ClapInputField = { id: 'height', label: 'Height', description: 'Height', + category: ClapInputCategory.HEIGHT, type: 'number', minValue: 256, maxValue: 2048, diff --git a/packages/app/src/services/editors/workflow-editor/workflows/common/loras/canWorkflowUseLora.ts b/packages/app/src/services/editors/workflow-editor/workflows/common/loras/canWorkflowUseLora.ts new file mode 100644 index 00000000..425ea8db --- /dev/null +++ b/packages/app/src/services/editors/workflow-editor/workflows/common/loras/canWorkflowUseLora.ts @@ -0,0 +1,9 @@ +import { ClapInputCategory, ClapWorkflow } from '@aitube/clap/dist/types' + +export function canWorkflowUseLora(workflow: ClapWorkflow): boolean { + return workflow.inputFields.some( + ({ category }) => category === ClapInputCategory.LORA + // category === ClapInputCategory.LORA_HF_MODEL || + // category === ClapInputCategory.LORA_WEIGHT_URL + ) +} diff --git a/packages/app/src/services/editors/workflow-editor/workflows/common/loras/getWorkflowInputField.ts b/packages/app/src/services/editors/workflow-editor/workflows/common/loras/getWorkflowInputField.ts new file mode 100644 index 00000000..7e8e995d --- /dev/null +++ b/packages/app/src/services/editors/workflow-editor/workflows/common/loras/getWorkflowInputField.ts @@ -0,0 +1,8 @@ +import { ClapInputCategory, ClapInputField, ClapWorkflow } from '@aitube/clap' + +export function getWorkflowInputField( + workflow: ClapWorkflow, + category: ClapInputCategory +): ClapInputField | undefined { + return workflow.inputFields.find((field) => field.category === category) +} diff --git a/packages/app/src/services/editors/workflow-editor/workflows/common/loras/getWorkflowLora.ts b/packages/app/src/services/editors/workflow-editor/workflows/common/loras/getWorkflowLora.ts new file mode 100644 index 00000000..057c7904 --- /dev/null +++ b/packages/app/src/services/editors/workflow-editor/workflows/common/loras/getWorkflowLora.ts @@ -0,0 +1,28 @@ +import { ClapInputCategory, ClapWorkflow } from '@aitube/clap' +import { Lora } from '@/services/editors/workflow-editor/workflows/common/types' +import { defaultLoraModels } from '@/services/editors/workflow-editor/workflows/common/loras' + +import { getWorkflowInputField } from './getWorkflowInputField' + +export function getWorkflowLora(workflow: ClapWorkflow): Lora | undefined { + const inputField = getWorkflowInputField(workflow, ClapInputCategory.LORA) + + if (!inputField) { + return undefined + } + + const loraRepoOrUrl: string = workflow.inputValues[inputField.id] + + if (!loraRepoOrUrl) { + return undefined + } + + const loraModel = defaultLoraModels.find((lora) => ( + lora.repoOrUrl === loraRepoOrUrl + )) + + if (!loraModel) { + return undefined + } + return loraModel +} diff --git a/packages/app/src/services/editors/workflow-editor/workflows/common/loras/index.ts b/packages/app/src/services/editors/workflow-editor/workflows/common/loras/index.ts new file mode 100644 index 00000000..25f4a7f5 --- /dev/null +++ b/packages/app/src/services/editors/workflow-editor/workflows/common/loras/index.ts @@ -0,0 +1,73 @@ +// a list of FLUX.1 LoRA models that can be used with various providers + +import { LoraBaseModel, Lora } from '../types' + +export const defaultLoraModels: Lora[] = [ + { + id: 'lora://hf.co/models/alvdansen/flux-koda', + label: 'Koda', + baseModel: LoraBaseModel.FLUX, + description: + "Koda captures the nostalgic essence of early 1990s photography, evoking memories of disposable cameras and carefree travels. It specializes in creating images with a distinct vintage quality, characterized by slightly washed-out colors, soft focus, and the occasional light leak or film grain. The model excels at producing slice-of-life scenes that feel spontaneous and candid, as if plucked from a family photo album or a backpacker's travel diary.", + + thumbnailUrl: + 'https://hf.co/alvdansen/flux-koda/resolve/main/images/ComfyUI_00566_%20(2).png', + + projectUrl: 'https://hf.co/alvdansen/flux-koda', + + author: '@alvdansen', + + // trigger (usually some kind of unique string sequence, eg TOK) + trigger: 'flmft style', + + extensions: + 'kodachrome, blurry, realistic, still life, depth of field, scenery, no humans, monochrome, greyscale, traditional media, horizon, looking at viewer, light particles, shadow', + + repoOrUrl: 'alvdansen/flux-koda', + }, + { + id: 'lora://hf.co/models/veryVANYA/ps1-style-flux', + label: 'PS1 Style', + baseModel: LoraBaseModel.FLUX, + description: `late 90s/early 2000s ps1/n64 console graphics. + +5000 steps + +trained on 15 gpt4o captioned and adjusted ps1/n64 game screenshots using https://github.com/ostris/ai-toolkit/tree/main`, + + thumbnailUrl: + 'https://huggingface.co/veryVANYA/ps1-style-flux/resolve/main/24440109.jpeg', + + projectUrl: 'https://hf.co/veryVANYA/ps1-style-flux', + author: '@veryVANYA', + + // trigger (usually some kind of unique string sequence, eg TOK) + trigger: 'ps1', + + extensions: 'ps1 game screenshot', + + repoOrUrl: 'veryVANYA/ps1-style-flux', + }, + /* + { + id: 'lora://hf.co/models/jbilcke-hf/experimental-model-1', + label: 'Experimental Model 1', + baseModel: LoraBaseModel.FLUX, + description: 'A model for internal testing', + + thumbnailUrl: '', + + projectUrl: 'clapper.app', + + author: '@jbilcke-hf', + + // trigger (usually some kind of unique string sequence, eg TOK) + trigger: 'TKE1', + + extensions: + 'movie screencap from , in , with film grain.', + + repoOrUrl: 'jbilcke-hf/experimental-model-1', + }, + */ +] diff --git a/packages/app/src/services/editors/workflow-editor/workflows/common/types.ts b/packages/app/src/services/editors/workflow-editor/workflows/common/types.ts index 071c443a..0759d5eb 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/common/types.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/common/types.ts @@ -1,3 +1,35 @@ import { ClapWorkflow } from '@aitube/clap' export type DynamicClapWorkflow = () => Promise + +export enum LoraBaseModel { + FLUX = 'FLUX', + SDXL = 'SDXL', +} + +export type Lora = { + id: string + + label: string + + baseModel: LoraBaseModel.FLUX + + description: string + + thumbnailUrl: string + + // URL to the page presenting the LoRA (eg. HF model page) + projectUrl: string + + author: string + + // trigger (usually some kind of unique string sequence, eg TOK) + trigger: string + + // additional keywords suggested by the author + extensions: string + + // name of the model repository on Hugging Face + // or direct URL to the weights + repoOrUrl: string +} diff --git a/packages/app/src/services/editors/workflow-editor/workflows/elevenlabs/index.ts b/packages/app/src/services/editors/workflow-editor/workflows/elevenlabs/index.ts index a60623fb..0db86550 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/elevenlabs/index.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/elevenlabs/index.ts @@ -19,6 +19,7 @@ export const elevenlabsWorkflows: ClapWorkflow[] = [ author: 'ElevenLabs', thumbnailUrl: '', nonCommercial: false, + canSupportLora: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.ELEVENLABS, category: ClapWorkflowCategory.VOICE_GENERATION, @@ -40,6 +41,7 @@ export const elevenlabsWorkflows: ClapWorkflow[] = [ author: 'ElevenLabs', thumbnailUrl: '', nonCommercial: false, + canSupportLora: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.ELEVENLABS, category: ClapWorkflowCategory.SOUND_GENERATION, diff --git a/packages/app/src/services/editors/workflow-editor/workflows/falai/defaultWorkflows.ts b/packages/app/src/services/editors/workflow-editor/workflows/falai/defaultWorkflows.ts index df147620..a214f1dc 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/falai/defaultWorkflows.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/falai/defaultWorkflows.ts @@ -22,6 +22,7 @@ import { genericOverlappingTiles, genericInferenceSteps, genericImageUrl, + genericLora, } from '../common/defaultValues' import { sampleDrivingVideo, sampleVoice } from '@/lib/core/constants' @@ -45,6 +46,7 @@ export const defaultWorkflows: ClapWorkflow[] = [ author: '', thumbnailUrl: '', nonCommercial: false, + canSupportLora: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.FALAI, category: ClapWorkflowCategory.VIDEO_GENERATION, @@ -65,6 +67,38 @@ export const defaultWorkflows: ClapWorkflow[] = [ cond_aug: 0.02, }, }, + /* + { + id: 'falai://fal-ai/flux-general', + label: 'Flux.1-[DEV] with LoRAs', + description: '', + tags: ['Flux', 'LoRA'], + author: '', + thumbnailUrl: '', + nonCommercial: false, + canSupportLora: false, + engine: ClapWorkflowEngine.REST_API, + provider: ClapWorkflowProvider.FALAI, + category: ClapWorkflowCategory.IMAGE_GENERATION, + data: 'fal-ai/flux-general', + schema: '', + inputFields: [ + genericPrompt, + genericWidth2048, + genericHeight2048, + genericInferenceSteps, + ], + inputValues: { + [genericPrompt.id]: genericPrompt.defaultValue, + [genericWidth2048.id]: genericWidth2048.defaultValue, + [genericHeight2048.id]: genericHeight2048.defaultValue, + [genericInferenceSteps.id]: genericInferenceSteps.defaultValue, + + // support LoRA for this model is a bit tricky, as the parameter must be in JSON + // (this is an array of LoraWeight objects, see: https://fal.ai/models/fal-ai/flux-general/playground) + }, + }, + */ { id: 'falai://fal-ai/flux-realism', label: 'Flux Realism LoRA', @@ -73,6 +107,7 @@ export const defaultWorkflows: ClapWorkflow[] = [ author: '', thumbnailUrl: '', nonCommercial: false, + canSupportLora: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.FALAI, category: ClapWorkflowCategory.IMAGE_GENERATION, @@ -102,6 +137,7 @@ export const defaultWorkflows: ClapWorkflow[] = [ author: 'BFL (https://BlackForestLabs.ai)', thumbnailUrl: '', nonCommercial: false, + canSupportLora: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.FALAI, category: ClapWorkflowCategory.IMAGE_GENERATION, @@ -133,6 +169,7 @@ export const defaultWorkflows: ClapWorkflow[] = [ author: 'BFL (https://BlackForestLabs.ai)', thumbnailUrl: '', nonCommercial: false, + canSupportLora: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.FALAI, category: ClapWorkflowCategory.IMAGE_GENERATION, @@ -153,6 +190,7 @@ export const defaultWorkflows: ClapWorkflow[] = [ author: 'BFL (https://BlackForestLabs.ai)', thumbnailUrl: '', nonCommercial: false, + canSupportLora: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.FALAI, category: ClapWorkflowCategory.IMAGE_GENERATION, @@ -184,6 +222,7 @@ export const defaultWorkflows: ClapWorkflow[] = [ author: 'Stability AI', thumbnailUrl: '', nonCommercial: false, + canSupportLora: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.FALAI, category: ClapWorkflowCategory.IMAGE_GENERATION, @@ -215,6 +254,7 @@ export const defaultWorkflows: ClapWorkflow[] = [ author: 'Stability AI', thumbnailUrl: '', nonCommercial: false, + canSupportLora: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.FALAI, category: ClapWorkflowCategory.IMAGE_GENERATION, @@ -246,6 +286,7 @@ export const defaultWorkflows: ClapWorkflow[] = [ author: 'AuraSR', thumbnailUrl: '', nonCommercial: false, + canSupportLora: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.FALAI, category: ClapWorkflowCategory.IMAGE_UPSCALING, diff --git a/packages/app/src/services/editors/workflow-editor/workflows/replicate/defaultWorkflows.ts b/packages/app/src/services/editors/workflow-editor/workflows/replicate/defaultWorkflows.ts index bfcde08e..16ba166f 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/replicate/defaultWorkflows.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/replicate/defaultWorkflows.ts @@ -9,6 +9,7 @@ import { genericHeight1024, genericHeight2048, genericImage, + genericLora, genericPrompt, genericVideo, genericWidth1024, @@ -16,9 +17,40 @@ import { } from '../common/defaultValues' export const defaultWorkflows: ClapWorkflow[] = [ + { + id: 'replicate://lucataco/flux-dev-lora', + label: 'Flux-dev-lora', + description: '', + tags: ['flux'], + author: '@lucataco', + thumbnailUrl: '', + engine: ClapWorkflowEngine.REST_API, + category: ClapWorkflowCategory.IMAGE_GENERATION, + provider: ClapWorkflowProvider.REPLICATE, + data: 'lucataco/flux-dev-lora', + // data: 'lucataco/flux-dev-lora:94a0c19e55e36f75d657ecf9eada9f16a233b5329fb9cdf8e2b9ecd093e5c97e', + /** + * Inputs of the workflow (this is used to build an UI for the automatically) + */ + inputFields: [ + genericPrompt, + genericWidth2048, + genericHeight2048, + { + ...genericLora, + id: 'hf_lora', + }, + ], + inputValues: { + prompt: genericPrompt.defaultValue, + width: genericWidth2048.defaultValue, + height: genericHeight2048.defaultValue, + hf_lora: genericLora.defaultValue, + }, + }, { id: 'replicate://black-forest-labs/flux-pro', - label: 'FLUX.1 [pro]', + label: 'Flux-pro', description: '', tags: ['flux'], author: 'BFL (https://BlackForestLabs.ai)', @@ -47,7 +79,7 @@ export const defaultWorkflows: ClapWorkflow[] = [ }, { id: 'replicate://black-forest-labs/flux-schnell', - label: 'FLUX.1 [schnell]', + label: 'Flux-schnell', description: '', tags: ['flux'], author: 'BFL (https://BlackForestLabs.ai)', @@ -68,7 +100,7 @@ export const defaultWorkflows: ClapWorkflow[] = [ }, { id: 'replicate://black-forest-labs/flux-dev', - label: 'FLUX.1 [dev]', + label: 'Flux-dev', description: '', tags: ['flux'], author: 'BFL (https://BlackForestLabs.ai)', diff --git a/packages/app/src/services/settings/useSettings.ts b/packages/app/src/services/settings/useSettings.ts index aec88403..bf8a07d1 100644 --- a/packages/app/src/services/settings/useSettings.ts +++ b/packages/app/src/services/settings/useSettings.ts @@ -2,7 +2,12 @@ import { create } from 'zustand' import { persist } from 'zustand/middleware' -import { getValidNumber, ClapWorkflowProvider } from '@aitube/clap' +import { + getValidNumber, + ClapWorkflowProvider, + ClapWorkflow, + ClapWorkflowCategory, +} from '@aitube/clap' import { parseRenderingStrategy, RenderingStrategy } from '@aitube/timeline' import { ComfyIcuAccelerator, @@ -18,10 +23,7 @@ import { getDefaultSettingsState } from './getDefaultSettingsState' import { getValidComfyWorkflowTemplate } from '@/lib/utils/getValidComfyWorkflowTemplate' import { parseComfyIcuAccelerator } from '@/lib/utils/parseComfyIcuAccelerator' -// that may not be the best way to do this, -// and importing useWorkflowEditor here is a bit tricky -import { findWorkflows } from '@/components/toolbars/top-menu/lists/getWorkflowProviders' -import { useWorkflowEditor } from '../editors/workflow-editor/useWorkflowEditor' +import { parseWorkflow } from './workflows/parseWorkflow' export const useSettings = create()( persist( @@ -303,147 +305,137 @@ export const useSettings = create()( ), }) }, - setAssistantWorkflow: (assistantWorkflow?: string) => { + setAssistantWorkflow: (assistantWorkflow?: ClapWorkflow) => { const { assistantWorkflow: defaultAssistantWorkflow } = getDefaultSettingsState() set({ - assistantWorkflow: getValidString( - assistantWorkflow, - defaultAssistantWorkflow - ), + assistantWorkflow: assistantWorkflow + ? JSON.stringify(assistantWorkflow) + : defaultAssistantWorkflow, }) }, - setAssistantTurboWorkflow: (assistantTurboWorkflow?: string) => { + setAssistantTurboWorkflow: (assistantTurboWorkflow?: ClapWorkflow) => { const { assistantTurboWorkflow: defaultAssistantTurboWorkflow } = getDefaultSettingsState() set({ - assistantTurboWorkflow: getValidString( - assistantTurboWorkflow, - defaultAssistantTurboWorkflow - ), + assistantTurboWorkflow: assistantTurboWorkflow + ? JSON.stringify(assistantTurboWorkflow) + : defaultAssistantTurboWorkflow, }) }, - setImageGenerationWorkflow: (imageGenerationWorkflow?: string) => { + setImageGenerationWorkflow: (imageGenerationWorkflow?: ClapWorkflow) => { const { imageGenerationWorkflow: defaultImageGenerationWorkflow } = getDefaultSettingsState() set({ - imageGenerationWorkflow: getValidString( - imageGenerationWorkflow, - defaultImageGenerationWorkflow - ), + imageGenerationWorkflow: imageGenerationWorkflow + ? JSON.stringify(imageGenerationWorkflow) + : defaultImageGenerationWorkflow, }) }, setImageGenerationTurboWorkflow: ( - imageGenerationTurboWorkflow?: string + imageGenerationTurboWorkflow?: ClapWorkflow ) => { const { imageGenerationTurboWorkflow: defaultImageGenerationTurboWorkflow, } = getDefaultSettingsState() set({ - imageGenerationTurboWorkflow: getValidString( - imageGenerationTurboWorkflow, - defaultImageGenerationTurboWorkflow - ), + imageGenerationTurboWorkflow: imageGenerationTurboWorkflow + ? JSON.stringify(imageGenerationTurboWorkflow) + : defaultImageGenerationTurboWorkflow, }) }, - setImageUpscalingWorkflow: (imageUpscalingWorkflow?: string) => { + setImageUpscalingWorkflow: (imageUpscalingWorkflow?: ClapWorkflow) => { const { imageUpscalingWorkflow: defaultImageUpscalingWorkflow } = getDefaultSettingsState() set({ - imageUpscalingWorkflow: getValidString( - imageUpscalingWorkflow, - defaultImageUpscalingWorkflow - ), + imageUpscalingWorkflow: imageUpscalingWorkflow + ? JSON.stringify(imageUpscalingWorkflow) + : defaultImageUpscalingWorkflow, }) }, - setImageDepthWorkflow: (imageDepthWorkflow?: string) => { + setImageDepthWorkflow: (imageDepthWorkflow?: ClapWorkflow) => { const { imageDepthWorkflow: defaultImageDepthWorkflow } = getDefaultSettingsState() set({ - imageDepthWorkflow: getValidString( - imageDepthWorkflow, - defaultImageDepthWorkflow - ), + imageDepthWorkflow: imageDepthWorkflow + ? JSON.stringify(imageDepthWorkflow) + : defaultImageDepthWorkflow, }) }, - setImageSegmentationWorkflow: (imageSegmentationWorkflow?: string) => { + setImageSegmentationWorkflow: ( + imageSegmentationWorkflow?: ClapWorkflow + ) => { const { imageSegmentationWorkflow: defaultImageSegmentationWorkflow } = getDefaultSettingsState() set({ - imageSegmentationWorkflow: getValidString( - imageSegmentationWorkflow, - defaultImageSegmentationWorkflow - ), + imageSegmentationWorkflow: imageSegmentationWorkflow + ? JSON.stringify(imageSegmentationWorkflow) + : defaultImageSegmentationWorkflow, }) }, - setVideoGenerationWorkflow: (videoGenerationWorkflow?: string) => { + setVideoGenerationWorkflow: (videoGenerationWorkflow?: ClapWorkflow) => { const { videoGenerationWorkflow: defaultVideoGenerationWorkflow } = getDefaultSettingsState() set({ - videoGenerationWorkflow: getValidString( - videoGenerationWorkflow, - defaultVideoGenerationWorkflow - ), + videoGenerationWorkflow: videoGenerationWorkflow + ? JSON.stringify(videoGenerationWorkflow) + : defaultVideoGenerationWorkflow, }) }, - setVideoUpscalingWorkflow: (videoUpscalingWorkflow?: string) => { + setVideoUpscalingWorkflow: (videoUpscalingWorkflow?: ClapWorkflow) => { const { videoUpscalingWorkflow: defaultVideoUpscalingWorkflow } = getDefaultSettingsState() set({ - videoUpscalingWorkflow: getValidString( - videoUpscalingWorkflow, - defaultVideoUpscalingWorkflow - ), + videoUpscalingWorkflow: videoUpscalingWorkflow + ? JSON.stringify(videoUpscalingWorkflow) + : defaultVideoUpscalingWorkflow, }) }, - setVideoDepthWorkflow: (videoDepthWorkflow?: string) => { + setVideoDepthWorkflow: (videoDepthWorkflow?: ClapWorkflow) => { const { videoDepthWorkflow: defaultVideoDepthWorkflow } = getDefaultSettingsState() set({ - videoDepthWorkflow: getValidString( - videoDepthWorkflow, - defaultVideoDepthWorkflow - ), + videoDepthWorkflow: videoDepthWorkflow + ? JSON.stringify(videoDepthWorkflow) + : defaultVideoDepthWorkflow, }) }, - setVideoSegmentationWorkflow: (videoSegmentationWorkflow?: string) => { + setVideoSegmentationWorkflow: ( + videoSegmentationWorkflow?: ClapWorkflow + ) => { const { videoSegmentationWorkflow: defaultVideoSegmentationWorkflow } = getDefaultSettingsState() set({ - videoSegmentationWorkflow: getValidString( - videoSegmentationWorkflow, - defaultVideoSegmentationWorkflow - ), + videoSegmentationWorkflow: videoSegmentationWorkflow + ? JSON.stringify(videoSegmentationWorkflow) + : defaultVideoSegmentationWorkflow, }) }, - setSoundGenerationWorkflow: (soundGenerationWorkflow?: string) => { + setSoundGenerationWorkflow: (soundGenerationWorkflow?: ClapWorkflow) => { const { soundGenerationWorkflow: defaultSoundGenerationWorkflow } = getDefaultSettingsState() set({ - soundGenerationWorkflow: getValidString( - soundGenerationWorkflow, - defaultSoundGenerationWorkflow - ), + soundGenerationWorkflow: soundGenerationWorkflow + ? JSON.stringify(soundGenerationWorkflow) + : defaultSoundGenerationWorkflow, }) }, - setVoiceGenerationWorkflow: (voiceGenerationWorkflow?: string) => { + setVoiceGenerationWorkflow: (voiceGenerationWorkflow?: ClapWorkflow) => { const { voiceGenerationWorkflow: defaultVoiceGenerationWorkflow } = getDefaultSettingsState() set({ - voiceGenerationWorkflow: getValidString( - voiceGenerationWorkflow, - defaultVoiceGenerationWorkflow - ), + voiceGenerationWorkflow: voiceGenerationWorkflow + ? JSON.stringify(voiceGenerationWorkflow) + : defaultVoiceGenerationWorkflow, }) }, - setMusicGenerationWorkflow: (musicGenerationWorkflow?: string) => { + setMusicGenerationWorkflow: (musicGenerationWorkflow?: ClapWorkflow) => { const { musicGenerationWorkflow: defaultVoiceGenerationWorkflow } = getDefaultSettingsState() set({ - musicGenerationWorkflow: getValidString( - musicGenerationWorkflow, - defaultVoiceGenerationWorkflow - ), + musicGenerationWorkflow: musicGenerationWorkflow + ? JSON.stringify(musicGenerationWorkflow) + : defaultVoiceGenerationWorkflow, }) }, setImageRenderingStrategy: ( @@ -735,82 +727,86 @@ export const useSettings = create()( const state = get() const defaultSettings = getDefaultSettingsState() - // I think this is causing some issues, - // with the settings having a dependency over the workflows, - // which creates a loop - // - // we should probably this step else where - const availableWorkflows = - useWorkflowEditor.getState().availableWorkflows + const assistantWorkflow = parseWorkflow( + state.assistantWorkflow || defaultSettings.assistantWorkflow, + ClapWorkflowCategory.ASSISTANT + ) - const assistantWorkflowId = - state.assistantWorkflow || defaultSettings.assistantWorkflow + const assistantTurboWorkflow = parseWorkflow( + state.assistantTurboWorkflow || + defaultSettings.assistantTurboWorkflow, + ClapWorkflowCategory.ASSISTANT + ) - const assistantTurboWorkflowId = - state.assistantTurboWorkflow || defaultSettings.assistantTurboWorkflow - - const imageGenerationWorkflowId = + const imageGenerationWorkflow = parseWorkflow( state.imageGenerationWorkflow || - defaultSettings.imageGenerationWorkflow + defaultSettings.imageGenerationWorkflow, + ClapWorkflowCategory.IMAGE_GENERATION + ) - const imageGenerationTurboWorkflowId = + const imageGenerationTurboWorkflow = parseWorkflow( state.imageGenerationTurboWorkflow || - defaultSettings.imageGenerationTurboWorkflow - - const imageUpscalingWorkflowId = - state.imageUpscalingWorkflow || defaultSettings.imageUpscalingWorkflow - - const imageDepthWorkflowId = - state.imageDepthWorkflow || defaultSettings.imageDepthWorkflow - - const imageSegmentationWorkflowId = + defaultSettings.imageGenerationTurboWorkflow, + ClapWorkflowCategory.IMAGE_GENERATION + ) + + const imageUpscalingWorkflow = parseWorkflow( + state.imageUpscalingWorkflow || + defaultSettings.imageUpscalingWorkflow, + ClapWorkflowCategory.IMAGE_UPSCALING + ) + + const imageDepthWorkflow = parseWorkflow( + state.imageDepthWorkflow || defaultSettings.imageDepthWorkflow, + ClapWorkflowCategory.IMAGE_DEPTH_MAPPING + ) + + const imageSegmentationWorkflow = parseWorkflow( state.imageSegmentationWorkflow || - defaultSettings.imageSegmentationWorkflow + defaultSettings.imageSegmentationWorkflow, + ClapWorkflowCategory.IMAGE_SEGMENTATION + ) - const videoGenerationWorkflowId = + const videoGenerationWorkflow = parseWorkflow( state.videoGenerationWorkflow || - defaultSettings.videoGenerationWorkflow + defaultSettings.videoGenerationWorkflow, + ClapWorkflowCategory.VIDEO_GENERATION + ) - const videoDepthWorkflowId = - state.videoDepthWorkflow || defaultSettings.videoDepthWorkflow + const videoDepthWorkflow = parseWorkflow( + state.videoDepthWorkflow || defaultSettings.videoDepthWorkflow, + ClapWorkflowCategory.VIDEO_DEPTH_MAPPING + ) - const videoSegmentationWorkflowId = + const videoSegmentationWorkflow = parseWorkflow( state.videoSegmentationWorkflow || - defaultSettings.videoSegmentationWorkflow + defaultSettings.videoSegmentationWorkflow, + ClapWorkflowCategory.VIDEO_SEGMENTATION + ) - const videoUpscalingWorkflowId = - state.videoUpscalingWorkflow || defaultSettings.videoUpscalingWorkflow + const videoUpscalingWorkflow = parseWorkflow( + state.videoUpscalingWorkflow || + defaultSettings.videoUpscalingWorkflow, + ClapWorkflowCategory.VIDEO_UPSCALING + ) - const soundGenerationWorkflowId = + const soundGenerationWorkflow = parseWorkflow( state.soundGenerationWorkflow || - defaultSettings.soundGenerationWorkflow + defaultSettings.soundGenerationWorkflow, + ClapWorkflowCategory.SOUND_GENERATION + ) - const voiceGenerationWorkflowId = + const voiceGenerationWorkflow = parseWorkflow( state.voiceGenerationWorkflow || - defaultSettings.voiceGenerationWorkflow + defaultSettings.voiceGenerationWorkflow, + ClapWorkflowCategory.VOICE_GENERATION + ) - const musicGenerationWorkflowId = + const musicGenerationWorkflow = parseWorkflow( state.musicGenerationWorkflow || - defaultSettings.musicGenerationWorkflow - - const { workflowIds } = findWorkflows(availableWorkflows, { - workflowIds: [ - assistantWorkflowId, - assistantTurboWorkflowId, - imageGenerationWorkflowId, - imageGenerationTurboWorkflowId, - imageUpscalingWorkflowId, - imageDepthWorkflowId, - imageSegmentationWorkflowId, - videoGenerationWorkflowId, - videoDepthWorkflowId, - videoSegmentationWorkflowId, - videoUpscalingWorkflowId, - soundGenerationWorkflowId, - voiceGenerationWorkflowId, - musicGenerationWorkflowId, - ], - }) + defaultSettings.musicGenerationWorkflow, + ClapWorkflowCategory.MUSIC_GENERATION + ) return { // why do we need those fallbacks? because some users will leave the fields empty, @@ -884,21 +880,20 @@ export const useSettings = create()( videoNegativePrompt: state.videoNegativePrompt || defaultSettings.videoNegativePrompt, - assistantWorkflow: workflowIds[assistantWorkflowId], - assistantTurboWorkflow: workflowIds[assistantTurboWorkflowId], - imageGenerationWorkflow: workflowIds[imageGenerationWorkflowId], - imageGenerationTurboWorkflow: - workflowIds[imageGenerationTurboWorkflowId], - imageUpscalingWorkflow: workflowIds[imageUpscalingWorkflowId], - imageDepthWorkflow: workflowIds[imageDepthWorkflowId], - imageSegmentationWorkflow: workflowIds[imageSegmentationWorkflowId], - videoGenerationWorkflow: workflowIds[videoGenerationWorkflowId], - videoDepthWorkflow: workflowIds[videoDepthWorkflowId], - videoSegmentationWorkflow: workflowIds[videoSegmentationWorkflowId], - videoUpscalingWorkflow: workflowIds[videoUpscalingWorkflowId], - soundGenerationWorkflow: workflowIds[soundGenerationWorkflowId], - voiceGenerationWorkflow: workflowIds[voiceGenerationWorkflowId], - musicGenerationWorkflow: workflowIds[musicGenerationWorkflowId], + assistantWorkflow, + assistantTurboWorkflow, + imageGenerationWorkflow, + imageGenerationTurboWorkflow, + imageUpscalingWorkflow, + imageDepthWorkflow, + imageSegmentationWorkflow, + videoGenerationWorkflow, + videoDepthWorkflow, + videoSegmentationWorkflow, + videoUpscalingWorkflow, + soundGenerationWorkflow, + voiceGenerationWorkflow, + musicGenerationWorkflow, imageRenderingStrategy: state.imageRenderingStrategy || diff --git a/packages/app/src/services/settings/workflows/parseWorkflow.ts b/packages/app/src/services/settings/workflows/parseWorkflow.ts new file mode 100644 index 00000000..e3b04425 --- /dev/null +++ b/packages/app/src/services/settings/workflows/parseWorkflow.ts @@ -0,0 +1,76 @@ +import { + findWorkflows, + WorkflowSearchResults, +} from '@/components/toolbars/top-menu/lists/getWorkflowProviders' +import { useWorkflowEditor } from '@/services/editors' +import { + ClapWorkflow, + ClapWorkflowCategory, + ClapWorkflowEngine, + ClapWorkflowProvider, +} from '@aitube/clap' +import { WorkflowEditorStore } from '@aitube/clapper-services' + +export function parseWorkflow( + input: string, + category: ClapWorkflowCategory +): ClapWorkflow { + const noWorkflow: ClapWorkflow = { + id: `empty://${category}`, + label: 'No workflow', + description: '', + tags: [], + author: '', + thumbnailUrl: '', + nonCommercial: false, + canSupportLora: false, + engine: ClapWorkflowEngine.DEFAULT, + category, + provider: ClapWorkflowProvider.NONE, + data: '', + schema: '', + inputFields: [], + inputValues: {}, + } + + // console.log("parseWorkflow:", { input }) + + try { + const maybeWorkflow = + typeof input === 'string' + ? (JSON.parse(input) as ClapWorkflow) + : (input as ClapWorkflow) // fallback in case some users had a bad version which didn't serialize to JSON + // console.log("maybeWorkflow:", { maybeWorkflow }) + const looksValid = + typeof maybeWorkflow?.id === 'string' && + typeof maybeWorkflow?.label === 'string' && + typeof maybeWorkflow?.description === 'string' && + typeof maybeWorkflow?.author === 'string' && + typeof maybeWorkflow?.thumbnailUrl === 'string' && + typeof maybeWorkflow?.data === 'string' && + Array.isArray(maybeWorkflow?.inputFields) && + typeof maybeWorkflow?.inputValues === 'object' + if (!looksValid) { + throw new Error(`the workflow data seems invalid`) + } + return maybeWorkflow + } catch (err) { + // console.log("error:", err) + // MIGRATION OF OLDER SETTINGS + // in case the user has an old version of the settings, the "workflow" + // will be a simple ID. So we try to recover that + const results: WorkflowSearchResults = findWorkflows( + useWorkflowEditor.getState().availableWorkflows, + { workflowId: input } + ) + + if (results.workflow) { + return results.workflow + } + + // for now let's assume we ave two cases: + // 1. the user has an old version of the settings, and we need to migrate it + // 2. the user has an empty + return noWorkflow + } +} diff --git a/packages/app/tsconfig.json b/packages/app/tsconfig.json index c2cc2f87..737ee26f 100644 --- a/packages/app/tsconfig.json +++ b/packages/app/tsconfig.json @@ -21,8 +21,18 @@ "paths": { "@/*": ["./src/*"] }, - "types": ["@webgpu/types", "@types/dom-speech-recognition"] + "types": [ + "@types/fluent-ffmpeg", + "@webgpu/types", + "@types/dom-speech-recognition" + ] }, - "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts", "vitest.config.mts"], + "include": [ + "next-env.d.ts", + "**/*.ts", + "**/*.tsx", + ".next/types/**/*.ts", + "vitest.config.mts" + ], "exclude": ["node_modules"] } diff --git a/packages/clap/src/index.ts b/packages/clap/src/index.ts index 46e8d8b1..0bfe226b 100644 --- a/packages/clap/src/index.ts +++ b/packages/clap/src/index.ts @@ -24,6 +24,7 @@ export { ClapTracks, ClapVoice, ClapCompletionMode, + ClapInputCategory, ClapInputField, ClapInputFieldNumber, ClapInputFieldInteger, diff --git a/packages/clap/src/types.ts b/packages/clap/src/types.ts index 2ae2bbb6..238b9392 100644 --- a/packages/clap/src/types.ts +++ b/packages/clap/src/types.ts @@ -434,6 +434,28 @@ export type ClapSegment = { seed: number } +// to help automatically map Clapper fields to any kind of workflows, +// we are going to use those enums +// that will help determine the semantic meaning of each field +export enum ClapInputCategory { + PROMPT = "PROMPT", + IMAGE_URL = "IMAGE_URL", + SOUND_URL = "SOUND_URL", + VIDEO_URL = "VIDEO_URL", + WIDTH = "WIDTH", + HEIGHT = "HEIGHT", + SEED = "SEED", + LORA = "LORA", + //LORA_HF_MODEL = "LORA_HF_MODEL", + //LORA_WEIGHT_URL = "LORA_WEIGHT_URL", + //MISC_HF_MODEL = "MISC_HF_MODEL", + // MISC_WEIGHT_URL = "MISC_WEIGHT_URL", + ITERATION_STEPS = "ITERATION_STEPS", + GUIDANCE_SCALE = "GUIDANCE_SCALE", + UPSCALING_FACTOR = "UPSCALING_FACTOR", + UNKNOWN = "UNKNOWN", +} + export type ClapInputFieldNumber = { type: 'number' minValue: number @@ -479,6 +501,15 @@ export type ClapInputField = { * Description of what the input field does */ description: string + + /** + * The category of the input + * This helps us associate an arbitrary parameter name + * to one of the category recognized by clapper + * + * eg "hf_model", "hf_model_repo" -> ClapInputCategory.LORA + */ + category: ClapInputCategory } & ( | ClapInputFieldNumber | ClapInputFieldInteger @@ -629,7 +660,7 @@ export type ClapWorkflow = { category: ClapWorkflowCategory provider: ClapWorkflowProvider - + /** * The workflow data itself * diff --git a/packages/clapper-services/src/settings.ts b/packages/clapper-services/src/settings.ts index 1a4553c1..6332260a 100644 --- a/packages/clapper-services/src/settings.ts +++ b/packages/clapper-services/src/settings.ts @@ -155,20 +155,20 @@ export type SettingsControls = { setVideoPromptSuffix: (videoPromptSuffix?: string) => void setVideoNegativePrompt: (videoNegativePrompt?: string) => void - setAssistantWorkflow: (assistantWorkflow?: string) => void - setAssistantTurboWorkflow: (assistantTurboWorkflow?: string) => void - setImageGenerationWorkflow: (imageGenerationWorkflow?: string) => void - setImageGenerationTurboWorkflow: (imageGenerationTurboWorkflow?: string) => void - setImageUpscalingWorkflow: (imageUpscalingWorkflow?: string) => void - setImageDepthWorkflow: (imageDepthWorkflow?: string) => void - setImageSegmentationWorkflow: (imageSegmentationWorkflow?: string) => void - setVideoGenerationWorkflow: (videoGenerationWorkflow?: string) => void - setVideoDepthWorkflow: (videoDepthWorkflow?: string) => void - setVideoSegmentationWorkflow: (videoSegmentationWorkflow?: string) => void - setVideoUpscalingWorkflow: (videoUpscalingWorkflow?: string) => void - setSoundGenerationWorkflow: (soundGenerationWorkflow?: string) => void - setVoiceGenerationWorkflow: (voiceGenerationWorkflow?: string) => void - setMusicGenerationWorkflow: (musicGenerationWorkflow?: string) => void + setAssistantWorkflow: (assistantWorkflow?: ClapWorkflow) => void + setAssistantTurboWorkflow: (assistantTurboWorkflow?: ClapWorkflow) => void + setImageGenerationWorkflow: (imageGenerationWorkflow?: ClapWorkflow) => void + setImageGenerationTurboWorkflow: (imageGenerationTurboWorkflow?: ClapWorkflow) => void + setImageUpscalingWorkflow: (imageUpscalingWorkflow?: ClapWorkflow) => void + setImageDepthWorkflow: (imageDepthWorkflow?: ClapWorkflow) => void + setImageSegmentationWorkflow: (imageSegmentationWorkflow?: ClapWorkflow) => void + setVideoGenerationWorkflow: (videoGenerationWorkflow?: ClapWorkflow) => void + setVideoDepthWorkflow: (videoDepthWorkflow?: ClapWorkflow) => void + setVideoSegmentationWorkflow: (videoSegmentationWorkflow?: ClapWorkflow) => void + setVideoUpscalingWorkflow: (videoUpscalingWorkflow?: ClapWorkflow) => void + setSoundGenerationWorkflow: (soundGenerationWorkflow?: ClapWorkflow) => void + setVoiceGenerationWorkflow: (voiceGenerationWorkflow?: ClapWorkflow) => void + setMusicGenerationWorkflow: (musicGenerationWorkflow?: ClapWorkflow) => void setImageRenderingStrategy: (imageRenderingStrategy?: RenderingStrategy) => void setImageUpscalingRenderingStrategy: (imageUpscalingRenderingStrategy?: RenderingStrategy) => void