diff --git a/packages/app/src/app/api/resolve/providers/bigmodel/index.ts b/packages/app/src/app/api/resolve/providers/bigmodel/index.ts index 20081458..29f08942 100644 --- a/packages/app/src/app/api/resolve/providers/bigmodel/index.ts +++ b/packages/app/src/app/api/resolve/providers/bigmodel/index.ts @@ -36,7 +36,7 @@ export async function resolveSegment( // https://bigmodel.cn/dev/api#cogvideox const result = await callCogVideoX(request.settings.bigModelApiKey, { - model: request.settings.videoGenerationWorkflow, + model: request.settings.videoGenerationWorkflow.data, image_url: request.prompts.video.image, }) diff --git a/packages/app/src/app/api/resolve/providers/replicate/index.ts b/packages/app/src/app/api/resolve/providers/replicate/index.ts index ce5b72ca..7aa2c422 100644 --- a/packages/app/src/app/api/resolve/providers/replicate/index.ts +++ b/packages/app/src/app/api/resolve/providers/replicate/index.ts @@ -18,7 +18,6 @@ export async function resolveSegment( const segment = request.segment if (request.segment.category == ClapSegmentCategory.STORYBOARD) { - const { workflowValues } = getWorkflowInputValues( request.settings.imageGenerationWorkflow ) @@ -31,11 +30,11 @@ export async function resolveSegment( } const aspectRatio = - request.meta.orientation === ClapMediaOrientation.SQUARE - ? "1:1" - : request.meta.orientation === ClapMediaOrientation.PORTRAIT - ? "9:16" - : "16:9" + request.meta.orientation === ClapMediaOrientation.SQUARE + ? '1:1' + : request.meta.orientation === ClapMediaOrientation.PORTRAIT + ? '9:16' + : '16:9' if ( request.settings.imageGenerationWorkflow.data === 'fofr/pulid-lightning' @@ -47,9 +46,10 @@ export async function resolveSegment( } else if ( request.settings.imageGenerationWorkflow.data === 'lucataco/flux-dev-lora' ) { - // note: this isn't the right place to do this, because maybe the LoRAs are dynamic - const loraModel = getWorkflowLora(request.settings.imageGenerationWorkflow) + const loraModel = getWorkflowLora( + request.settings.imageGenerationWorkflow + ) params = { // for some reason this model doesn't support arbitrary width and height, @@ -58,14 +58,13 @@ export async function resolveSegment( hf_lora: workflowValues['hf_lora'] || '', - prompt: [ - loraModel?.trigger, - request.prompts.image.positive - ].filter(x => x).join(' '), + prompt: [loraModel?.trigger, request.prompts.image.positive] + .filter((x) => x) + .join(' '), - disable_safety_checker: !request.settings.censorNotForAllAudiencesContent, + disable_safety_checker: + !request.settings.censorNotForAllAudiencesContent, } - } else if ( request.settings.imageGenerationWorkflow.data === 'zsxkib/pulid' ) { @@ -86,9 +85,7 @@ export async function resolveSegment( { input: params } )) as any - segment.assetUrl = `${response[0] || ''}` - } else if (request.segment.category === ClapSegmentCategory.DIALOGUE) { const response = (await replicate.run( request.settings.voiceGenerationWorkflow.data as any, @@ -96,7 +93,8 @@ export async function resolveSegment( input: { text: request.prompts.voice.positive, audio: request.prompts.voice.identity, - disable_safety_checker: !request.settings.censorNotForAllAudiencesContent, + disable_safety_checker: + !request.settings.censorNotForAllAudiencesContent, }, } )) as any @@ -107,7 +105,8 @@ export async function resolveSegment( { input: { image: request.prompts.video.image, - disable_safety_checker: !request.settings.censorNotForAllAudiencesContent, + disable_safety_checker: + !request.settings.censorNotForAllAudiencesContent, }, } )) as any diff --git a/packages/app/src/app/api/resolve/route.ts b/packages/app/src/app/api/resolve/route.ts index 7983abed..c1ea2823 100644 --- a/packages/app/src/app/api/resolve/route.ts +++ b/packages/app/src/app/api/resolve/route.ts @@ -132,8 +132,6 @@ export async function POST(req: NextRequest) { segment.outputType === ClapOutputType.AUDIO || segment.outputType === ClapOutputType.VIDEO ) { - - // TODO this should be down in the browser side, so that we can scale better const { durationInMs, hasAudio } = await getMediaInfo(segment.assetUrl) segment.assetDurationInMs = durationInMs diff --git a/packages/app/src/components/toolbars/top-menu/lists/ImageGenerationWorkflows.tsx b/packages/app/src/components/toolbars/top-menu/lists/ImageGenerationWorkflows.tsx index 005cf11f..7f357055 100644 --- a/packages/app/src/components/toolbars/top-menu/lists/ImageGenerationWorkflows.tsx +++ b/packages/app/src/components/toolbars/top-menu/lists/ImageGenerationWorkflows.tsx @@ -89,7 +89,6 @@ export function ImageGenerationWorkflows() { workflow={w} currentLora={workflowLora} onChange={(newLora?: Lora) => { - console.log(`onChange:`, { w, newLora, diff --git a/packages/app/src/components/toolbars/top-menu/lists/LoraModelList.tsx b/packages/app/src/components/toolbars/top-menu/lists/LoraModelList.tsx index 93b1ad03..405f1ab2 100644 --- a/packages/app/src/components/toolbars/top-menu/lists/LoraModelList.tsx +++ b/packages/app/src/components/toolbars/top-menu/lists/LoraModelList.tsx @@ -30,7 +30,9 @@ export function LoraModelList({
{workflow.label} - {currentLora ? `(${currentLora.label})` : `(no lora selected)`} + + {currentLora ? `(${currentLora.label})` : `(no lora selected)`} +
diff --git a/packages/app/src/lib/utils/decodeOutput.ts b/packages/app/src/lib/utils/decodeOutput.ts index ae078d6e..346bbf57 100644 --- a/packages/app/src/lib/utils/decodeOutput.ts +++ b/packages/app/src/lib/utils/decodeOutput.ts @@ -14,7 +14,6 @@ export async function decodeOutput(input: any): Promise { ? urlOrBase64 : await fetchContentToBase64(urlOrBase64) - if (base64Url.startsWith('data:image/')) { if ( base64Url.startsWith('data:image/jpeg') || diff --git a/packages/app/src/lib/utils/fetchContentToBase64.ts b/packages/app/src/lib/utils/fetchContentToBase64.ts index 8e687a1a..767ef9c5 100644 --- a/packages/app/src/lib/utils/fetchContentToBase64.ts +++ b/packages/app/src/lib/utils/fetchContentToBase64.ts @@ -1,6 +1,5 @@ export async function fetchContentToBase64(url: string) { - - const predictedFormat = url.split(".").pop()?.trim().toLowerCase() + const predictedFormat = url.split('.').pop()?.trim().toLowerCase() const res = await fetch(url, { method: 'GET', @@ -15,18 +14,28 @@ export async function fetchContentToBase64(url: string) { const buffer = Buffer.from(await blob.arrayBuffer()) // some providers such as Replicate return a generic octet-stream type in the headers - const type = blob.type === "application/octet-stream" - ? (predictedFormat === "webp" ? "image/webp" : - predictedFormat === "jpeg" ? "image/jpeg" : - predictedFormat === "jpg" ? "image/jpeg" : - predictedFormat === "png" ? "image/png" : - predictedFormat === "avif" ? "image/avif" : - predictedFormat === "heic" ? "image/heic" : - predictedFormat === "mp4" ? "video/mp4" : - predictedFormat === "mp3" ? "audio/mp3" : - predictedFormat === "wav" ? "audio/wav" : - "application/octet-stream" - ) : blob.type + const type = + blob.type === 'application/octet-stream' + ? predictedFormat === 'webp' + ? 'image/webp' + : predictedFormat === 'jpeg' + ? 'image/jpeg' + : predictedFormat === 'jpg' + ? 'image/jpeg' + : predictedFormat === 'png' + ? 'image/png' + : predictedFormat === 'avif' + ? 'image/avif' + : predictedFormat === 'heic' + ? 'image/heic' + : predictedFormat === 'mp4' + ? 'video/mp4' + : predictedFormat === 'mp3' + ? 'audio/mp3' + : predictedFormat === 'wav' + ? 'audio/wav' + : 'application/octet-stream' + : blob.type return 'data:' + type + ';base64,' + buffer.toString('base64') } diff --git a/packages/app/src/services/autocomplete/useAutocomplete.ts b/packages/app/src/services/autocomplete/useAutocomplete.ts index 997420e6..ff5863bf 100644 --- a/packages/app/src/services/autocomplete/useAutocomplete.ts +++ b/packages/app/src/services/autocomplete/useAutocomplete.ts @@ -64,7 +64,7 @@ export const useAutocomplete = create((set, get) => ({ set({ isRunning: true }) try { - const storyboards = filterSegments( + const storyboards = filterSegments( ClapSegmentFilteringMode.ANY, range, timeline.segments, diff --git a/packages/app/src/services/editors/workflow-editor/workflows/_documentation_/demo_examples.ts b/packages/app/src/services/editors/workflow-editor/workflows/_documentation_/demo_examples.ts index 42c89d79..03159628 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/_documentation_/demo_examples.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/_documentation_/demo_examples.ts @@ -10,6 +10,7 @@ // a workflow is a core concept within the OpenClap format itself // (a .clap file can contain a workflow) import { + ClapInputCategory, ClapWorkflow, ClapWorkflowCategory, ClapWorkflowEngine, @@ -48,6 +49,8 @@ const justAnExample_DoNotUseThis: ClapWorkflow[] = [ // a screenshot preview of the thumbnail thumbnailUrl: '', + nonCommercial: false, + // the engine used for this workflow engine: ClapWorkflowEngine.COMFYUI_WORKFLOW, @@ -64,6 +67,8 @@ const justAnExample_DoNotUseThis: ClapWorkflow[] = [ // containing the workflow, serialized so it can fit in a string data: '{"5":{"inputs":{"width":1024,"height":1024,"batch_size":1},"class_type":"EmptyLatentImage"},"6":{"inputs":{"clip":["11",0],"text":"detailed cinematic dof render of an old dusty detailed CRT monitor on a wooden desk in a dim room with items around, messy dirty room. On the screen are the letters "FLUX Schnell" glowing softly. High detail hard surface render"},"class_type":"CLIPTextEncode"},"8":{"inputs":{"vae":["10",0],"samples":["13",0]},"class_type":"VAEDecode"},"9":{"inputs":{"images":["8",0],"filename_prefix":"ComfyUI"},"class_type":"SaveImage"},"10":{"inputs":{"vae_name":"flux1-ae.safetensors"},"class_type":"VAELoader"},"11":{"inputs":{"type":"flux","clip_name1":"t5xxl_fp16.safetensors","clip_name2":"clip_l.safetensors"},"class_type":"DualCLIPLoader"},"12":{"inputs":{"unet_name":"flux1-schnell.safetensors","weight_dtype":"default"},"class_type":"UNETLoader"},"13":{"inputs":{"noise":["25",0],"guider":["22",0],"sigmas":["17",0],"sampler":["16",0],"latent_image":["5",0]},"class_type":"SamplerCustomAdvanced"},"16":{"inputs":{"sampler_name":"euler"},"class_type":"KSamplerSelect"},"17":{"inputs":{"model":["12",0],"steps":4,"denoise":1,"scheduler":"simple"},"class_type":"BasicScheduler"},"22":{"inputs":{"model":["12",0],"conditioning":["6",0]},"class_type":"BasicGuider"},"25":{"inputs":{"noise_seed":24016052484185},"class_type":"RandomNoise"}}', + schema: '', + // this describe the parameters of the workflow // this should be as close as possible to the actual parameters // of the workflow (eg. this could come from some kind of registry, community website API etc) @@ -73,6 +78,7 @@ const justAnExample_DoNotUseThis: ClapWorkflow[] = [ id: 'custom_field', label: 'Custom field', description: 'A field unique to this workflow', + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 1, diff --git a/packages/app/src/services/editors/workflow-editor/workflows/aitube/index.ts b/packages/app/src/services/editors/workflow-editor/workflows/aitube/index.ts index 20bbac01..1b262784 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/aitube/index.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/aitube/index.ts @@ -25,7 +25,6 @@ export const aitubeWorkflows: ClapWorkflow[] = [ author: 'AiTube.at', thumbnailUrl: '', nonCommercial: false, - canSupportLora: false, engine: ClapWorkflowEngine.OPENCLAP, category: ClapWorkflowCategory.IMAGE_GENERATION, provider: ClapWorkflowProvider.AITUBE, @@ -46,7 +45,6 @@ export const aitubeWorkflows: ClapWorkflow[] = [ author: 'AiTube.at', thumbnailUrl: '', nonCommercial: false, - canSupportLora: false, engine: ClapWorkflowEngine.OPENCLAP, category: ClapWorkflowCategory.MUSIC_GENERATION, provider: ClapWorkflowProvider.AITUBE, diff --git a/packages/app/src/services/editors/workflow-editor/workflows/anthropic/index.ts b/packages/app/src/services/editors/workflow-editor/workflows/anthropic/index.ts index e2f05b53..997f57a8 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/anthropic/index.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/anthropic/index.ts @@ -25,7 +25,6 @@ export const anthropicWorkflows: ClapWorkflow[] = [ author: 'Anthropic', thumbnailUrl: '', nonCommercial: false, - canSupportLora: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.ANTHROPIC, @@ -44,7 +43,6 @@ export const anthropicWorkflows: ClapWorkflow[] = [ author: 'Anthropic', thumbnailUrl: '', nonCommercial: false, - canSupportLora: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.ANTHROPIC, @@ -63,7 +61,6 @@ export const anthropicWorkflows: ClapWorkflow[] = [ author: 'Anthropic', thumbnailUrl: '', nonCommercial: false, - canSupportLora: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.ANTHROPIC, diff --git a/packages/app/src/services/editors/workflow-editor/workflows/bigmodel/index.ts b/packages/app/src/services/editors/workflow-editor/workflows/bigmodel/index.ts index def1b90b..0c3c8b90 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/bigmodel/index.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/bigmodel/index.ts @@ -15,7 +15,6 @@ export const bigModelWorkflows: ClapWorkflow[] = [ author: '', thumbnailUrl: '', nonCommercial: false, - canSupportLora: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.BIGMODEL, category: ClapWorkflowCategory.VIDEO_GENERATION, diff --git a/packages/app/src/services/editors/workflow-editor/workflows/cohere/index.ts b/packages/app/src/services/editors/workflow-editor/workflows/cohere/index.ts index b3e04148..004e8abd 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/cohere/index.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/cohere/index.ts @@ -23,10 +23,12 @@ export const cohereWorkflows: ClapWorkflow[] = [ tags: ['command-r'], author: 'Cohere', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.COHERE, data: 'command-r-plus', + schema: '', inputFields: [genericPrompt], inputValues: { prompt: genericPrompt.defaultValue, @@ -39,10 +41,12 @@ export const cohereWorkflows: ClapWorkflow[] = [ tags: ['command-r'], author: 'Cohere', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.COHERE, data: 'command-r', + schema: '', inputFields: [genericPrompt], inputValues: { prompt: genericPrompt.defaultValue, @@ -56,7 +60,6 @@ export const cohereWorkflows: ClapWorkflow[] = [ author: 'Cohere', thumbnailUrl: '', nonCommercial: false, - canSupportLora: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.COHERE, @@ -75,7 +78,6 @@ export const cohereWorkflows: ClapWorkflow[] = [ author: 'Cohere', thumbnailUrl: '', nonCommercial: false, - canSupportLora: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.COHERE, @@ -94,7 +96,6 @@ export const cohereWorkflows: ClapWorkflow[] = [ author: 'Cohere', thumbnailUrl: '', nonCommercial: false, - canSupportLora: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.COHERE, @@ -113,7 +114,6 @@ export const cohereWorkflows: ClapWorkflow[] = [ author: 'Cohere', thumbnailUrl: '', nonCommercial: false, - canSupportLora: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.COHERE, diff --git a/packages/app/src/services/editors/workflow-editor/workflows/comfyicu/index.ts b/packages/app/src/services/editors/workflow-editor/workflows/comfyicu/index.ts index 113b8be8..9d7158ca 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/comfyicu/index.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/comfyicu/index.ts @@ -30,7 +30,6 @@ export const comfyicuWorkflows: ClapWorkflow[] = [ author: 'BFL (https://BlackForestLabs.ai)', thumbnailUrl: '', nonCommercial: false, - canSupportLora: false, engine: ClapWorkflowEngine.COMFYUI_WORKFLOW, provider: ClapWorkflowProvider.COMFYICU, category: ClapWorkflowCategory.IMAGE_GENERATION, diff --git a/packages/app/src/services/editors/workflow-editor/workflows/comfyui/index.ts b/packages/app/src/services/editors/workflow-editor/workflows/comfyui/index.ts index de95c4ed..1ee4593f 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/comfyui/index.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/comfyui/index.ts @@ -20,7 +20,6 @@ export const comfyuiWorkflows: ClapWorkflow[] = [ author: '', thumbnailUrl: '', nonCommercial: false, - canSupportLora: false, engine: ClapWorkflowEngine.COMFYUI_WORKFLOW, provider: ClapWorkflowProvider.COMFYUI, category: ClapWorkflowCategory.IMAGE_GENERATION, @@ -53,7 +52,6 @@ export async function getDynamicComfyuiWorkflows(): Promise { author: 'You', thumbnailUrl: '', nonCommercial: false, - canSupportLora: false, engine: ClapWorkflowEngine.COMFYUI_WORKFLOW, provider: ClapWorkflowProvider.COMFYUI, category: ClapWorkflowCategory.IMAGE_GENERATION, @@ -72,7 +70,6 @@ export async function getDynamicComfyuiWorkflows(): Promise { author: 'You', thumbnailUrl: '', nonCommercial: false, - canSupportLora: false, engine: ClapWorkflowEngine.COMFYUI_WORKFLOW, provider: ClapWorkflowProvider.COMFYUI, category: ClapWorkflowCategory.VIDEO_GENERATION, @@ -91,7 +88,6 @@ export async function getDynamicComfyuiWorkflows(): Promise { author: 'You', thumbnailUrl: '', nonCommercial: false, - canSupportLora: false, engine: ClapWorkflowEngine.COMFYUI_WORKFLOW, provider: ClapWorkflowProvider.COMFYUI, category: ClapWorkflowCategory.VOICE_GENERATION, @@ -110,7 +106,6 @@ export async function getDynamicComfyuiWorkflows(): Promise { author: 'You', thumbnailUrl: '', nonCommercial: false, - canSupportLora: false, engine: ClapWorkflowEngine.COMFYUI_WORKFLOW, provider: ClapWorkflowProvider.COMFYUI, category: ClapWorkflowCategory.MUSIC_GENERATION, @@ -129,7 +124,6 @@ export async function getDynamicComfyuiWorkflows(): Promise { author: 'You', thumbnailUrl: '', nonCommercial: false, - canSupportLora: false, engine: ClapWorkflowEngine.COMFYUI_WORKFLOW, provider: ClapWorkflowProvider.COMFYUI, category: ClapWorkflowCategory.SOUND_GENERATION, diff --git a/packages/app/src/services/editors/workflow-editor/workflows/common/defaultValues.ts b/packages/app/src/services/editors/workflow-editor/workflows/common/defaultValues.ts index c2f6db16..17b10e43 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/common/defaultValues.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/common/defaultValues.ts @@ -141,7 +141,7 @@ export const genericInferenceSteps: ClapInputField = { id: 'num_inference_steps', label: 'Inference steps', description: 'Number of inference steps', - category: ClapInputCategory.INFERENCE_STEPS, + category: ClapInputCategory.ITERATION_STEPS, type: 'number', minValue: 1, maxValue: 50, @@ -164,7 +164,7 @@ export const genericOverlappingTiles: ClapInputField = { label: 'Overlapping Tiles', description: 'Overlapping tiles should reduce visible seams, but doubles the inference time.', - category: ClapInputCategory.CUSTOM, + category: ClapInputCategory.UNKNOWN, type: 'boolean', defaultValue: true, } @@ -174,7 +174,7 @@ export const genericMotionBucketId: ClapInputField = { label: 'Motion Bucket ID', description: 'The motion bucket ID determines the motion of the generated video. The higher the number, the more motion there will be.', - category: ClapInputCategory.CUSTOM, + category: ClapInputCategory.UNKNOWN, type: 'number', // <-- TODO: replace by 'integer' (might break stuff) minValue: 0, maxValue: 255, @@ -186,7 +186,7 @@ export const genericConditioningAugmentation: ClapInputField = { label: 'Conditioning Augmentation', description: 'The conditoning augmentation determines the amount of noise that will be added to the conditioning frame. The higher the number, the more noise there will be, and the less the video will look like the initial image. Increase it for more motion.', - category: ClapInputCategory.CUSTOM, + category: ClapInputCategory.UNKNOWN, type: 'number', minValue: 0, maxValue: 1, diff --git a/packages/app/src/services/editors/workflow-editor/workflows/common/loras/getWorkflowLora.ts b/packages/app/src/services/editors/workflow-editor/workflows/common/loras/getWorkflowLora.ts index 057c7904..445f3130 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/common/loras/getWorkflowLora.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/common/loras/getWorkflowLora.ts @@ -11,15 +11,15 @@ export function getWorkflowLora(workflow: ClapWorkflow): Lora | undefined { return undefined } - const loraRepoOrUrl: string = workflow.inputValues[inputField.id] + const loraRepoOrUrl: string = `${workflow.inputValues[inputField.id] || ''}` if (!loraRepoOrUrl) { return undefined } - const loraModel = defaultLoraModels.find((lora) => ( - lora.repoOrUrl === loraRepoOrUrl - )) + const loraModel = defaultLoraModels.find( + (lora) => lora.repoOrUrl === loraRepoOrUrl + ) if (!loraModel) { return undefined diff --git a/packages/app/src/services/editors/workflow-editor/workflows/elevenlabs/index.ts b/packages/app/src/services/editors/workflow-editor/workflows/elevenlabs/index.ts index 0db86550..a60623fb 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/elevenlabs/index.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/elevenlabs/index.ts @@ -19,7 +19,6 @@ export const elevenlabsWorkflows: ClapWorkflow[] = [ author: 'ElevenLabs', thumbnailUrl: '', nonCommercial: false, - canSupportLora: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.ELEVENLABS, category: ClapWorkflowCategory.VOICE_GENERATION, @@ -41,7 +40,6 @@ export const elevenlabsWorkflows: ClapWorkflow[] = [ author: 'ElevenLabs', thumbnailUrl: '', nonCommercial: false, - canSupportLora: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.ELEVENLABS, category: ClapWorkflowCategory.SOUND_GENERATION, diff --git a/packages/app/src/services/editors/workflow-editor/workflows/falai/defaultWorkflows.ts b/packages/app/src/services/editors/workflow-editor/workflows/falai/defaultWorkflows.ts index a214f1dc..808686ee 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/falai/defaultWorkflows.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/falai/defaultWorkflows.ts @@ -46,7 +46,6 @@ export const defaultWorkflows: ClapWorkflow[] = [ author: '', thumbnailUrl: '', nonCommercial: false, - canSupportLora: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.FALAI, category: ClapWorkflowCategory.VIDEO_GENERATION, @@ -76,7 +75,6 @@ export const defaultWorkflows: ClapWorkflow[] = [ author: '', thumbnailUrl: '', nonCommercial: false, - canSupportLora: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.FALAI, category: ClapWorkflowCategory.IMAGE_GENERATION, @@ -107,7 +105,6 @@ export const defaultWorkflows: ClapWorkflow[] = [ author: '', thumbnailUrl: '', nonCommercial: false, - canSupportLora: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.FALAI, category: ClapWorkflowCategory.IMAGE_GENERATION, @@ -137,7 +134,6 @@ export const defaultWorkflows: ClapWorkflow[] = [ author: 'BFL (https://BlackForestLabs.ai)', thumbnailUrl: '', nonCommercial: false, - canSupportLora: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.FALAI, category: ClapWorkflowCategory.IMAGE_GENERATION, @@ -169,7 +165,6 @@ export const defaultWorkflows: ClapWorkflow[] = [ author: 'BFL (https://BlackForestLabs.ai)', thumbnailUrl: '', nonCommercial: false, - canSupportLora: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.FALAI, category: ClapWorkflowCategory.IMAGE_GENERATION, @@ -190,7 +185,6 @@ export const defaultWorkflows: ClapWorkflow[] = [ author: 'BFL (https://BlackForestLabs.ai)', thumbnailUrl: '', nonCommercial: false, - canSupportLora: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.FALAI, category: ClapWorkflowCategory.IMAGE_GENERATION, @@ -222,7 +216,6 @@ export const defaultWorkflows: ClapWorkflow[] = [ author: 'Stability AI', thumbnailUrl: '', nonCommercial: false, - canSupportLora: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.FALAI, category: ClapWorkflowCategory.IMAGE_GENERATION, @@ -254,7 +247,6 @@ export const defaultWorkflows: ClapWorkflow[] = [ author: 'Stability AI', thumbnailUrl: '', nonCommercial: false, - canSupportLora: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.FALAI, category: ClapWorkflowCategory.IMAGE_GENERATION, @@ -286,7 +278,6 @@ export const defaultWorkflows: ClapWorkflow[] = [ author: 'AuraSR', thumbnailUrl: '', nonCommercial: false, - canSupportLora: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.FALAI, category: ClapWorkflowCategory.IMAGE_UPSCALING, @@ -310,10 +301,12 @@ export const defaultWorkflows: ClapWorkflow[] = [ tags: ['TTS'], author: 'MetaVoice (themetavoice.xyz)', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.FALAI, category: ClapWorkflowCategory.VOICE_GENERATION, data: 'fal-ai/metavoice-v1', + schema: '', inputFields: [genericPrompt, { ...genericAudio, id: 'audio_url' }], inputValues: { [genericPrompt.id]: genericPrompt.defaultValue, @@ -327,10 +320,12 @@ export const defaultWorkflows: ClapWorkflow[] = [ tags: ['audio', 'sound'], author: 'Stability AI', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.FALAI, category: ClapWorkflowCategory.SOUND_GENERATION, data: 'fal-ai/stable-audio', + schema: '', inputFields: [genericPrompt], inputValues: { [genericPrompt.id]: genericPrompt.defaultValue, @@ -343,10 +338,12 @@ export const defaultWorkflows: ClapWorkflow[] = [ tags: ['audio', 'music'], author: 'Stability AI', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.FALAI, category: ClapWorkflowCategory.MUSIC_GENERATION, data: 'fal-ai/stable-audio', + schema: '', inputFields: [genericPrompt], inputValues: { [genericPrompt.id]: genericPrompt.defaultValue, @@ -360,10 +357,12 @@ export const defaultWorkflows: ClapWorkflow[] = [ author: 'https://github.com/KwaiVGI/LivePortrait', thumbnailUrl: 'https://github.com/KwaiVGI/LivePortrait/raw/main/assets/docs/showcase2.gif', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.FALAI, category: ClapWorkflowCategory.VIDEO_GENERATION, data: 'fal-ai/live-portrait', + schema: '', inputFields: [genericImageUrl, genericVideoUrl], inputValues: { [genericImageUrl.id]: genericImageUrl.defaultValue, diff --git a/packages/app/src/services/editors/workflow-editor/workflows/fireworksai/index.ts b/packages/app/src/services/editors/workflow-editor/workflows/fireworksai/index.ts index 92742090..1b3a116e 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/fireworksai/index.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/fireworksai/index.ts @@ -18,10 +18,12 @@ export const fireworksaiWorkflows: ClapWorkflow[] = [ tags: [], author: 'Stability AI', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.FIREWORKSAI, category: ClapWorkflowCategory.IMAGE_GENERATION, data: 'stability/sd3', + schema: '', /** * Inputs of the workflow (this is used to build an UI for the automatically) */ @@ -47,10 +49,12 @@ export const fireworksaiWorkflows: ClapWorkflow[] = [ tags: [], author: 'Stability AI', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.FIREWORKSAI, category: ClapWorkflowCategory.IMAGE_GENERATION, data: 'accounts/stability/models/sd3-turbo', + schema: '', /** * Inputs of the workflow (this is used to build an UI for the automatically) */ @@ -76,10 +80,12 @@ export const fireworksaiWorkflows: ClapWorkflow[] = [ tags: [], author: 'Stability AI', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.FIREWORKSAI, category: ClapWorkflowCategory.IMAGE_GENERATION, data: 'fireworks/stable-diffusion-xl-1024-v1-0', + schema: '', /** * Inputs of the workflow (this is used to build an UI for the automatically) */ @@ -105,10 +111,12 @@ export const fireworksaiWorkflows: ClapWorkflow[] = [ tags: [], author: 'Stability AI', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.FIREWORKSAI, category: ClapWorkflowCategory.IMAGE_GENERATION, data: 'accounts/fireworks/models/playground-v2-5-1024px-aesthetic', + schema: '', /** * Inputs of the workflow (this is used to build an UI for the automatically) */ @@ -134,10 +142,12 @@ export const fireworksaiWorkflows: ClapWorkflow[] = [ tags: [], author: 'Meta', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.FIREWORKSAI, category: ClapWorkflowCategory.ASSISTANT, data: 'fireworks/llama-v3-70b-instruct', + schema: '', inputFields: [genericPrompt], inputValues: { prompt: genericPrompt.defaultValue, diff --git a/packages/app/src/services/editors/workflow-editor/workflows/google/index.ts b/packages/app/src/services/editors/workflow-editor/workflows/google/index.ts index 15e62906..3cef0c7a 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/google/index.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/google/index.ts @@ -27,10 +27,12 @@ export const googleWorkflows: ClapWorkflow[] = [ tags: ['Gemini', 'flash'], author: 'Google (GCP Vertex AI)', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.GOOGLE, data: 'gemini-1.5-flash-001', + schema: '', inputFields: [genericPrompt], inputValues: { prompt: genericPrompt.defaultValue, @@ -43,10 +45,12 @@ export const googleWorkflows: ClapWorkflow[] = [ tags: ['Gemini', 'pro'], author: 'Google (GCP Vertex AI)', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.GOOGLE, data: 'gemini-1.5-pro-001', + schema: '', inputFields: [genericPrompt], inputValues: { prompt: genericPrompt.defaultValue, @@ -59,10 +63,12 @@ export const googleWorkflows: ClapWorkflow[] = [ tags: ['Gemini', 'vision'], author: 'Google (GCP Vertex AI)', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.GOOGLE, data: 'gemini-1.0-vision-001', + schema: '', inputFields: [genericPrompt], inputValues: { prompt: genericPrompt.defaultValue, @@ -75,10 +81,12 @@ export const googleWorkflows: ClapWorkflow[] = [ tags: ['Gemini', 'vision'], author: 'Google (GCP Vertex AI)', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.GOOGLE, data: 'gemini-1.0-pro-002', + schema: '', inputFields: [genericPrompt], inputValues: { prompt: genericPrompt.defaultValue, @@ -91,10 +99,12 @@ export const googleWorkflows: ClapWorkflow[] = [ tags: ['Claude', 'Sonnet'], author: 'Google (GCP Vertex AI)', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.GOOGLE, data: 'claude-3-5-sonnet@20240620', + schema: '', inputFields: [genericPrompt], inputValues: { prompt: genericPrompt.defaultValue, @@ -107,10 +117,12 @@ export const googleWorkflows: ClapWorkflow[] = [ tags: ['Claude', 'Opus'], author: 'Google (GCP Vertex AI)', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.GOOGLE, data: 'claude-3-opus@20240229', + schema: '', inputFields: [genericPrompt], inputValues: { prompt: genericPrompt.defaultValue, @@ -123,10 +135,12 @@ export const googleWorkflows: ClapWorkflow[] = [ tags: ['Claude', 'Haiku'], author: 'Google (GCP Vertex AI)', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.GOOGLE, data: 'claude-3-haiku@20240307', + schema: '', inputFields: [genericPrompt], inputValues: { prompt: genericPrompt.defaultValue, diff --git a/packages/app/src/services/editors/workflow-editor/workflows/groq/index.ts b/packages/app/src/services/editors/workflow-editor/workflows/groq/index.ts index 9444ca19..c6e6811f 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/groq/index.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/groq/index.ts @@ -23,10 +23,12 @@ export const groqWorkflows: ClapWorkflow[] = [ tags: ['Mixtral'], author: 'Mistral AI', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.GROQ, data: 'Mixtral-8x7b-32768', + schema: '', inputFields: [genericPrompt], inputValues: { prompt: genericPrompt.defaultValue, @@ -39,10 +41,12 @@ export const groqWorkflows: ClapWorkflow[] = [ tags: ['Gemma'], author: 'Google', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.GROQ, data: 'Gemma-7b-lt', + schema: '', inputFields: [genericPrompt], inputValues: { prompt: genericPrompt.defaultValue, @@ -55,10 +59,12 @@ export const groqWorkflows: ClapWorkflow[] = [ tags: ['Llama3'], author: 'Meta', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.GROQ, data: 'Llama3-70b-8192', + schema: '', inputFields: [genericPrompt], inputValues: { prompt: genericPrompt.defaultValue, @@ -71,10 +77,12 @@ export const groqWorkflows: ClapWorkflow[] = [ tags: ['Llama3'], author: 'Google', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.GROQ, data: 'Llama3-8b-8192', + schema: '', inputFields: [genericPrompt], inputValues: { prompt: genericPrompt.defaultValue, diff --git a/packages/app/src/services/editors/workflow-editor/workflows/huggingface/index.ts b/packages/app/src/services/editors/workflow-editor/workflows/huggingface/index.ts index 7ec17f20..c579c3b7 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/huggingface/index.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/huggingface/index.ts @@ -18,10 +18,12 @@ export const huggingfaceWorkflows: ClapWorkflow[] = [ tags: ['flux'], author: 'BFL (https://BlackForestLabs.ai)', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.HUGGINGFACE, category: ClapWorkflowCategory.IMAGE_GENERATION, data: 'black-forest-labs/FLUX.1-schnell', + schema: '', inputFields: [genericPrompt, genericWidth2048, genericHeight2048], inputValues: { prompt: genericPrompt.defaultValue, @@ -36,10 +38,12 @@ export const huggingfaceWorkflows: ClapWorkflow[] = [ tags: ['flux'], author: 'BFL (https://BlackForestLabs.ai)', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.HUGGINGFACE, category: ClapWorkflowCategory.IMAGE_GENERATION, data: 'black-forest-labs/FLUX.1-dev', + schema: '', /** * Inputs of the workflow (this is used to build an UI for the automatically) */ @@ -66,10 +70,12 @@ export const huggingfaceWorkflows: ClapWorkflow[] = [ author: '@jbilcke-hf', // TODO add specific field about licensing? thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.HUGGINGFACE, category: ClapWorkflowCategory.IMAGE_GENERATION, data: 'jbilcke-hf/flux-dev-panorama-lora-2', + schema: '', inputFields: [genericPrompt, genericWidth2048, genericHeight2048], inputValues: { prompt: genericPrompt.defaultValue, @@ -84,10 +90,12 @@ export const huggingfaceWorkflows: ClapWorkflow[] = [ tags: ['TTS'], author: 'Coqui', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.HUGGINGFACE, category: ClapWorkflowCategory.VOICE_GENERATION, data: 'coqui/XTTS-v2', + schema: '', /** * Inputs of the workflow (this is used to build an UI for the automatically) */ @@ -103,10 +111,12 @@ export const huggingfaceWorkflows: ClapWorkflow[] = [ tags: ['TTS'], author: 'MyShell.ai', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.HUGGINGFACE, category: ClapWorkflowCategory.VOICE_GENERATION, data: 'myshell-ai/OpenVoiceV2', + schema: '', /** * Inputs of the workflow (this is used to build an UI for the automatically) */ @@ -122,10 +132,12 @@ export const huggingfaceWorkflows: ClapWorkflow[] = [ tags: ['TTS'], author: 'MyShell.ai', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.HUGGINGFACE, category: ClapWorkflowCategory.VOICE_GENERATION, data: 'myshell-ai/OpenVoice', + schema: '', /** * Inputs of the workflow (this is used to build an UI for the automatically) */ @@ -141,10 +153,12 @@ export const huggingfaceWorkflows: ClapWorkflow[] = [ tags: ['TTS'], author: 'WhisperSpeech', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.HUGGINGFACE, category: ClapWorkflowCategory.VOICE_GENERATION, data: 'WhisperSpeech/WhisperSpeech', + schema: '', /** * Inputs of the workflow (this is used to build an UI for the automatically) */ @@ -160,10 +174,12 @@ export const huggingfaceWorkflows: ClapWorkflow[] = [ tags: ['TTS'], author: 'MetaVoice (themetavoice.xyz)', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.HUGGINGFACE, category: ClapWorkflowCategory.VOICE_GENERATION, data: 'metavoiceio/metavoice-1B-v0.1', + schema: '', /** * Inputs of the workflow (this is used to build an UI for the automatically) */ @@ -179,10 +195,12 @@ export const huggingfaceWorkflows: ClapWorkflow[] = [ tags: ['TTS'], author: 'ParlerTTS', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.HUGGINGFACE, category: ClapWorkflowCategory.VOICE_GENERATION, data: 'parler-tts/parler_tts_mini_v0.1', + schema: '', /** * Inputs of the workflow (this is used to build an UI for the automatically) */ @@ -198,10 +216,12 @@ export const huggingfaceWorkflows: ClapWorkflow[] = [ tags: ['TTS'], author: 'ParlerTTS', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.HUGGINGFACE, category: ClapWorkflowCategory.VOICE_GENERATION, data: 'parler-tts/parler-tts-mini-expresso', + schema: '', /** * Inputs of the workflow (this is used to build an UI for the automatically) */ @@ -217,10 +237,12 @@ export const huggingfaceWorkflows: ClapWorkflow[] = [ tags: ['Zephyr'], author: 'Hugging Face H4', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.HUGGINGFACE, category: ClapWorkflowCategory.ASSISTANT, data: 'HuggingFaceH4/zephyr-7b-beta', + schema: '', inputFields: [genericPrompt], inputValues: { prompt: genericPrompt.defaultValue, @@ -233,10 +255,12 @@ export const huggingfaceWorkflows: ClapWorkflow[] = [ tags: ['Zephyr'], author: 'Mistral AI', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.HUGGINGFACE, category: ClapWorkflowCategory.ASSISTANT, data: 'mistralai/Mixtral-8x7B-Instruct-v0.1', + schema: '', inputFields: [genericPrompt], inputValues: { prompt: genericPrompt.defaultValue, @@ -250,10 +274,12 @@ export const huggingfaceWorkflows: ClapWorkflow[] = [ tags: ['music'], author: 'CVSSP', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.HUGGINGFACE, category: ClapWorkflowCategory.MUSIC_GENERATION, data: 'cvssp/audioldm2-music', + schema: '', // Inputs of the workflow (this is used to build an UI for the automatically) inputFields: [ genericPrompt, diff --git a/packages/app/src/services/editors/workflow-editor/workflows/letzai/index.ts b/packages/app/src/services/editors/workflow-editor/workflows/letzai/index.ts index 7b863ea7..24031d1f 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/letzai/index.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/letzai/index.ts @@ -14,14 +14,16 @@ export const letzAiWorkflows: ClapWorkflow[] = [ { id: 'letzai://api/images', label: 'Create image with LetzAI', - image: '', + description: '', tags: ['image'], author: '', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.LETZAI, category: ClapWorkflowCategory.IMAGE_GENERATION, data: 'api/images', // <- this value isn't really used, it's just to put something here + schema: '', inputFields: [genericPrompt, genericWidth1024, genericHeight1024], inputValues: { [genericPrompt.id]: genericPrompt.defaultValue, diff --git a/packages/app/src/services/editors/workflow-editor/workflows/mistralai/index.ts b/packages/app/src/services/editors/workflow-editor/workflows/mistralai/index.ts index 229beef3..16d92952 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/mistralai/index.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/mistralai/index.ts @@ -23,10 +23,12 @@ export const mistralaiWorkflows: ClapWorkflow[] = [ tags: ['Mistral', '7b'], author: 'Mistral AI', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.MISTRALAI, data: 'open-mistral-7b', + schema: '', inputFields: [genericPrompt], inputValues: { prompt: genericPrompt.defaultValue, @@ -39,10 +41,12 @@ export const mistralaiWorkflows: ClapWorkflow[] = [ tags: ['Mixtral', '8x7b'], author: 'Mistral AI', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.MISTRALAI, data: 'open-mixtral-8x7b', + schema: '', inputFields: [genericPrompt], inputValues: { prompt: genericPrompt.defaultValue, @@ -55,10 +59,12 @@ export const mistralaiWorkflows: ClapWorkflow[] = [ tags: ['Mixtral', '8x22b'], author: 'Mistral AI', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.MISTRALAI, data: 'open-mixtral-8x22b', + schema: '', inputFields: [genericPrompt], inputValues: { prompt: genericPrompt.defaultValue, @@ -71,10 +77,12 @@ export const mistralaiWorkflows: ClapWorkflow[] = [ tags: ['Mistral', 'medium'], author: 'Mistral AI', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.MISTRALAI, data: 'mistral-medium', + schema: '', inputFields: [genericPrompt], inputValues: { prompt: genericPrompt.defaultValue, @@ -87,10 +95,12 @@ export const mistralaiWorkflows: ClapWorkflow[] = [ tags: ['Mistral', 'large'], author: 'Mistral AI', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.MISTRALAI, data: 'mistral-large-2402', + schema: '', inputFields: [genericPrompt], inputValues: { prompt: genericPrompt.defaultValue, diff --git a/packages/app/src/services/editors/workflow-editor/workflows/openai/index.ts b/packages/app/src/services/editors/workflow-editor/workflows/openai/index.ts index e56f6099..b91594c7 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/openai/index.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/openai/index.ts @@ -23,10 +23,12 @@ export const openaiWorkflows: ClapWorkflow[] = [ tags: ['GPT-4o', 'mini'], author: 'OpenAI', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.OPENAI, data: 'gpt-4o-mini', + schema: '', inputFields: [genericPrompt], inputValues: { prompt: genericPrompt.defaultValue, @@ -39,10 +41,12 @@ export const openaiWorkflows: ClapWorkflow[] = [ tags: ['GPT-4o'], author: 'OpenAI', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.OPENAI, data: 'gpt-4o', + schema: '', inputFields: [genericPrompt], inputValues: { prompt: genericPrompt.defaultValue, @@ -55,10 +59,12 @@ export const openaiWorkflows: ClapWorkflow[] = [ tags: ['GPT-4', 'mini'], author: 'OpenAI', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.OPENAI, data: 'gpt-4o-mini', + schema: '', inputFields: [genericPrompt], inputValues: { prompt: genericPrompt.defaultValue, @@ -71,10 +77,12 @@ export const openaiWorkflows: ClapWorkflow[] = [ tags: ['GPT-4', 'turbo'], author: 'OpenAI', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.ASSISTANT, provider: ClapWorkflowProvider.OPENAI, data: 'gpt-4-turbo', + schema: '', inputFields: [genericPrompt], inputValues: { prompt: genericPrompt.defaultValue, diff --git a/packages/app/src/services/editors/workflow-editor/workflows/piapi/index.ts b/packages/app/src/services/editors/workflow-editor/workflows/piapi/index.ts index 1b6740b9..359ac265 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/piapi/index.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/piapi/index.ts @@ -10,14 +10,16 @@ export const piApiWorkflows: ClapWorkflow[] = [ { id: 'piapi://Midjourney/Imagine', label: 'Midjourne Imagine', - image: '', + description: '', tags: ['Midjourney'], author: '', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.PIAPI, category: ClapWorkflowCategory.IMAGE_GENERATION, data: 'mj/v2/imagine', // <- this value isn't really used, it's just to put something here + schema: '', inputFields: [ genericPrompt, // genericRatio @@ -34,14 +36,16 @@ export const piApiWorkflows: ClapWorkflow[] = [ { id: 'piapi://LumaLabs/DreamMachine', label: 'Luma Labs Dream Machine', - image: '', + description: '', tags: ['Dream Machine'], author: '', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.PIAPI, category: ClapWorkflowCategory.VIDEO_GENERATION, data: 'luma/v1/video', // <- this value isn't really used, it's just to put something here + schema: '', inputFields: [ genericImageUrl, // genericRatio diff --git a/packages/app/src/services/editors/workflow-editor/workflows/replicate/defaultWorkflows.ts b/packages/app/src/services/editors/workflow-editor/workflows/replicate/defaultWorkflows.ts index 16ba166f..23d6da81 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/replicate/defaultWorkflows.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/replicate/defaultWorkflows.ts @@ -24,10 +24,12 @@ export const defaultWorkflows: ClapWorkflow[] = [ tags: ['flux'], author: '@lucataco', thumbnailUrl: '', + nonCommercial: true, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.IMAGE_GENERATION, provider: ClapWorkflowProvider.REPLICATE, data: 'lucataco/flux-dev-lora', + schema: '', // data: 'lucataco/flux-dev-lora:94a0c19e55e36f75d657ecf9eada9f16a233b5329fb9cdf8e2b9ecd093e5c97e', /** * Inputs of the workflow (this is used to build an UI for the automatically) @@ -55,10 +57,12 @@ export const defaultWorkflows: ClapWorkflow[] = [ tags: ['flux'], author: 'BFL (https://BlackForestLabs.ai)', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, category: ClapWorkflowCategory.IMAGE_GENERATION, provider: ClapWorkflowProvider.REPLICATE, data: 'black-forest-labs/flux-pro', + schema: '', /** * Inputs of the workflow (this is used to build an UI for the automatically) */ @@ -84,10 +88,12 @@ export const defaultWorkflows: ClapWorkflow[] = [ tags: ['flux'], author: 'BFL (https://BlackForestLabs.ai)', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.REPLICATE, category: ClapWorkflowCategory.IMAGE_GENERATION, data: 'black-forest-labs/flux-schnell', + schema: '', /** * Inputs of the workflow (this is used to build an UI for the automatically) */ @@ -105,10 +111,12 @@ export const defaultWorkflows: ClapWorkflow[] = [ tags: ['flux'], author: 'BFL (https://BlackForestLabs.ai)', thumbnailUrl: '', + nonCommercial: true, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.REPLICATE, category: ClapWorkflowCategory.IMAGE_GENERATION, data: 'black-forest-labs/flux-dev', + schema: '', /** * Inputs of the workflow (this is used to build an UI for the automatically) */ @@ -134,10 +142,12 @@ export const defaultWorkflows: ClapWorkflow[] = [ tags: ['SDXL'], author: 'Stability AI', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.REPLICATE, category: ClapWorkflowCategory.IMAGE_GENERATION, data: 'stability-ai/fast-sdxl', + schema: '', /** * Inputs of the workflow (this is used to build an UI for the automatically) */ @@ -163,10 +173,12 @@ export const defaultWorkflows: ClapWorkflow[] = [ tags: ['OpenVoice'], author: '@chenxwh', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.REPLICATE, category: ClapWorkflowCategory.VOICE_GENERATION, data: 'chenxwh/openvoice', + schema: '', /** * Inputs of the workflow (this is used to build an UI for the automatically) */ @@ -182,10 +194,12 @@ export const defaultWorkflows: ClapWorkflow[] = [ tags: ['video', 'upscaling'], author: '', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.REPLICATE, category: ClapWorkflowCategory.VIDEO_UPSCALING, data: 'lucataco/real-esrgan-video', + schema: '', inputFields: [ { ...genericVideo, @@ -203,10 +217,12 @@ export const defaultWorkflows: ClapWorkflow[] = [ tags: ['image', 'upscaling'], author: '', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.REPLICATE, category: ClapWorkflowCategory.IMAGE_UPSCALING, data: 'nightmareai/real-esrgan', + schema: '', inputFields: [genericImage], inputValues: { prompt: genericImage.defaultValue, diff --git a/packages/app/src/services/editors/workflow-editor/workflows/stabilityai/index.ts b/packages/app/src/services/editors/workflow-editor/workflows/stabilityai/index.ts index 358f9e99..50a1f043 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/stabilityai/index.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/stabilityai/index.ts @@ -21,10 +21,12 @@ export const stabilityaiWorkflows: ClapWorkflow[] = [ tags: ['SVD'], author: 'Stability AI', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.STABILITYAI, category: ClapWorkflowCategory.VIDEO_GENERATION, data: 'image-to-video', + schema: '', /** * Inputs of the workflow (this is used to build an UI for the automatically) */ @@ -40,10 +42,12 @@ export const stabilityaiWorkflows: ClapWorkflow[] = [ tags: ['SD', 'Ultra'], author: 'Stability AI', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.STABILITYAI, category: ClapWorkflowCategory.IMAGE_GENERATION, data: 'stable-image/generate/ultra', + schema: '', /** * Inputs of the workflow (this is used to build an UI for the automatically) */ @@ -69,10 +73,12 @@ export const stabilityaiWorkflows: ClapWorkflow[] = [ tags: ['SD', 'Core'], author: 'Stability AI', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.STABILITYAI, category: ClapWorkflowCategory.IMAGE_GENERATION, data: 'stable-image/generate/core', + schema: '', /** * Inputs of the workflow (this is used to build an UI for the automatically) */ @@ -98,10 +104,12 @@ export const stabilityaiWorkflows: ClapWorkflow[] = [ tags: ['SD', 'SD3'], author: 'Stability AI', thumbnailUrl: '', + nonCommercial: false, engine: ClapWorkflowEngine.REST_API, provider: ClapWorkflowProvider.STABILITYAI, category: ClapWorkflowCategory.IMAGE_GENERATION, data: 'stable-image/generate/sd3', + schema: '', /** * Inputs of the workflow (this is used to build an UI for the automatically) */ diff --git a/packages/app/src/services/io/useIO.ts b/packages/app/src/services/io/useIO.ts index b4580311..d8f6910a 100644 --- a/packages/app/src/services/io/useIO.ts +++ b/packages/app/src/services/io/useIO.ts @@ -193,7 +193,7 @@ export const useIO = create((set, get) => ({ height: 576, defaultVideoModel: '', // <-- we should deprecate this no? - extraPositivePrompt: '', + extraPositivePrompt: [], screenplay: '', isLoop: false, isInteractive: false, @@ -631,16 +631,18 @@ export const useIO = create((set, get) => ({ } else { console.log(`editing the clap to add a new final video`) timelineSegments.push( - newSegment({ - category: ClapSegmentCategory.VIDEO, - status: ClapSegmentStatus.COMPLETED, - startTimeInMs: 0, - endTimeInMs: meta.durationInMs, - assetUrl: videoDataUrl, - assetDurationInMs: meta.durationInMs, - assetSourceType: getClapAssetSourceType(videoDataUrl), - outputGain: 1.0, - }) + await clapSegmentToTimelineSegment( + newSegment({ + category: ClapSegmentCategory.VIDEO, + status: ClapSegmentStatus.COMPLETED, + startTimeInMs: 0, + endTimeInMs: meta.durationInMs, + assetUrl: videoDataUrl, + assetDurationInMs: meta.durationInMs, + assetSourceType: getClapAssetSourceType(videoDataUrl), + outputGain: 1.0, + }) + ) ) } diff --git a/packages/app/src/services/metrics/constants.ts b/packages/app/src/services/metrics/constants.ts index 4d03cf84..a8e889aa 100644 --- a/packages/app/src/services/metrics/constants.ts +++ b/packages/app/src/services/metrics/constants.ts @@ -150,4 +150,13 @@ export const estimatedMetrics: Record< [ClapWorkflowProvider.EVERARTAI]: { // TODO list the most popular models }, + [ClapWorkflowProvider.LETZAI]: { + // TODO list the most popular models + }, + [ClapWorkflowProvider.BIGMODEL]: { + // TODO list the most popular models + }, + [ClapWorkflowProvider.PIAPI]: { + // TODO list the most popular models + }, } diff --git a/packages/app/src/services/metrics/getDefaultMetricsPerProvider.ts b/packages/app/src/services/metrics/getDefaultMetricsPerProvider.ts index f5456e95..62f44c9d 100644 --- a/packages/app/src/services/metrics/getDefaultMetricsPerProvider.ts +++ b/packages/app/src/services/metrics/getDefaultMetricsPerProvider.ts @@ -99,6 +99,15 @@ export function getDefaultMetricsPerProvider(): MetricsPerProvider { [ClapWorkflowProvider.EVERARTAI]: { ...getDefaultClapWorkflowProviderMetrics(), }, + [ClapWorkflowProvider.LETZAI]: { + ...getDefaultClapWorkflowProviderMetrics(), + }, + [ClapWorkflowProvider.BIGMODEL]: { + ...getDefaultClapWorkflowProviderMetrics(), + }, + [ClapWorkflowProvider.PIAPI]: { + ...getDefaultClapWorkflowProviderMetrics(), + }, } return metricsPerProvider } diff --git a/packages/app/src/services/settings/workflows/parseWorkflow.ts b/packages/app/src/services/settings/workflows/parseWorkflow.ts index e3b04425..ec14eb02 100644 --- a/packages/app/src/services/settings/workflows/parseWorkflow.ts +++ b/packages/app/src/services/settings/workflows/parseWorkflow.ts @@ -23,7 +23,6 @@ export function parseWorkflow( author: '', thumbnailUrl: '', nonCommercial: false, - canSupportLora: false, engine: ClapWorkflowEngine.DEFAULT, category, provider: ClapWorkflowProvider.NONE, @@ -40,7 +39,7 @@ export function parseWorkflow( typeof input === 'string' ? (JSON.parse(input) as ClapWorkflow) : (input as ClapWorkflow) // fallback in case some users had a bad version which didn't serialize to JSON - // console.log("maybeWorkflow:", { maybeWorkflow }) + // console.log("maybeWorkflow:", { maybeWorkflow }) const looksValid = typeof maybeWorkflow?.id === 'string' && typeof maybeWorkflow?.label === 'string' && diff --git a/packages/broadway/src/analysis/analyzeScreenplay.ts b/packages/broadway/src/analysis/analyzeScreenplay.ts index f627c080..ebc1d9fc 100644 --- a/packages/broadway/src/analysis/analyzeScreenplay.ts +++ b/packages/broadway/src/analysis/analyzeScreenplay.ts @@ -170,7 +170,7 @@ export async function analyzeScreenplay( } if (!entitiesByScreenplayLabel[uppercaseAssetName]) { const newEnt = newEntity({ - category: "location", + category: ClapSegmentCategory.LOCATION, triggerName: uppercaseAssetName, // uppercase label: uppercaseAssetName, diff --git a/packages/clap/src/utils/filterSegments.ts b/packages/clap/src/utils/filterSegments.ts index 336dd291..514b6935 100644 --- a/packages/clap/src/utils/filterSegments.ts +++ b/packages/clap/src/utils/filterSegments.ts @@ -1,4 +1,4 @@ -import { ClapSegment, ClapSegmentCategory, ClapSegmentFilteringMode } from "@/types" +import { ClapSegmentCategory, ClapSegmentFilteringMode } from "@/types" import { filterSegmentsWithinRange } from "@/utils/filterSegmentsWithinRange" /** @@ -13,27 +13,31 @@ import { filterSegmentsWithinRange } from "@/utils/filterSegmentsWithinRange" * to get a bit more flexibility * * @param mode - * @param segment + * @param range * @param segments * @param category optional, to also filter by category * @returns */ -export function filterSegments( +export function filterSegments( mode: ClapSegmentFilteringMode, - segment?: { + range?: { startTimeInMs: number endTimeInMs: number category?: ClapSegmentCategory }, - segments?: ClapSegment[], + segments?: T[], category?: ClapSegmentCategory -): ClapSegment[] { +): T[] { const array = Array.isArray(segments) ? segments : [] - if (!segment || !array.length) { return [] } + if (!range || !array.length) { return [] } - const { startTimeInMs, endTimeInMs } = segment + const { startTimeInMs, endTimeInMs } = range - return filterSegmentsWithinRange(mode, startTimeInMs, endTimeInMs, segments, category) + return filterSegmentsWithinRange(mode, startTimeInMs, endTimeInMs, segments, category) } \ No newline at end of file diff --git a/packages/clap/src/utils/filterSegmentsWithinRange.ts b/packages/clap/src/utils/filterSegmentsWithinRange.ts index f1f319ef..94f74915 100644 --- a/packages/clap/src/utils/filterSegmentsWithinRange.ts +++ b/packages/clap/src/utils/filterSegmentsWithinRange.ts @@ -17,13 +17,17 @@ import { ClapSegment, ClapSegmentCategory, ClapSegmentFilteringMode } from "@/ty * @param category optional, to also filter by category * @returns */ -export function filterSegmentsWithinRange( +export function filterSegmentsWithinRange( mode: ClapSegmentFilteringMode, startTimeInMs: number, endTimeInMs: number, - segments?: ClapSegment[], + segments?: T[], category?: ClapSegmentCategory -): ClapSegment[] { +): T[] { const array = Array.isArray(segments) ? segments : [] diff --git a/packages/timeline/src/demo.tsx b/packages/timeline/src/demo.tsx index 6e2070a9..cbc7b570 100644 --- a/packages/timeline/src/demo.tsx +++ b/packages/timeline/src/demo.tsx @@ -2,17 +2,18 @@ import React from 'react' import { createRoot } from 'react-dom/client' import './demo.css' -import { ClapTimeline, useTimeline } from '.' +import { ClapTimeline, SegmentEditionStatus, SegmentVisibility, TimelineSegment, useTimeline } from '.' +import { ClapAssetSource, ClapOutputType, ClapSegmentCategory, ClapSegmentStatus } from '@aitube/clap' const container = document.getElementById('root') const root = createRoot(container!) -const segment = { +const segment: TimelineSegment = { id: '105246fd-7be3-4dfa-9596-01068ad858e9', track: 1, startTimeInMs: 0, endTimeInMs: 500 * 4, - category: 'VIDEO', + category: ClapSegmentCategory.VIDEO, entityId: '', workflowId: '', sceneId: '7d12441e-e002-4882-80e0-37c09fde5dc9', @@ -20,12 +21,12 @@ const segment = { endTimeInLines: 0, prompt: 'movie', label: 'movie', - outputType: 'VIDEO', + outputType: ClapOutputType.VIDEO, renderId: '', - status: 'TO_GENERATE', + status: ClapSegmentStatus.TO_GENERATE, assetUrl: '', assetDurationInMs: 1000, - assetSourceType: 'EMPTY', + assetSourceType: ClapAssetSource.EMPTY, assetFileFormat: '', revision: 0, createdAt: '2024-08-15T21:08:38.438Z', @@ -33,7 +34,7 @@ const segment = { editedBy: 'auto', outputGain: 0, seed: 1523799474, - visibility: 'VISIBLE', + visibility: SegmentVisibility.VISIBLE, textures: {}, isSelected: false, isHovered: false, @@ -45,7 +46,7 @@ const segment = { isGrabbedOnBody: false, isGrabbedOnLeftHandle: false, isGrabbedOnRightHandle: false, - editionStatus: 'EDITABLE', + editionStatus: SegmentEditionStatus.EDITABLE, } useTimeline.setState({ @@ -202,7 +203,6 @@ useTimeline.setState({ hue: 0, occupied: false, visible: true, - contentHeight: 48, }, { id: 1, @@ -212,7 +212,6 @@ useTimeline.setState({ hue: 0, occupied: true, visible: true, - contentHeight: 100, }, ], segments: [segment], diff --git a/packages/timeline/src/utils/sliceSegments.ts b/packages/timeline/src/utils/sliceSegments.ts index d5d1c96e..8e5a57d4 100644 --- a/packages/timeline/src/utils/sliceSegments.ts +++ b/packages/timeline/src/utils/sliceSegments.ts @@ -1,18 +1,22 @@ -import { ClapSegmentFilteringMode, filterSegmentsWithinRange } from "@aitube/clap" +import { ClapSegmentCategory, ClapSegmentFilteringMode, filterSegmentsWithinRange } from "@aitube/clap" import { TimelineSegment } from "@/types" // TODO put this in a web workers for smoother operations? -export function sliceSegments({ +export function sliceSegments({ segments, afterTimeInMs, beforeTimeInMs, }: { - segments: TimelineSegment[] + segments: T[] afterTimeInMs: number beforeTimeInMs: number -}): TimelineSegment[] { - const result = filterSegmentsWithinRange( +}): T[] { + const result = filterSegmentsWithinRange( ClapSegmentFilteringMode.ANY, afterTimeInMs, beforeTimeInMs,