From 5a65de0c97eca371cf55195b0687bc25ce9f2d40 Mon Sep 17 00:00:00 2001 From: Daniel Mauricio Flores Date: Sun, 18 Aug 2024 15:51:24 +0200 Subject: [PATCH] feat: add ComfyUI workflow settings edition - Add new form component to set settings of a ComfyUI workflow. - Add utils to query ComfyUI workflows. - Add generator of inputFields/inputValues based on ComfyUI workflows. - Clean integration with comfyui-sdk. - Types enhancements. - Add example of the default workflow used by ComfyUI --- .../comfyui/default_comfyui_workflow_api.json | 107 +++ .../api/resolve/providers/comfyui/index.ts | 129 ++- .../resolve/providers/comfyui/utils.spec.ts | 220 ++++-- .../api/resolve/providers/comfyui/utils.ts | 735 ++++++++++++++---- .../app/src/components/forms/FormArea.tsx | 27 +- .../forms/FormComfyUIWorkflowSettings.tsx | 176 +++++ .../app/src/components/settings/image.tsx | 41 +- .../app/src/services/resolver/useResolver.ts | 1 - .../app/src/services/settings/useSettings.ts | 34 +- .../settings/workflows/parseWorkflow.ts | 7 + packages/clap/src/types.ts | 20 +- 11 files changed, 1165 insertions(+), 332 deletions(-) create mode 100644 packages/app/public/workflows/comfyui/default_comfyui_workflow_api.json create mode 100644 packages/app/src/components/forms/FormComfyUIWorkflowSettings.tsx diff --git a/packages/app/public/workflows/comfyui/default_comfyui_workflow_api.json b/packages/app/public/workflows/comfyui/default_comfyui_workflow_api.json new file mode 100644 index 00000000..4a96fc4a --- /dev/null +++ b/packages/app/public/workflows/comfyui/default_comfyui_workflow_api.json @@ -0,0 +1,107 @@ +{ + "3": { + "inputs": { + "seed": 156680208700286, + "steps": 10, + "cfg": 8, + "sampler_name": "euler", + "scheduler": "normal", + "denoise": 1, + "model": [ + "4", + 0 + ], + "positive": [ + "6", + 0 + ], + "negative": [ + "7", + 0 + ], + "latent_image": [ + "5", + 0 + ] + }, + "class_type": "KSampler", + "_meta": { + "title": "KSampler" + } + }, + "4": { + "inputs": { + "ckpt_name": "v1-5-pruned-emaonly.ckpt" + }, + "class_type": "CheckpointLoaderSimple", + "_meta": { + "title": "Load Checkpoint" + } + }, + "5": { + "inputs": { + "width": 512, + "height": 512, + "batch_size": 1 + }, + "class_type": "EmptyLatentImage", + "_meta": { + "title": "Empty Latent Image" + } + }, + "6": { + "inputs": { + "text": "beautiful scenery nature glass bottle landscape, , purple galaxy bottle,", + "clip": [ + "4", + 1 + ] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Prompt)" + } + }, + "7": { + "inputs": { + "text": "text, watermark", + "clip": [ + "4", + 1 + ] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Prompt)" + } + }, + "8": { + "inputs": { + "samples": [ + "3", + 0 + ], + "vae": [ + "4", + 2 + ] + }, + "class_type": "VAEDecode", + "_meta": { + "title": "VAE Decode" + } + }, + "9": { + "inputs": { + "filename_prefix": "ComfyUI", + "images": [ + "8", + 0 + ] + }, + "class_type": "SaveImage", + "_meta": { + "title": "Save Image" + } + } +} \ No newline at end of file diff --git a/packages/app/src/app/api/resolve/providers/comfyui/index.ts b/packages/app/src/app/api/resolve/providers/comfyui/index.ts index 7bb9a0b2..5d1c0d7e 100644 --- a/packages/app/src/app/api/resolve/providers/comfyui/index.ts +++ b/packages/app/src/app/api/resolve/providers/comfyui/index.ts @@ -1,24 +1,13 @@ import { ResolveRequest } from '@aitube/clapper-services' -import { - ClapAssetSource, - ClapSegmentCategory, - ClapSegmentStatus, - generateSeed, - getClapAssetSourceType, -} from '@aitube/clap' +import { ClapAssetSource, ClapSegmentCategory } from '@aitube/clap' import { TimelineSegment } from '@aitube/timeline' -import { - BasicCredentials, - CallWrapper, - ComfyApi, - PromptBuilder, - TSamplerName, - TSchedulerName, -} from '@saintno/comfyui-sdk' - -import { getWorkflowInputValues } from '../getWorkflowInputValues' +import { BasicCredentials, CallWrapper, ComfyApi } from '@saintno/comfyui-sdk' import { decodeOutput } from '@/lib/utils/decodeOutput' -import { ComfyUIWorkflowApiUtils } from './utils' +import { + ClapperComfyUiInputIds, + ComfyUIWorkflowApiGraph, + createPromptBuilder, +} from './utils' export async function resolveSegment( request: ResolveRequest @@ -51,64 +40,60 @@ export async function resolveSegment( ).init() if (request.segment.category === ClapSegmentCategory.STORYBOARD) { - const comfyApiWorkflow = JSON.parse( - request.settings.imageGenerationWorkflow.data - ) + const imageGenerationWorkflow = request.settings.imageGenerationWorkflow + + if (!imageGenerationWorkflow.inputValues[ClapperComfyUiInputIds.PROMPT]) { + throw new Error( + `This workflow doesn't seem to have an input required by Clapper (e.g. a node with an input called "prompt")` + ) + } - const txt2ImgPrompt = new ComfyUIWorkflowApiUtils( - comfyApiWorkflow - ).createPromptBuilder() - - const workflow = txt2ImgPrompt - // TODO: this mapping should be detect/filled automatically (see line 86) - .input('ckpt_name', 'SDXL/realvisxlV40_v40LightningBakedvae.safetensors') - .input('seed', generateSeed()) - .input('steps', 6) - .input('cfg', 1) - .input('sampler_name', 'dpmpp_2m_sde_gpu') - .input('scheduler', 'sgm_uniform') - .input('width', request.meta.width) - .input('height', request.meta.height) - .input('batch_size', 1) - .input('positive', request.prompts.image.positive) - - // for the moment we only have non-working "mock" sample code, - // to fully implement the comfyui client, we need to work on a system - // to automatically detect the architecture of the workflow, its parameters, - // the default values, and fill them - // - // to make things easier, we are going to assume that the ClapWorkflow object - // is 100% correctly defined, and that we can rely on `inputFields` and `inputValues` - // - // that way, the responsibility of automatically identifying the inputs from a raw JSON workflow - // (eg. coming from OpenArt.ai) will be done by a separate pre-processing code - - const inputFields = - request.settings.imageGenerationWorkflow.inputFields || [] - - // since this is a random "wild" workflow, it is possible - // that the field name is a bit different - // we try to look into the workflow input fields - // to find the best match - const promptFields = [ - inputFields.find((f) => f.id === 'prompt'), // exactMatch, - inputFields.find((f) => f.id.includes('prompt')), // similarName, - inputFields.find((f) => f.type === 'string'), // similarType - ].filter((x) => typeof x !== 'undefined') - - const promptField = promptFields[0] - if (!promptField) { + if (!imageGenerationWorkflow.inputValues[ClapperComfyUiInputIds.OUTPUT]) { throw new Error( - `this workflow doesn't seem to have a parameter called "prompt"` + `This workflow doesn't seem to have a node output required by Clapper (e.g. a 'Save Image' node)` ) } - // TODO: modify the serialized workflow payload - // to inject our params: - // ...getWorkflowInputValues(request.settings.imageGenerationWorkflow), - // [promptField.id]: request.prompts.image.positive, + const comfyApiWorkflowPromptBuilder = createPromptBuilder( + ComfyUIWorkflowApiGraph.fromString(imageGenerationWorkflow.data) + ) + + const { inputFields, inputValues } = + request.settings.imageGenerationWorkflow + + inputFields.forEach((inputField) => { + comfyApiWorkflowPromptBuilder.input( + inputField.id, + inputValues[inputField.id] + ) + }) + + // Set main inputs + comfyApiWorkflowPromptBuilder + .input( + (inputValues[ClapperComfyUiInputIds.PROMPT] as any).id, + request.prompts.image.positive + ) + .input( + (inputValues[ClapperComfyUiInputIds.NEGATIVE_PROMPT] as any).id, + request.prompts.image.negative + ) + .input( + (inputValues[ClapperComfyUiInputIds.WIDTH] as any).id, + request.meta.width + ) + .input( + (inputValues[ClapperComfyUiInputIds.HEIGHT] as any).id, + request.meta.height + ) + + // Set output + comfyApiWorkflowPromptBuilder.setOutputNode( + ClapperComfyUiInputIds.OUTPUT, + (inputValues[ClapperComfyUiInputIds.OUTPUT] as any).id + ) - const pipeline = new CallWrapper(api, workflow) + const pipeline = new CallWrapper(api, comfyApiWorkflowPromptBuilder) .onPending(() => console.log('Task is pending')) .onStart(() => console.log('Task is started')) .onPreview((blob) => console.log(blob)) @@ -126,8 +111,8 @@ export async function resolveSegment( throw new Error(`failed to run the pipeline (no output)`) } - const imagePaths = rawOutput.output?.images.map((img: any) => - api.getPathImage(img) + const imagePaths = rawOutput[ClapperComfyUiInputIds.OUTPUT]?.images.map( + (img: any) => api.getPathImage(img) ) console.log(`imagePaths:`, imagePaths) diff --git a/packages/app/src/app/api/resolve/providers/comfyui/utils.spec.ts b/packages/app/src/app/api/resolve/providers/comfyui/utils.spec.ts index 51097c9a..c637db6f 100644 --- a/packages/app/src/app/api/resolve/providers/comfyui/utils.spec.ts +++ b/packages/app/src/app/api/resolve/providers/comfyui/utils.spec.ts @@ -1,5 +1,11 @@ import { expect, test } from 'vitest' -import { ComfyUIWorkflowApiUtils } from './utils' +import { + ClapperComfyUiInputIds, + ComfyUIWorkflowApiGraph, + createPromptBuilder, + findNegativePromptInputsFromWorkflow, + findPromptInputsFromWorkflow, +} from './utils' // Default workflow used by ComfyUI, downloaded for API const workflowRaw = { @@ -166,7 +172,7 @@ const workflowRawWithTokens = { } test('should return all nodes that have inputs', () => { - const nodesWithInputs = new ComfyUIWorkflowApiUtils( + const nodesWithInputs = new ComfyUIWorkflowApiGraph( workflowRaw ).getNodesWithInputs() @@ -186,7 +192,7 @@ test('should return all nodes that have inputs', () => { }) test('should return the correct output node', () => { - const outputNode = new ComfyUIWorkflowApiUtils(workflowRaw).getOutputNode() + const outputNode = new ComfyUIWorkflowApiGraph(workflowRaw).getOutputNode() expect(outputNode).toEqual({ id: '9', @@ -202,7 +208,7 @@ test('should return the correct output node', () => { }) test('should build the correct graph from the workflow', () => { - const { adjList, dependencyList, inDegree } = new ComfyUIWorkflowApiUtils( + const { adjList, dependencyList, inDegree } = new ComfyUIWorkflowApiGraph( workflowRaw ).getGraphData() @@ -225,43 +231,43 @@ test('should build the correct graph from the workflow', () => { }) test('should return the correct inputs by node id', () => { - const workflow = new ComfyUIWorkflowApiUtils(workflowRaw) + const workflow = new ComfyUIWorkflowApiGraph(workflowRaw) expect(workflow.getInputsByNodeId('3')).toEqual([ { type: 'number', name: 'seed', value: 156680208700286, - key: '3.inputs.seed', + id: '3.inputs.seed', nodeId: '3', }, { type: 'number', name: 'steps', value: 20, - key: '3.inputs.steps', + id: '3.inputs.steps', nodeId: '3', }, - { type: 'number', name: 'cfg', value: 8, key: '3.inputs.cfg', nodeId: '3' }, + { type: 'number', name: 'cfg', value: 8, id: '3.inputs.cfg', nodeId: '3' }, { type: 'string', name: 'sampler_name', value: 'euler', - key: '3.inputs.sampler_name', + id: '3.inputs.sampler_name', nodeId: '3', }, { type: 'string', name: 'scheduler', value: 'normal', - key: '3.inputs.scheduler', + id: '3.inputs.scheduler', nodeId: '3', }, { type: 'number', name: 'denoise', value: 1, - key: '3.inputs.denoise', + id: '3.inputs.denoise', nodeId: '3', }, ]) @@ -271,7 +277,7 @@ test('should return the correct inputs by node id', () => { type: 'string', name: 'ckpt_name', value: 'v1-5-pruned-emaonly.ckpt', - key: '4.inputs.ckpt_name', + id: '4.inputs.ckpt_name', nodeId: '4', }, ]) @@ -281,21 +287,21 @@ test('should return the correct inputs by node id', () => { type: 'number', name: 'width', value: 512, - key: '5.inputs.width', + id: '5.inputs.width', nodeId: '5', }, { type: 'number', name: 'height', value: 512, - key: '5.inputs.height', + id: '5.inputs.height', nodeId: '5', }, { type: 'number', name: 'batch_size', value: 1, - key: '5.inputs.batch_size', + id: '5.inputs.batch_size', nodeId: '5', }, ]) @@ -306,7 +312,7 @@ test('should return the correct inputs by node id', () => { name: 'text', value: 'beautiful scenery nature glass bottle landscape, , purple galaxy bottle,', - key: '6.inputs.text', + id: '6.inputs.text', nodeId: '6', }, ]) @@ -315,106 +321,176 @@ test('should return the correct inputs by node id', () => { expect(nonExistentNodeInputs).toBeNull() }) -test('should detect the correct main inputs', () => { - const mainInputs = new ComfyUIWorkflowApiUtils(workflowRaw).detectMainInputs() +test('should detect the correct positive and negative prompt inputs', () => { + const workflow = new ComfyUIWorkflowApiGraph(workflowRaw) + const positivePromptInput = findPromptInputsFromWorkflow(workflow) + const negativePromptInput = findNegativePromptInputsFromWorkflow(workflow) - expect(mainInputs).toEqual([ + expect(positivePromptInput).toEqual([ { + id: '6.inputs.text', nodeId: '6', name: 'text', + type: 'string', value: 'beautiful scenery nature glass bottle landscape, , purple galaxy bottle,', }, - { nodeId: '7', name: 'text', value: 'text, watermark' }, ]) - expect(mainInputs).not.toEqual( - expect.arrayContaining([ - expect.objectContaining({ name: 'width' }), - expect.objectContaining({ name: 'cfg' }), - ]) - ) -}) - -test('should detect the correct positive and negative prompt inputs', () => { - const workflow = new ComfyUIWorkflowApiUtils(workflowRaw) - const positivePrompts = workflow.detectPositivePromptInput() - const negativePrompts = workflow.detectNegativePromptInput() - - expect(positivePrompts).toEqual([ + expect(negativePromptInput).toEqual([ { - nodeId: '6', + id: '7.inputs.text', + nodeId: '7', name: 'text', - type: 'positive', - value: - 'beautiful scenery nature glass bottle landscape, , purple galaxy bottle,', + type: 'string', + value: 'text, watermark', }, ]) - expect(negativePrompts).toEqual([ - { nodeId: '7', name: 'text', type: 'negative', value: 'text, watermark' }, - ]) - - expect(positivePrompts).not.toEqual( + expect(positivePromptInput).not.toEqual( expect.arrayContaining([expect.objectContaining({ name: 'steps' })]) ) - expect(negativePrompts).not.toEqual( + + expect(negativePromptInput).not.toEqual( expect.arrayContaining([expect.objectContaining({ name: 'cfg' })]) ) }) test('should detect the correct positive and negative prompt inputs using clapper tokens', () => { - const workflow = new ComfyUIWorkflowApiUtils(workflowRawWithTokens) - const positivePrompts = workflow.detectPositivePromptInput() - const negativePrompts = workflow.detectNegativePromptInput() + const workflow = new ComfyUIWorkflowApiGraph(workflowRawWithTokens) + const positivePromptInput = findPromptInputsFromWorkflow(workflow) + const negativePromptInput = findNegativePromptInputsFromWorkflow(workflow) - expect(positivePrompts).toEqual([ + expect(positivePromptInput).toEqual([ { + id: '6.inputs.text', nodeId: '6', - type: 'positive', + type: 'string', name: 'text', value: '@clapper/prompt', }, ]) - expect(negativePrompts).toEqual([ - { nodeId: '7', type: 'negative', name: 'text', value: '@clapper/negative' }, + expect(negativePromptInput).toEqual([ + { + id: '7.inputs.text', + nodeId: '7', + type: 'string', + name: 'text', + value: '@clapper/negative', + }, ]) - expect(positivePrompts).not.toEqual( + expect(positivePromptInput).not.toEqual( expect.arrayContaining([expect.objectContaining({ name: 'steps' })]) ) - expect(negativePrompts).not.toEqual( + expect(negativePromptInput).not.toEqual( expect.arrayContaining([expect.objectContaining({ name: 'cfg' })]) ) }) +test('should correctly search workflow inputs', () => { + const workflow = new ComfyUIWorkflowApiGraph(workflowRaw) + expect( + workflow.findInput({ + name: 'text', + }) + ).toEqual([ + { + id: '6.inputs.text', + nodeId: '6', + name: 'text', + type: 'string', + value: + 'beautiful scenery nature glass bottle landscape, , purple galaxy bottle,', + }, + { + id: '7.inputs.text', + nodeId: '7', + name: 'text', + type: 'string', + value: 'text, watermark', + }, + ]) + expect( + workflow.findInput({ + name: 'text', + type: 'string', + nodeOutputToNodeInput: 'positive', + }) + ).toEqual([ + { + id: '6.inputs.text', + nodeId: '6', + name: 'text', + type: 'string', + value: + 'beautiful scenery nature glass bottle landscape, , purple galaxy bottle,', + }, + ]) + expect( + workflow.findInput({ + name: /tex.*/, + type: /str.*/, + nodeOutputToNodeInput: /posi.*/, + }) + ).toEqual([ + { + id: '6.inputs.text', + nodeId: '6', + name: 'text', + type: 'string', + value: + 'beautiful scenery nature glass bottle landscape, , purple galaxy bottle,', + }, + ]) +}) + test('should create the PromptBuilder', () => { - const promptBuilder = new ComfyUIWorkflowApiUtils( - workflowRaw - ).createPromptBuilder() + const promptBuilder = createPromptBuilder( + new ComfyUIWorkflowApiGraph(workflowRaw) + ) expect(promptBuilder.mapOutputKeys).toEqual({ - output: '9', + [ClapperComfyUiInputIds.OUTPUT]: '9', }) expect(promptBuilder.mapInputKeys).toEqual({ - seed: '3.inputs.seed', - steps: '3.inputs.steps', - cfg: '3.inputs.cfg', - sampler_name: '3.inputs.sampler_name', - scheduler: '3.inputs.scheduler', - denoise: '3.inputs.denoise', - ckpt_name: '4.inputs.ckpt_name', - width: '5.inputs.width', - height: '5.inputs.height', - batch_size: '5.inputs.batch_size', - text: '7.inputs.text', - filename_prefix: '9.inputs.filename_prefix', - positive: '6.inputs.text', - negative: '7.inputs.text', + '3.inputs.seed': '3.inputs.seed', + '3.inputs.steps': '3.inputs.steps', + '3.inputs.cfg': '3.inputs.cfg', + '3.inputs.sampler_name': '3.inputs.sampler_name', + '3.inputs.scheduler': '3.inputs.scheduler', + '3.inputs.denoise': '3.inputs.denoise', + '4.inputs.ckpt_name': '4.inputs.ckpt_name', + '5.inputs.width': '5.inputs.width', + '5.inputs.height': '5.inputs.height', + '5.inputs.batch_size': '5.inputs.batch_size', + '7.inputs.text': '7.inputs.text', + '9.inputs.filename_prefix': '9.inputs.filename_prefix', + '6.inputs.text': '6.inputs.text', + [ClapperComfyUiInputIds.PROMPT]: ClapperComfyUiInputIds.PROMPT, + [ClapperComfyUiInputIds.NEGATIVE_PROMPT]: + ClapperComfyUiInputIds.NEGATIVE_PROMPT, + [ClapperComfyUiInputIds.WIDTH]: ClapperComfyUiInputIds.WIDTH, + [ClapperComfyUiInputIds.HEIGHT]: ClapperComfyUiInputIds.HEIGHT, }) expect(promptBuilder.prompt).toEqual(workflowRaw) }) +test('should convert correctly the workflow to string', () => { + const workflow = new ComfyUIWorkflowApiGraph(workflowRaw) + expect(workflow.toJson()).toEqual(workflowRaw) +}) + +test('should edit correctly an input of the workflow', () => { + const workflow = new ComfyUIWorkflowApiGraph(workflowRaw) + workflow.setInputValue('3.inputs.seed', 1121) + workflowRaw['3'].inputs.seed = 1111 + expect(workflow.toJson()).not.toEqual(workflowRaw) + workflow.setInputValue('3.inputs.seed', 3333) + workflowRaw['3'].inputs.seed = 3333 + expect(workflow.toJson()).toEqual(workflowRaw) +}) + /** * Error handling */ @@ -435,7 +511,7 @@ const workflowRawWithCycles = { test('should fail if workflow has cycles', () => { expect(() => { - new ComfyUIWorkflowApiUtils(workflowRawWithCycles) + new ComfyUIWorkflowApiGraph(workflowRawWithCycles) }).toThrow( 'The provided workflow has cycles, impossible to get the output node.' ) diff --git a/packages/app/src/app/api/resolve/providers/comfyui/utils.ts b/packages/app/src/app/api/resolve/providers/comfyui/utils.ts index 86d31c71..0d0b6347 100644 --- a/packages/app/src/app/api/resolve/providers/comfyui/utils.ts +++ b/packages/app/src/app/api/resolve/providers/comfyui/utils.ts @@ -1,4 +1,23 @@ +import { + ClapInputCategory, + ClapInputField, + ClapInputFields, + ClapInputValues, + ClapWorkflow, + ClapWorkflowCategory, + ClapWorkflowEngine, + ClapWorkflowProvider, +} from '@aitube/clap' import { PromptBuilder } from '@saintno/comfyui-sdk' +import unionBy from 'lodash/unionBy' + +export enum ClapperComfyUiInputIds { + PROMPT = '@clapper/prompt', + NEGATIVE_PROMPT = '@clapper/negative/prompt', + WIDTH = '@clapper/width', + HEIGHT = '@clapper/height', + OUTPUT = '@clapper/output', +} type NodeRawData = { inputs?: Record @@ -12,39 +31,114 @@ type NodeData = NodeRawData & { id: string } -type ComfyUIWorkflowApiRaw = Record +type ComfyUIWorkflowApiJson = Record type INPUT_TYPES = 'string' | 'number' -type NodeInput = { +export type ComfyUiWorkflowApiNodeInput = { + id: string + nodeId: string // Infered primitive type of the input based on its value type: INPUT_TYPES name: string value: any - key: string - nodeId: string } -type PromptClapperInput = { - // Infered clapper input type based on input value, input node relationships, etc - type?: 'positive' | 'negative' - nodeId: string - name: string - value: any +export class ComfyUIWorkflowApiGraphEdge { + source: ComfyUIWorkflowApiGraphNode + target: ComfyUIWorkflowApiGraphNode + relationship: string + metadata?: any + + constructor( + source: ComfyUIWorkflowApiGraphNode, + target: ComfyUIWorkflowApiGraphNode, + relationship: string, + metadata?: any + ) { + this.source = source + this.target = target + this.relationship = relationship + this.metadata = metadata + } +} + +export class ComfyUIWorkflowApiGraphNode { + id: string + inputs?: Record = {} + _meta?: Record = {} + class_type?: string = '' + outboundEdges: ComfyUIWorkflowApiGraphEdge[] = [] + inboundEdges: ComfyUIWorkflowApiGraphEdge[] = [] + + constructor(id: string, inputs?: Record, meta?: any) { + this.id = id + } + + addOutboundEdge( + targetNode: ComfyUIWorkflowApiGraphNode, + relationship: string, + metadata?: any + ) { + const edge = new ComfyUIWorkflowApiGraphEdge( + this, + targetNode, + relationship, + metadata + ) + this.outboundEdges.push(edge) + targetNode.inboundEdges.push(edge) + } + + addInboundEdge( + sourceNode: ComfyUIWorkflowApiGraphNode, + relationship: string, + metadata?: any + ) { + const edge = new ComfyUIWorkflowApiGraphEdge( + sourceNode, + this, + relationship, + metadata + ) + this.inboundEdges.push(edge) + sourceNode.outboundEdges.push(edge) + } + + getOutboundEdges(): ComfyUIWorkflowApiGraphEdge[] { + return this.outboundEdges + } + + getInboundEdges(): ComfyUIWorkflowApiGraphEdge[] { + return this.inboundEdges + } + + toJson(): Record { + return structuredClone({ + inputs: this.inputs, + class_type: this.class_type, + _meta: this._meta, + }) + } } /** * Utils to query ComfyUI workflow-api nodes data */ -export class ComfyUIWorkflowApiUtils { - private workflow: ComfyUIWorkflowApiRaw - private adjList: Record - private dependencyList: Record - private dependantList: Record - private inDegree: Record - - constructor(workflow: ComfyUIWorkflowApiRaw) { - this.workflow = workflow +export class ComfyUIWorkflowApiGraph { + private json: ComfyUIWorkflowApiJson + private nodes: Record = {} + private adjList: Record = {} + private dependencyList: Record< + string, + { from: string; inputName: string }[] + > = {} + private dependantList: Record = + {} + private inDegree: Record = {} + + constructor(workflow: ComfyUIWorkflowApiJson) { + this.json = structuredClone(workflow) const { adjList, dependencyList, dependantList, inDegree } = this.buildGraphData() this.adjList = adjList @@ -76,25 +170,29 @@ export class ComfyUIWorkflowApiUtils { {} const inDegree: Record = {} - for (const nodeId of Object.keys(this.workflow)) { + for (const nodeId of Object.keys(this.json)) { + const node = new ComfyUIWorkflowApiGraphNode(nodeId) + node.inputs = this.json[nodeId].inputs + node._meta = this.json[nodeId]._meta + node.class_type = this.json[nodeId].class_type + this.nodes[nodeId] = node + adjList[nodeId] = [] dependencyList[nodeId] = [] dependantList[nodeId] = [] inDegree[nodeId] = 0 } - for (const [nodeId, nodeData] of Object.entries(this.workflow)) { - const completeNodeData: NodeData = { id: nodeId, ...nodeData } - if (completeNodeData.inputs) { - for (const [inputName, value] of Object.entries( - completeNodeData.inputs - )) { + for (const node of Object.values(this.nodes)) { + if (node.inputs) { + for (const [inputName, value] of Object.entries(node.inputs)) { if (Array.isArray(value)) { - const dependency = value[0] as string - adjList[dependency].push(nodeId) - dependencyList[nodeId].push({ from: dependency, inputName }) - dependantList[dependency].push({ to: nodeId, inputName }) - inDegree[nodeId] += 1 + const dependencyNodeId = value[0] as string + adjList[dependencyNodeId].push(node.id) + dependencyList[node.id].push({ from: dependencyNodeId, inputName }) + node.addInboundEdge(this.nodes[dependencyNodeId], inputName) + dependantList[dependencyNodeId].push({ to: node.id, inputName }) + inDegree[node.id] += 1 } } } @@ -103,6 +201,9 @@ export class ComfyUIWorkflowApiUtils { return { adjList, dependencyList, dependantList, inDegree } } + /** + * Check for corrupted workflows with loops + */ private detectCycles(): boolean { const visited: Record = {} const recursionStack: Record = {} @@ -140,29 +241,36 @@ export class ComfyUIWorkflowApiUtils { /** * Get all nodes that have inputs. */ - getNodesWithInputs(): NodeData[] { - const nodesWithInputs: NodeData[] = [] - for (const [nodeId, nodeData] of Object.entries(this.workflow)) { - if (nodeData.inputs && Object.keys(nodeData.inputs).length > 0) { - const completeNodeData: NodeData = { id: nodeId, ...nodeData } - nodesWithInputs.push(completeNodeData) + getNodesWithInputs(): ComfyUIWorkflowApiGraphNode[] { + const nodesWithInputs: ComfyUIWorkflowApiGraphNode[] = [] + for (const node of Object.values(this.nodes)) { + if (node.inputs && Object.keys(node.inputs).length > 0) { + nodesWithInputs.push(node) } } return nodesWithInputs } + getNodes(): ComfyUIWorkflowApiGraphNode[] { + return Object.values(structuredClone(this.nodes)) + } + + getNodesDict(): Record { + return structuredClone(this.nodes) + } + /** - * Get all inputs in the workflow. + * Returns a copy of all inputs in the workflow. */ - getInputs(): NodeInput[] { + getInputs(): Record { const nodesWithInputs = this.getNodesWithInputs() - let inputs: any[] = [] + const inputs = {} for (const node of nodesWithInputs) { const inputSchemas = this.getInputsByNodeId(node.id) - if (inputSchemas?.length) { - inputs = inputs.concat(...inputSchemas) - } + inputSchemas?.forEach((inputSchema) => { + inputs[`${inputSchema.nodeId}.inputs.${inputSchema.name}`] = inputSchema + }) } return inputs @@ -196,9 +304,9 @@ export class ComfyUIWorkflowApiUtils { // Last node in sortedOrder is the output node // TODO: handle multiple outputs - if (sortedOrder.length === Object.keys(this.workflow).length) { + if (sortedOrder.length === Object.keys(this.json).length) { const outputNodeId = sortedOrder[sortedOrder.length - 1] - return { id: outputNodeId, ...this.workflow[outputNodeId] } + return { id: outputNodeId, ...this.json[outputNodeId] } } else { // If there are cycles, fail throw new Error( @@ -212,13 +320,13 @@ export class ComfyUIWorkflowApiUtils { * Ignore input connections (e.g. inputs with value ['3', 0]) * @param nodeId the id of the node */ - getInputsByNodeId(nodeId: string): NodeInput[] | null { - const nodeData = this.workflow[nodeId] + getInputsByNodeId(nodeId: string): ComfyUiWorkflowApiNodeInput[] | null { + const nodeData = this.nodes[nodeId] if (!nodeData || !nodeData.inputs) { return null } - const inputs: NodeInput[] = [] + const inputs: ComfyUiWorkflowApiNodeInput[] = [] for (const [name, value] of Object.entries(nodeData.inputs)) { if (Array.isArray(value)) continue @@ -231,7 +339,7 @@ export class ComfyUIWorkflowApiUtils { type: inputType, name: name, value: value, - key: `${nodeId}.inputs.${name}`, + id: `${nodeId}.inputs.${name}`, nodeId, }) } @@ -240,135 +348,444 @@ export class ComfyUIWorkflowApiUtils { } /** - * Search for the main inputs for Clapper - * e.g. prompt, negative prompt + * A simple search of workflow inputs + * @param query */ - detectMainInputs(): PromptClapperInput[] { - const nodesWithInputs = this.getNodesWithInputs() - const mainInputs: PromptClapperInput[] = [] + findInput(query: { + // By type of node input + type?: string | RegExp + // By name of node input + name?: string | RegExp + // If any output of the node containing the input + // is targeting another node's input with the given + // name + nodeOutputToNodeInput?: string | RegExp + // By value + value?: (value) => boolean + }): ComfyUiWorkflowApiNodeInput[] { + const inputs = this.getInputs() - for (const node of nodesWithInputs) { - const { id: nodeId, inputs, class_type, _meta } = node - const nodeInputs = this.getInputsByNodeId(node.id) - - if (nodeInputs) { - for (const nodeInput of nodeInputs) { - // Based on the type or input name - const isStringInput = nodeInput.type === 'string' - const nameContainsTextOrPrompt = - nodeInput.name.includes('text') || nodeInput.name.includes('prompt') - // Based on the node type - const classIsCLIPTextEncode = class_type === 'CLIPTextEncode' - // Based on the node title - const titleContainsPrompt = _meta?.title - ?.toLowerCase() - .includes('prompt') - // Based on Clapper string tokens - const hasClapperTokens = - isStringInput && - nodeInput.value?.toLowerCase().includes('@clapper/') - - if ( - (isStringInput && nameContainsTextOrPrompt) || - classIsCLIPTextEncode || - titleContainsPrompt || - hasClapperTokens - ) { - mainInputs.push({ - name: nodeInput.name, - value: nodeInput.value, - nodeId: nodeInput.nodeId, - }) - } - } + // Helper function to match string or RegExp + const matches = ( + value: string, + query: string | RegExp | undefined + ): boolean => { + if (!query) return true + if (typeof query === 'string') { + return value === query + } else if (query instanceof RegExp) { + return query.test(value) } + return false } - return mainInputs + return Object.values(inputs).filter( + (input) => + (matches(input.type, query.type) && + matches(input.name, query.name) && + (!query.nodeOutputToNodeInput || + this.nodes[input.nodeId].outboundEdges.some((edge) => + matches(edge.relationship, query.nodeOutputToNodeInput) + )) && + !query.value) || + query.value?.(input.value) + ) } - /** - * Detect positive prompt inputs in the workflow - */ - detectPositivePromptInput(): PromptClapperInput[] { - const mainInputs = this.detectMainInputs() - const positivePromptInputs = mainInputs - .filter((input) => { - const deps = this.dependantList[input.nodeId] - return deps.some((dep) => dep.inputName === 'positive') - }) - .map((input) => { - input.type = 'positive' - return input - }) + setInputValue( + inputKey: string, + value: any, + options?: { + ignoreErrors?: boolean + } + ) { + const inputs = this.getInputs() + if (!options?.ignoreErrors && !inputs[inputKey]) { + throw new Error("Input doesn't exist in the workflow") + } + const input = inputs[inputKey] + if (!input) return + if (!this.nodes[input.nodeId].inputs) this.nodes[input.nodeId].inputs = {} + this.nodes[input.nodeId].inputs![input.name] = value + } - return positivePromptInputs + toJson(): Record { + const nodes = this.nodes + const nodesJson = {} + for (const key of Object.keys(nodes)) { + nodesJson[key] = nodes[key].toJson() + } + return nodesJson } - /** - * Detect negative prompt inputs in the workflow - */ - detectNegativePromptInput(): PromptClapperInput[] { - const mainInputs = this.detectMainInputs() - const negativePromptInputs = mainInputs - .filter((input) => { - const deps = this.dependantList[input.nodeId] - return deps.some((dep) => dep.inputName === 'negative') - }) - .map((input) => { - input.type = 'negative' - return input - }) + toString() { + return JSON.stringify(this.toJson()) + } - return negativePromptInputs + static fromString( + workflowString: string | undefined + ): ComfyUIWorkflowApiGraph { + if (!workflowString) throw new Error('Invalid workflow.') + const workflowRaw = JSON.parse(workflowString) + return new ComfyUIWorkflowApiGraph(workflowRaw) } - /** - * Takes a workflow and converts it to PromptBuilder - */ - createPromptBuilder(): PromptBuilder { - const positivePrompts = this.detectPositivePromptInput() - const negativePrompts = this.detectNegativePromptInput() - const inputs = this.getInputs() - const outputNode = this.getOutputNode() + static isValidWorkflow(workflowString: string | undefined): boolean { + try { + ComfyUIWorkflowApiGraph.fromString(workflowString) + return true + } catch {} + return false + } +} - const promptBuilder = new PromptBuilder( - this.workflow, - inputs.map((input) => input.name), - ['output'] - ) +/** + * Shortcut methods to query Clapper related data from a workflow + * @param workflow + * @returns + */ +export function findPromptInputsFromWorkflow( + workflow: ComfyUIWorkflowApiGraph +): ComfyUiWorkflowApiNodeInput[] { + return unionBy( + workflow.findInput({ + name: /.*(positive).*/i, + type: 'string', + }), + workflow.findInput({ + name: /.*(text|prompt).*/i, + type: 'string', + nodeOutputToNodeInput: /.*positive.*/i, + }), + workflow.findInput({ + value: (value) => /.*\@clapper\/prompt.*/i.test(value), + }), + 'id' + ) +} - const processed: Record = {} +export function findNegativePromptInputsFromWorkflow( + workflow: ComfyUIWorkflowApiGraph +) { + return unionBy( + workflow.findInput({ + name: /.*(text|prompt|negative).*/i, + type: 'string', + nodeOutputToNodeInput: /.*negative.*/i, + }), + workflow.findInput({ + value: (value) => /.*\@clapper\/prompt_negative.*/i.test(value), + }), + 'id' + ) +} - positivePrompts.forEach((input) => { - processed['positive'] = true - promptBuilder.setInputNode( - 'positive', - `${input.nodeId}.inputs.${input.name}` - ) - promptBuilder.input('positive', input.value) - }) +export function findWidthInputsFromWorkflow(workflow: ComfyUIWorkflowApiGraph) { + return unionBy( + workflow.findInput({ + name: /.*(width).*/i, + type: 'number', + }), + workflow.findInput({ + type: 'number', + nodeOutputToNodeInput: /.*width.*/i, + }), + workflow.findInput({ + value: (value) => /.*\@clapper\/width.*/i.test(value), + }), + 'id' + ) +} - negativePrompts.forEach((input) => { - processed['negative'] = true - promptBuilder.setInputNode( - 'negative', - `${input.nodeId}.inputs.${input.name}` - ) - promptBuilder.input('negative', input.value) - }) +export function findHeightInputsFromWorkflow( + workflow: ComfyUIWorkflowApiGraph +) { + return unionBy( + workflow.findInput({ + name: /.*(height).*/i, + type: 'number', + }), + workflow.findInput({ + type: 'number', + nodeOutputToNodeInput: /.*height.*/i, + }), + workflow.findInput({ + value: (value) => /.*\@clapper\/height.*/i.test(value), + }), + 'id' + ) +} - inputs.forEach((input) => { - promptBuilder.setInputNode(input.name, input.key) - if (!processed[input.key]) { - promptBuilder.input(input.name, input.value) - } - }) +/** + * Returns input fields / input values required by ComfyUi + * @param workflow + */ - if (outputNode) { - promptBuilder.setOutputNode('output', outputNode.id) - } +export function getInputsFromComfyUiWorkflow(workflowString: string): { + inputFields: ClapInputFields + inputValues: ClapInputValues +} { + const workflowGraph = ComfyUIWorkflowApiGraph.fromString(workflowString) + + const nodes = workflowGraph.getNodes() + const textInputs = workflowGraph.findInput({ + type: 'string', + name: /.*(text|prompt).*/, + }) + const dimensionInputs = workflowGraph.findInput({ + type: 'number', + name: /.*(width|height).*/, + }) + const promptNodeInputs = findPromptInputsFromWorkflow(workflowGraph) + const negativePromptNodeInputs = + findNegativePromptInputsFromWorkflow(workflowGraph) + const widthNodeInputs = findWidthInputsFromWorkflow(workflowGraph) + const heightNodeInputs = findHeightInputsFromWorkflow(workflowGraph) + const outputNode = workflowGraph.getOutputNode() + + const inputValues = { + [ClapperComfyUiInputIds.PROMPT]: { + id: promptNodeInputs?.[0].id, + label: `${promptNodeInputs?.[0].id} (from node ${promptNodeInputs?.[0].nodeId})`, + }, + [ClapperComfyUiInputIds.NEGATIVE_PROMPT]: { + id: negativePromptNodeInputs?.[0].id, + label: `${negativePromptNodeInputs?.[0].id} (from node ${negativePromptNodeInputs?.[0].nodeId})`, + }, + [ClapperComfyUiInputIds.WIDTH]: { + id: widthNodeInputs?.[0].id, + label: `${widthNodeInputs?.[0].id} (from node ${widthNodeInputs?.[0].nodeId})`, + }, + [ClapperComfyUiInputIds.HEIGHT]: { + id: heightNodeInputs?.[0].id, + label: `${heightNodeInputs?.[0].id} (from node ${heightNodeInputs?.[0].nodeId})`, + }, + [ClapperComfyUiInputIds.OUTPUT]: { + id: outputNode?.id, + label: `${outputNode?._meta?.title} (id: ${outputNode?.id})`, + }, + } - return promptBuilder + const inputLabels = { + [ClapperComfyUiInputIds.PROMPT]: 'Prompt node input', + [ClapperComfyUiInputIds.NEGATIVE_PROMPT]: 'Negative prompt node input', + [ClapperComfyUiInputIds.WIDTH]: 'Width node input', + [ClapperComfyUiInputIds.HEIGHT]: 'Height node input', + [ClapperComfyUiInputIds.OUTPUT]: 'Output node', + } + + const inputFields: ClapInputField<{ + options?: { + id: string + label: string + value: any + }[] + mainInput?: any + }>[] = [ + // Required fields that should be available in the workflow, otherwise + // Clapper doesn't know how to input its settings (prompts, dimensions, etc) + { + id: '@clapper/mainInputs', + label: 'Main settings', + type: 'group' as any, + category: ClapInputCategory.UNKNOWN, + description: 'Main inputs', + defaultValue: '', + inputFields: [ + { + id: ClapperComfyUiInputIds.PROMPT, + label: inputLabels[ClapperComfyUiInputIds.PROMPT], + type: 'nodeInput' as any, + category: ClapInputCategory.PROMPT, + description: 'The input where Clapper will put the segment prompt', + defaultValue: '', + metadata: { + options: unionBy(promptNodeInputs, textInputs, 'id').map((p) => ({ + id: p.id, + label: `${p.name} (from node ${p.nodeId})`, + value: p.id, + })), + }, + }, + { + id: ClapperComfyUiInputIds.NEGATIVE_PROMPT, + label: inputLabels[ClapperComfyUiInputIds.NEGATIVE_PROMPT], + type: 'nodeInput' as any, + category: ClapInputCategory.PROMPT, + description: + 'The node input where Clapper will put the segment negative prompt', + defaultValue: '', + metadata: { + options: unionBy(negativePromptNodeInputs, textInputs, 'id').map( + (p) => ({ + id: p.id, + label: `${p.name} (from node ${p.nodeId})`, + value: p.id, + }) + ), + }, + }, + { + id: ClapperComfyUiInputIds.WIDTH, + label: inputLabels[ClapperComfyUiInputIds.WIDTH], + type: 'nodeInput' as any, + category: ClapInputCategory.WIDTH, + description: + 'The node input where Clapper will put the required image width', + defaultValue: '', + metadata: { + options: unionBy(widthNodeInputs, dimensionInputs, 'id').map( + (p) => ({ + id: p.id, + label: `${p.name} (from node ${p.nodeId})`, + value: p.id, + }) + ), + }, + }, + { + id: ClapperComfyUiInputIds.HEIGHT, + label: inputLabels[ClapperComfyUiInputIds.HEIGHT], + type: 'nodeInput' as any, + description: + 'The node input where Clapper will put the required image height', + category: ClapInputCategory.HEIGHT, + defaultValue: 1000, + metadata: { + options: unionBy(heightNodeInputs, dimensionInputs, 'id').map( + (p) => ({ + id: p.id, + label: `${p.name} (from node ${p.nodeId})`, + value: p.id, + }) + ), + }, + }, + { + id: ClapperComfyUiInputIds.OUTPUT, + label: inputLabels[ClapperComfyUiInputIds.OUTPUT], + type: 'node' as any, + category: ClapInputCategory.UNKNOWN, + description: 'The node from which Clapper will get the output image', + defaultValue: '', + metadata: { + options: nodes.map((p) => ({ + id: p.id, + label: `${p._meta?.title || 'Untitled node'} (id: ${p.id})`, + value: p.id, + })), + }, + }, + ], + }, + // Other input fields based on the workflow nodes + { + id: '@clapper/otherInputs', + label: 'Node settings', + type: 'group' as any, + category: ClapInputCategory.UNKNOWN, + description: 'Main inputs', + defaultValue: '', + inputFields: workflowGraph + .getNodesWithInputs() + // Discard nodes with only inputs connections + .filter(({ id }) => workflowGraph.getInputsByNodeId(id)?.length) + .map(({ id, _meta }) => { + return { + id: '@clapper/node/' + id, + label: `${_meta?.title} (id: ${id})`, + type: 'group' as any, + category: ClapInputCategory.UNKNOWN, + description: `Settings for ${_meta?.title}`, + defaultValue: '', + inputFields: workflowGraph.getInputsByNodeId(id)?.map((input) => { + const mainInputKey = Object.keys(inputValues).find( + (key) => inputValues[key]?.id == input.id + ) + inputValues[input.id] = input.value + return { + id: input.id, + label: input.name, + type: input.type as any, + category: ClapInputCategory.UNKNOWN, + description: '', + defaultValue: input.value, + metadata: { + mainInput: inputLabels[mainInputKey as string], + }, + } + }), + } + }), + }, + ] + + return { + inputFields, + inputValues, + } +} + +/** + * Takes a workflow graph and converts it to PromptBuilder + */ +export function createPromptBuilder( + workflowApiGraph: ComfyUIWorkflowApiGraph +): PromptBuilder { + const inputs = workflowApiGraph.getInputs() + const outputNode = workflowApiGraph.getOutputNode() + + const inputKeys = Object.values(inputs) + .map((input) => input.id) + .concat([ + ClapperComfyUiInputIds.PROMPT, + ClapperComfyUiInputIds.NEGATIVE_PROMPT, + ClapperComfyUiInputIds.WIDTH, + ClapperComfyUiInputIds.HEIGHT, + ]) + + const promptBuilder = new PromptBuilder( + workflowApiGraph.toJson(), + inputKeys, + [ClapperComfyUiInputIds.OUTPUT] + ) + + // We don't need proper names for input keys, + // as we just use PromptBuilder for its websocket api + inputKeys.forEach((inputKey) => { + promptBuilder.setInputNode(inputKey, inputKey) + }) + + if (outputNode) { + promptBuilder.setOutputNode(ClapperComfyUiInputIds.OUTPUT, outputNode.id) + } + + return promptBuilder +} + +export function convertComfyUiWorkflowApiToClapWorkflow( + workflowString: string +): ClapWorkflow { + try { + const { inputFields, inputValues } = + getInputsFromComfyUiWorkflow(workflowString) + return { + id: 'comfyui://settings.comfyWorkflowForImage', + label: 'Custom Image Workflow', + description: 'Custom ComfyUI workflow to generate images', + tags: ['custom', 'image generation'], + author: 'You', + thumbnailUrl: '', + nonCommercial: false, + engine: ClapWorkflowEngine.COMFYUI_WORKFLOW, + provider: ClapWorkflowProvider.COMFYUI, + category: ClapWorkflowCategory.IMAGE_GENERATION, + data: workflowString, + schema: '', + inputFields, + inputValues, + } + } catch (e) { + throw e } } diff --git a/packages/app/src/components/forms/FormArea.tsx b/packages/app/src/components/forms/FormArea.tsx index 7d9e23c9..d943b190 100644 --- a/packages/app/src/components/forms/FormArea.tsx +++ b/packages/app/src/components/forms/FormArea.tsx @@ -13,6 +13,8 @@ import { Input } from '../ui/input' import { FormField } from './FormField' import { useTheme } from '@/services' import { Textarea } from '../ui/textarea' +import { MdError } from 'react-icons/md' +import { Tooltip, TooltipContent, TooltipTrigger } from '../ui/tooltip' export function FormArea( { @@ -27,6 +29,7 @@ export function FormArea( onChange, type, rows = 4, + error, ...props }: { label?: ReactNode @@ -41,6 +44,7 @@ export function FormArea( type?: HTMLInputTypeAttribute rows?: number props?: any + error?: string | null } // & Omit, "value" | "defaultValue" | "placeholder" | "type" | "className" | "disabled" | "onChange"> // & ComponentProps @@ -66,11 +70,18 @@ export function FormArea( ) return ( - +