|
| 1 | +import { ResolveRequest } from '@aitube/clapper-services' |
| 2 | +import { |
| 3 | + ClapSegmentCategory, |
| 4 | + ClapSegmentStatus, |
| 5 | + generateSeed, |
| 6 | + getClapAssetSourceType, |
| 7 | +} from '@aitube/clap' |
| 8 | +import { TimelineSegment } from '@aitube/timeline' |
| 9 | +import { |
| 10 | + CallWrapper, |
| 11 | + ComfyApi, |
| 12 | + PromptBuilder, |
| 13 | + TSamplerName, |
| 14 | + TSchedulerName, |
| 15 | +} from '@saintno/comfyui-sdk' |
| 16 | + |
| 17 | +import { getWorkflowInputValues } from '../getWorkflowInputValues' |
| 18 | + |
| 19 | +export async function resolveSegment( |
| 20 | + request: ResolveRequest |
| 21 | +): Promise<TimelineSegment> { |
| 22 | + if (!request.settings.comfyUiClientId) { |
| 23 | + throw new Error(`Missing client id for "ComfyUI"`) |
| 24 | + } |
| 25 | + |
| 26 | + const segment: TimelineSegment = { ...request.segment } |
| 27 | + |
| 28 | + // for API doc please see: |
| 29 | + // https://github.com/tctien342/comfyui-sdk/blob/main/examples/example-t2i.ts |
| 30 | + const api = new ComfyApi( |
| 31 | + request.settings.comfyUiApiUrl || 'http://localhost:8189' |
| 32 | + ).init() |
| 33 | + |
| 34 | + if (request.segment.category === ClapSegmentCategory.STORYBOARD) { |
| 35 | + |
| 36 | + const comfyApiWorkflow = JSON.parse( |
| 37 | + request.settings.imageGenerationWorkflow.data |
| 38 | + ) |
| 39 | + |
| 40 | + const txt2ImgPrompt = new PromptBuilder( |
| 41 | + comfyApiWorkflow, |
| 42 | + // TODO: this list should be detect/filled automatically (see line 86) |
| 43 | + [ |
| 44 | + 'positive', |
| 45 | + 'negative', |
| 46 | + 'checkpoint', |
| 47 | + 'seed', |
| 48 | + 'batch', |
| 49 | + 'step', |
| 50 | + 'cfg', |
| 51 | + 'sampler', |
| 52 | + 'sheduler', |
| 53 | + 'width', |
| 54 | + 'height', |
| 55 | + ], |
| 56 | + // TODO: this list should be detect/filled automatically (see line 86) |
| 57 | + ['images'] |
| 58 | + ) |
| 59 | + // TODO: those input sets should be detect/filled automatically (see line 86) |
| 60 | + .setInputNode('checkpoint', '4.inputs.ckpt_name') |
| 61 | + .setInputNode('seed', '3.inputs.seed') |
| 62 | + .setInputNode('batch', '5.inputs.batch_size') |
| 63 | + .setInputNode('negative', '7.inputs.text') |
| 64 | + .setInputNode('positive', '6.inputs.text') |
| 65 | + .setInputNode('cfg', '3.inputs.cfg') |
| 66 | + .setInputNode('sampler', '3.inputs.sampler_name') |
| 67 | + .setInputNode('sheduler', '3.inputs.scheduler') |
| 68 | + .setInputNode('step', '3.inputs.steps') |
| 69 | + .setInputNode('width', '5.inputs.width') |
| 70 | + .setInputNode('height', '5.inputs.height') |
| 71 | + .setOutputNode('images', '9') |
| 72 | + |
| 73 | + const workflow = txt2ImgPrompt |
| 74 | + // TODO: this mapping should be detect/filled automatically (see line 86) |
| 75 | + .input('checkpoint', 'SDXL/realvisxlV40_v40LightningBakedvae.safetensors') |
| 76 | + .input('seed', generateSeed()) |
| 77 | + .input('step', 6) |
| 78 | + .input('cfg', 1) |
| 79 | + .input<TSamplerName>('sampler', 'dpmpp_2m_sde_gpu') |
| 80 | + .input<TSchedulerName>('sheduler', 'sgm_uniform') |
| 81 | + .input('width', request.meta.width) |
| 82 | + .input('height', request.meta.height) |
| 83 | + .input('batch', 1) |
| 84 | + .input('positive', request.prompts.image.positive) |
| 85 | + |
| 86 | + // for the moment we only have non-working "mock" sample code, |
| 87 | + // to fully implement the comfyui client, we need to work on a system |
| 88 | + // to automatically detect the architecture of the workflow, its parameters, |
| 89 | + // the default values, and fill them |
| 90 | + // |
| 91 | + // to make things easier, we are going to assume that the ClapWorkflow object |
| 92 | + // is 100% correctly defined, and that we can rely on `inputFields` and `inputValues` |
| 93 | + // |
| 94 | + // that way, the responsibility of automatically identifying the inputs from a raw JSON workflow |
| 95 | + // (eg. coming from OpenArt.ai) will be done by a separate pre-processing code |
| 96 | + |
| 97 | + const inputFields = |
| 98 | + request.settings.imageGenerationWorkflow.inputFields || [] |
| 99 | + |
| 100 | + // since this is a random "wild" workflow, it is possible |
| 101 | + // that the field name is a bit different |
| 102 | + // we try to look into the workflow input fields |
| 103 | + // to find the best match |
| 104 | + const promptFields = [ |
| 105 | + inputFields.find((f) => f.id === 'prompt'), // exactMatch, |
| 106 | + inputFields.find((f) => f.id.includes('prompt')), // similarName, |
| 107 | + inputFields.find((f) => f.type === 'string'), // similarType |
| 108 | + ].filter((x) => typeof x !== 'undefined') |
| 109 | + |
| 110 | + const promptField = promptFields[0] |
| 111 | + if (!promptField) { |
| 112 | + throw new Error( |
| 113 | + `this workflow doesn't seem to have a parameter called "prompt"` |
| 114 | + ) |
| 115 | + } |
| 116 | + |
| 117 | + // TODO: modify the serialized workflow payload |
| 118 | + // to inject our params: |
| 119 | + // ...getWorkflowInputValues(request.settings.imageGenerationWorkflow), |
| 120 | + // [promptField.id]: request.prompts.image.positive, |
| 121 | + |
| 122 | + const pipeline = new CallWrapper<typeof workflow>(api, workflow) |
| 123 | + .onPending(() => console.log('Task is pending')) |
| 124 | + .onStart(() => console.log('Task is started')) |
| 125 | + .onPreview((blob) => console.log(blob)) |
| 126 | + .onFinished((data) => { |
| 127 | + console.log( |
| 128 | + data.images?.images.map((img: any) => api.getPathImage(img)) |
| 129 | + ) |
| 130 | + }) |
| 131 | + .onProgress((info) => |
| 132 | + console.log('Processing node', info.node, `${info.value}/${info.max}`) |
| 133 | + ) |
| 134 | + .onFailed((err) => console.log('Task is failed', err)) |
| 135 | + |
| 136 | + const result = await pipeline.run() |
| 137 | + |
| 138 | + console.log(`result:`, result) |
| 139 | + } else { |
| 140 | + throw new Error( |
| 141 | + `Clapper doesn't support ${request.segment.category} generation for provider "ComfyUI". Please open a pull request with (working code) to solve this!` |
| 142 | + ) |
| 143 | + } |
| 144 | + |
| 145 | + return segment |
| 146 | +} |
0 commit comments