From 4e773b88a97f0519fc6421a535abd8ca7da9606a Mon Sep 17 00:00:00 2001 From: Julian Bilcke Date: Sun, 18 Aug 2024 21:10:08 +0200 Subject: [PATCH] fix replicate image width + fix replicate lora --- .../app/api/resolve/providers/falai/index.ts | 41 ++++++++++++++++++- .../api/resolve/providers/replicate/index.ts | 4 ++ packages/app/src/lib/core/constants.ts | 2 +- .../workflows/common/defaultValues.ts | 10 +++++ .../workflows/common/loras/index.ts | 4 ++ .../workflow-editor/workflows/common/types.ts | 3 ++ .../workflows/falai/defaultWorkflows.ts | 6 +-- 7 files changed, 65 insertions(+), 5 deletions(-) diff --git a/packages/app/src/app/api/resolve/providers/falai/index.ts b/packages/app/src/app/api/resolve/providers/falai/index.ts index 133290fc..a1266767 100644 --- a/packages/app/src/app/api/resolve/providers/falai/index.ts +++ b/packages/app/src/app/api/resolve/providers/falai/index.ts @@ -10,6 +10,7 @@ import { } from './types' import { getWorkflowInputValues } from '../getWorkflowInputValues' import { sampleVoice } from '@/lib/core/constants' +import { getWorkflowLora } from '@/services/editors/workflow-editor/workflows/common/loras/getWorkflowLora' export async function resolveSegment( request: ResolveRequest @@ -65,7 +66,7 @@ export async function resolveSegment( workflowValues.width || workflowDefaultValues.width, height: - request.meta.width || + request.meta.height || workflowValues.height || workflowDefaultValues.height, } @@ -100,6 +101,44 @@ export async function resolveSegment( request.settings.censorNotForAllAudiencesContent, }, })) as FalAiImageResponse + } else if (model === 'fal-ai/flux-general') { + // note: this isn't the right place to do this, because maybe the LoRAs are dynamic + const loraModel = getWorkflowLora( + request.settings.imageGenerationWorkflow + ) + if (!loraModel) { + throw new Error(`this model cannot be used without a valid LoRA`) + } + + const url = loraModel.repoOrUrl.startsWith('http') + ? loraModel.repoOrUrl + : `https://huggingface.co/${ + loraModel.repoOrUrl + }/resolve/main/${ + loraModel.fileName || 'lora_weights.safetensors' + }?download=true` + + const prompt = [ + loraModel.trigger, request.prompts.image.positive] + .filter((x) => x) + .join(' ') + + result = (await fal.run(model, { + input: { + prompt, + loras: [ + { + path: url, + scale: 1.0, // between 0 and 4 + }, + ], + image_size: imageSize, + num_images: 1, + sync_mode: true, + enable_safety_checker: + request.settings.censorNotForAllAudiencesContent, + }, + })) as FalAiImageResponse } else { result = (await fal.run(model, { input: { diff --git a/packages/app/src/app/api/resolve/providers/replicate/index.ts b/packages/app/src/app/api/resolve/providers/replicate/index.ts index 7aa2c422..ad5084ad 100644 --- a/packages/app/src/app/api/resolve/providers/replicate/index.ts +++ b/packages/app/src/app/api/resolve/providers/replicate/index.ts @@ -50,6 +50,10 @@ export async function resolveSegment( const loraModel = getWorkflowLora( request.settings.imageGenerationWorkflow ) + + if (!loraModel) { + throw new Error(`this model cannot be used without a valid LoRA`) + } params = { // for some reason this model doesn't support arbitrary width and height, diff --git a/packages/app/src/lib/core/constants.ts b/packages/app/src/lib/core/constants.ts index d7d5ca8c..25c53bcc 100644 --- a/packages/app/src/lib/core/constants.ts +++ b/packages/app/src/lib/core/constants.ts @@ -3,7 +3,7 @@ export const HARD_LIMIT_NB_MAX_ASSETS_TO_GENERATE_IN_PARALLEL = 32 export const APP_NAME = 'Clapper.app' -export const APP_REVISION = '20240816+1424' +export const APP_REVISION = '20240818+2050' export const APP_DOMAIN = 'Clapper.app' export const APP_LINK = 'https://clapper.app' diff --git a/packages/app/src/services/editors/workflow-editor/workflows/common/defaultValues.ts b/packages/app/src/services/editors/workflow-editor/workflows/common/defaultValues.ts index 17b10e43..997a6527 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/common/defaultValues.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/common/defaultValues.ts @@ -137,6 +137,16 @@ export const genericAudio: ClapInputField = { defaultValue: '', } +export const genericLoras: ClapInputField = { + id: 'loras', + label: 'LoRAs', + description: 'Comma-separated LoRAs', + category: ClapInputCategory.LORA, + type: 'string', + allowedValues: [], + defaultValue: '', +} + export const genericInferenceSteps: ClapInputField = { id: 'num_inference_steps', label: 'Inference steps', diff --git a/packages/app/src/services/editors/workflow-editor/workflows/common/loras/index.ts b/packages/app/src/services/editors/workflow-editor/workflows/common/loras/index.ts index 25f4a7f5..3f90fc2f 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/common/loras/index.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/common/loras/index.ts @@ -24,6 +24,8 @@ export const defaultLoraModels: Lora[] = [ 'kodachrome, blurry, realistic, still life, depth of field, scenery, no humans, monochrome, greyscale, traditional media, horizon, looking at viewer, light particles, shadow', repoOrUrl: 'alvdansen/flux-koda', + + fileName: 'araminta_k_flux_koda.safetensors', }, { id: 'lora://hf.co/models/veryVANYA/ps1-style-flux', @@ -47,6 +49,8 @@ trained on 15 gpt4o captioned and adjusted ps1/n64 game screenshots using https: extensions: 'ps1 game screenshot', repoOrUrl: 'veryVANYA/ps1-style-flux', + + fileName: 'ps1_style_flux_v1.safetensors' }, /* { diff --git a/packages/app/src/services/editors/workflow-editor/workflows/common/types.ts b/packages/app/src/services/editors/workflow-editor/workflows/common/types.ts index 0759d5eb..30323550 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/common/types.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/common/types.ts @@ -32,4 +32,7 @@ export type Lora = { // name of the model repository on Hugging Face // or direct URL to the weights repoOrUrl: string + + // optional filename + fileName: string } diff --git a/packages/app/src/services/editors/workflow-editor/workflows/falai/defaultWorkflows.ts b/packages/app/src/services/editors/workflow-editor/workflows/falai/defaultWorkflows.ts index 808686ee..ce394e08 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/falai/defaultWorkflows.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/falai/defaultWorkflows.ts @@ -23,6 +23,7 @@ import { genericInferenceSteps, genericImageUrl, genericLora, + genericLoras, } from '../common/defaultValues' import { sampleDrivingVideo, sampleVoice } from '@/lib/core/constants' @@ -66,7 +67,6 @@ export const defaultWorkflows: ClapWorkflow[] = [ cond_aug: 0.02, }, }, - /* { id: 'falai://fal-ai/flux-general', label: 'Flux.1-[DEV] with LoRAs', @@ -85,18 +85,18 @@ export const defaultWorkflows: ClapWorkflow[] = [ genericWidth2048, genericHeight2048, genericInferenceSteps, + genericLoras, ], inputValues: { [genericPrompt.id]: genericPrompt.defaultValue, [genericWidth2048.id]: genericWidth2048.defaultValue, [genericHeight2048.id]: genericHeight2048.defaultValue, [genericInferenceSteps.id]: genericInferenceSteps.defaultValue, - + genericLoras: genericInferenceSteps.defaultValue, // support LoRA for this model is a bit tricky, as the parameter must be in JSON // (this is an array of LoraWeight objects, see: https://fal.ai/models/fal-ai/flux-general/playground) }, }, - */ { id: 'falai://fal-ai/flux-realism', label: 'Flux Realism LoRA',