fix replicate image width + fix replicate lora

jbilcke-hf · Aug 18, 2024 · 4e773b8 · 4e773b8
1 parent d0e414a
commit 4e773b8
Show file tree

Hide file tree

Showing 7 changed files with 65 additions and 5 deletions.
diff --git a/packages/app/src/app/api/resolve/providers/falai/index.ts b/packages/app/src/app/api/resolve/providers/falai/index.ts
@@ -10,6 +10,7 @@ import {
 } from './types'
 import { getWorkflowInputValues } from '../getWorkflowInputValues'
 import { sampleVoice } from '@/lib/core/constants'
+import { getWorkflowLora } from '@/services/editors/workflow-editor/workflows/common/loras/getWorkflowLora'
 
 export async function resolveSegment(
   request: ResolveRequest
@@ -65,7 +66,7 @@ export async function resolveSegment(
         workflowValues.width ||
         workflowDefaultValues.width,
       height:
-        request.meta.width ||
+        request.meta.height ||
         workflowValues.height ||
         workflowDefaultValues.height,
     }
@@ -100,6 +101,44 @@ export async function resolveSegment(
             request.settings.censorNotForAllAudiencesContent,
         },
       })) as FalAiImageResponse
+    } else if (model === 'fal-ai/flux-general') {
+      // note: this isn't the right place to do this, because maybe the LoRAs are dynamic
+      const loraModel = getWorkflowLora(
+        request.settings.imageGenerationWorkflow
+      )
+      if (!loraModel) {
+        throw new Error(`this model cannot be used without a valid LoRA`)
+      }
+
+      const url = loraModel.repoOrUrl.startsWith('http')
+        ? loraModel.repoOrUrl
+        : `https://huggingface.co/${
+        loraModel.repoOrUrl
+        }/resolve/main/${
+        loraModel.fileName || 'lora_weights.safetensors'
+        }?download=true`
+
+      const prompt = [
+        loraModel.trigger, request.prompts.image.positive]
+        .filter((x) => x)
+        .join(' ')
+
+      result = (await fal.run(model, {
+        input: {
+          prompt,
+          loras: [
+            {
+              path: url,
+              scale: 1.0, // between 0 and 4
+            },
+          ],
+          image_size: imageSize,
+          num_images: 1,
+          sync_mode: true,
+          enable_safety_checker:
+            request.settings.censorNotForAllAudiencesContent,
+        },
+      })) as FalAiImageResponse
     } else {
       result = (await fal.run(model, {
         input: {

diff --git a/packages/app/src/app/api/resolve/providers/replicate/index.ts b/packages/app/src/app/api/resolve/providers/replicate/index.ts
@@ -50,6 +50,10 @@ export async function resolveSegment(
       const loraModel = getWorkflowLora(
         request.settings.imageGenerationWorkflow
       )
+
+      if (!loraModel) {
+        throw new Error(`this model cannot be used without a valid LoRA`)
+      }
 
       params = {
         // for some reason this model doesn't support arbitrary width and height,

diff --git a/packages/app/src/lib/core/constants.ts b/packages/app/src/lib/core/constants.ts
@@ -3,7 +3,7 @@
 export const HARD_LIMIT_NB_MAX_ASSETS_TO_GENERATE_IN_PARALLEL = 32
 
 export const APP_NAME = 'Clapper.app'
-export const APP_REVISION = '20240816+1424'
+export const APP_REVISION = '20240818+2050'
 
 export const APP_DOMAIN = 'Clapper.app'
 export const APP_LINK = 'https://clapper.app'

diff --git a/packages/app/src/services/editors/workflow-editor/workflows/common/defaultValues.ts b/packages/app/src/services/editors/workflow-editor/workflows/common/defaultValues.ts
@@ -137,6 +137,16 @@ export const genericAudio: ClapInputField = {
   defaultValue: '',
 }
 
+export const genericLoras: ClapInputField = {
+  id: 'loras',
+  label: 'LoRAs',
+  description: 'Comma-separated LoRAs',
+  category: ClapInputCategory.LORA,
+  type: 'string',
+  allowedValues: [],
+  defaultValue: '',
+}
+
 export const genericInferenceSteps: ClapInputField = {
   id: 'num_inference_steps',
   label: 'Inference steps',

diff --git a/packages/app/src/services/editors/workflow-editor/workflows/common/loras/index.ts b/packages/app/src/services/editors/workflow-editor/workflows/common/loras/index.ts
@@ -24,6 +24,8 @@ export const defaultLoraModels: Lora[] = [
       'kodachrome, blurry, realistic, still life, depth of field, scenery, no humans, monochrome, greyscale, traditional media, horizon, looking at viewer, light particles, shadow',
 
     repoOrUrl: 'alvdansen/flux-koda',
+
+    fileName: 'araminta_k_flux_koda.safetensors',
   },
   {
     id: 'lora://hf.co/models/veryVANYA/ps1-style-flux',
@@ -47,6 +49,8 @@ trained on 15 gpt4o captioned and adjusted ps1/n64 game screenshots using https:
     extensions: 'ps1 game screenshot',
 
     repoOrUrl: 'veryVANYA/ps1-style-flux',
+
+    fileName: 'ps1_style_flux_v1.safetensors'
   },
   /*
   {

diff --git a/packages/app/src/services/editors/workflow-editor/workflows/common/types.ts b/packages/app/src/services/editors/workflow-editor/workflows/common/types.ts
@@ -32,4 +32,7 @@ export type Lora = {
   // name of the model repository on Hugging Face
   // or direct URL to the weights
   repoOrUrl: string
+
+  // optional filename
+  fileName: string
 }
diff --git a/packages/app/src/services/editors/workflow-editor/workflows/falai/defaultWorkflows.ts b/packages/app/src/services/editors/workflow-editor/workflows/falai/defaultWorkflows.ts
@@ -23,6 +23,7 @@ import {
   genericInferenceSteps,
   genericImageUrl,
   genericLora,
+  genericLoras,
 } from '../common/defaultValues'
 import { sampleDrivingVideo, sampleVoice } from '@/lib/core/constants'
 
@@ -66,7 +67,6 @@ export const defaultWorkflows: ClapWorkflow[] = [
       cond_aug: 0.02,
     },
   },
-  /*
   {
     id: 'falai://fal-ai/flux-general',
     label: 'Flux.1-[DEV] with LoRAs',
@@ -85,18 +85,18 @@ export const defaultWorkflows: ClapWorkflow[] = [
       genericWidth2048,
       genericHeight2048,
       genericInferenceSteps,
+      genericLoras,
     ],
     inputValues: {
       [genericPrompt.id]: genericPrompt.defaultValue,
       [genericWidth2048.id]: genericWidth2048.defaultValue,
       [genericHeight2048.id]: genericHeight2048.defaultValue,
       [genericInferenceSteps.id]: genericInferenceSteps.defaultValue,
-
+      genericLoras: genericInferenceSteps.defaultValue,
       // support LoRA for this model is a bit tricky, as the parameter must be in JSON
       // (this is an array of LoraWeight objects, see: https://fal.ai/models/fal-ai/flux-general/playground)
     },
   },
-  */
   {
     id: 'falai://fal-ai/flux-realism',
     label: 'Flux Realism LoRA',