add flux (fal.ai)

package.json

@@ -83,6 +83,7 @@
     "@react-three/uikit-lucide": "^0.3.4",
     "@tailwindcss/container-queries": "^0.1.1",
     "@types/dom-speech-recognition": "^0.0.4",
+    "@types/pngjs": "^6.0.5",
     "@xenova/transformers": "github:xenova/transformers.js#v3",
     "@xyflow/react": "^12.0.3",
     "autoprefixer": "10.4.19",
@@ -103,6 +104,7 @@
     "monaco-editor": "^0.50.0",
     "next": "^14.2.5",
     "next-themes": "^0.3.0",
+    "pngjs": "^7.0.0",
     "qs": "^6.12.1",
     "query-string": "^9.0.0",
     "react": "^18.3.1",

src/app/api/resolve/providers/falai/index.ts

@@ -45,10 +45,10 @@ export async function resolveSegment(
     if (request.settings.imageGenerationModel === 'fal-ai/pulid') {
       if (!request.prompts.image.identity) {
         // throw new Error(`you selected model ${request.settings.falAiModelForImage}, but no character was found, so skipping`)
-        // console.log(`warning: user selected model ${request.settings.falAiModelForImage}, but no character was found. Falling back to fal-ai/fast-sdxl`)
+        // console.log(`warning: user selected model ${request.settings.falAiModelForImage}, but no character was found. Falling back to fal-ai/flux-pro`)
 
         // dirty fix to fallback to a non-face model
-        request.settings.imageGenerationModel = 'fal-ai/fast-sdxl'
+        request.settings.imageGenerationModel = 'fal-ai/flux-pro'
       }
     }
 

src/components/editors/EntityEditor/EntityViewer/index.tsx

@@ -11,6 +11,7 @@ import { useEntityEditor, useIO } from '@/services'
 import { cn } from '@/lib/utils'
 
 import { EntityList } from './EntityList'
+import { FormSlider } from '@/components/forms/FormSlider'
 
 export function EntityViewer({
   className = '',
@@ -163,9 +164,12 @@ export function EntityViewer({
                 value={draft.appearance || ''}
                 onChange={(value) => handleInputChange('appearance', value)}
               />
-              <FormInput
+              <FormSlider
                 label="Age"
-                value={draft.age?.toString() || ''}
+                value={draft.age || 18}
+                defaultValue={18}
+                minValue={18}
+                maxValue={110}
                 onChange={(value) => handleInputChange('age', value)}
               />
               <FormInput

src/components/editors/FilterEditor/FilterViewer/index.tsx

@@ -2,18 +2,101 @@ import {
   FormField,
   FormInput,
   FormSection,
+  FormSelect,
   FormSwitch,
 } from '@/components/forms'
-import { useFilterEditor, useUI } from '@/services'
+import { FormSlider } from '@/components/forms/FormSlider'
+import { useFilterEditor, useRenderer, useUI } from '@/services'
+import { getValidNumber } from '@aitube/clap'
+import { FilterWithParams } from '@aitube/clapper-services'
+import { useEffect, useState } from 'react'
+
+// TODO: move this to the renderer service
+// also since filters use WebGPU, I think one day we can run them in real-time
+// over the video as well (or maybe using WebGL)
+function useCurrentlyVisibleStoryboard(): string | undefined {
+  const { activeStoryboardSegment } = useRenderer((s) => s.bufferedSegments)
+
+  // can't return something if there is nothing
+  if (!activeStoryboardSegment?.assetUrl.startsWith('data:image/')) {
+    return undefined
+  }
+
+  return activeStoryboardSegment.assetUrl
+}
+
+function useFilteredStoryboard(input?: string): string | undefined {
+  const current = useFilterEditor((s) => s.current)
+  const runFilterPipeline = useFilterEditor((s) => s.runFilterPipeline)
+  const [result, setResult] = useState('')
+
+  const currentFiltersWithParams: FilterWithParams[] = current || []
+
+  console.log('current changed?', current)
+
+  useEffect(() => {
+    const fn = async (input?: string) => {
+      if (!input) {
+        return undefined
+      }
+      try {
+        console.log('running filter using WebGPU..')
+        const res = await runFilterPipeline(input)
+        setResult(res)
+      } catch (err) {
+        console.error(err)
+      }
+    }
+    fn(input)
+  }, [
+    input,
+    // whenever the pipeline, filter, input values.. change,
+    // we re-generate the output as well
+    currentFiltersWithParams,
+    JSON.stringify(currentFiltersWithParams),
+  ])
+
+  return result ? result : undefined
+}
 
 export function FilterViewer() {
   const current = useFilterEditor((s) => s.current)
   const setCurrent = useFilterEditor((s) => s.setCurrent)
   const undo = useFilterEditor((s) => s.undo)
   const redo = useFilterEditor((s) => s.redo)
 
+  const input = useCurrentlyVisibleStoryboard()
+  const output = useFilteredStoryboard(input)
+
   const hasBetaAccess = useUI((s) => s.hasBetaAccess)
 
+  const setFilterParamValue = (
+    filterId: string,
+    fieldId: string,
+    value: string | number | boolean
+  ) => {
+    console.log(`setFilterParamValue(${filterId}, ${fieldId}, ${value})`)
+    setCurrent(
+      (current || []).map((fwp) => {
+        if (fwp.filter.id === filterId) {
+          console.log('match!', fwp)
+          return {
+            ...fwp,
+            parameters: {
+              ...fwp.parameters,
+              [fieldId]: value,
+            },
+          }
+        } else {
+          console.log('no match..', fwp)
+          console.log('filterId:', filterId)
+          console.log('fieldId:', fieldId)
+        }
+        return fwp
+      })
+    )
+  }
+
   if (!hasBetaAccess) {
     return (
       <FormSection label={'Filter Editor'} className="p-4">
@@ -31,40 +114,71 @@ export function FilterViewer() {
   }
 
   return (
-    <>
-      {current.map(({ filter, parameters }) => (
-        <FormSection key={filter.id} label={filter.label} className="p-4">
-          {filter.parameters.map((filter) => (
-            <FormField key={filter.id}>
-              {filter.type === 'string' && (
-                <FormInput
-                  type="text"
-                  label={filter.label}
-                  value={parameters[filter.id]}
-                  defaultValue={filter.defaultValue}
-                />
-              )}
-              {filter.type === 'number' && (
-                <FormInput
-                  type="number"
-                  label={filter.label}
-                  value={parameters[filter.id]}
-                  defaultValue={filter.defaultValue}
-                />
-              )}
-              {filter.type === 'boolean' && (
-                <FormSwitch
-                  label={filter.label}
-                  checked={!!parameters[filter.id]}
-                  onCheckedChange={() => {
-                    // TODO
-                  }}
-                />
-              )}
-            </FormField>
-          ))}
-        </FormSection>
-      ))}
-    </>
+    <div className="flex flex-row space-x-2">
+      <div className="flex w-1/3 flex-col">
+        {current.map(({ filter, parameters }) => (
+          <FormSection key={filter.id} label={filter.label} className="p-4">
+            {filter.parameters.map((field) => (
+              <FormField key={field.id}>
+                {field.type === 'string' && (
+                  <FormSelect
+                    label={field.label}
+                    className=""
+                    selectedItemId={parameters[field.id] as string}
+                    selectedItemLabel={parameters[field.id] as string}
+                    defaultItemId={field.defaultValue}
+                    defaultItemLabel={field.defaultValue}
+                    items={field.allowedValues.map((value) => ({
+                      id: value,
+                      label: value,
+                      value,
+                    }))}
+                    onSelect={(value) => {
+                      setFilterParamValue(filter.id, field.id, value || '')
+                    }}
+                  />
+                )}
+                {field.type === 'number' && (
+                  <FormSlider
+                    className="w-full"
+                    label={field.label}
+                    value={getValidNumber(
+                      parameters[field.id],
+                      field.minValue,
+                      field.maxValue,
+                      field.defaultValue
+                    )}
+                    defaultValue={field.defaultValue}
+                    minValue={field.minValue}
+                    maxValue={field.maxValue}
+                    onChange={(newValue) => {
+                      const value = getValidNumber(
+                        newValue,
+                        field.minValue,
+                        field.maxValue,
+                        field.defaultValue
+                      )
+                      setFilterParamValue(filter.id, field.id, value || '')
+                    }}
+                  />
+                )}
+                {field.type === 'boolean' && (
+                  <FormSwitch
+                    label={field.label}
+                    checked={!!parameters[field.id]}
+                    onCheckedChange={(checked) => {
+                      setFilterParamValue(filter.id, field.id, checked || false)
+                    }}
+                  />
+                )}
+              </FormField>
+            ))}
+          </FormSection>
+        ))}
+      </div>
+      <div className="flex w-2/3 flex-col">
+        {output ? <img src={output}></img> : null}
+      </div>
+    </div>
   )
 }

src/components/forms/FormField.tsx

@@ -17,8 +17,8 @@ export function FormField({
     <div
       className={cn(
         `flex flex-col items-center justify-center`,
-        `full`,
-        `opacity-60`,
+        `w-full`,
+        `opacity-80`,
         `font-thin text-neutral-200`,
         // note: the parent component needs @container for this to work
         `@md:flex-row @md:space-x-3`

src/components/forms/FormSlider.tsx

@@ -0,0 +1,73 @@
+import {
+  ChangeEvent,
+  HTMLInputTypeAttribute,
+  ReactNode,
+  useMemo,
+  useRef,
+} from 'react'
+
+import { cn, getValidNumber, isValidNumber } from '@/lib/utils'
+
+import { Input } from '../ui/input'
+
+import { FormField } from './FormField'
+import { useTheme } from '@/services'
+import { Slider } from '../ui/slider'
+
+type SliderProps = React.ComponentProps<typeof Slider>
+
+export function FormSlider<T>(
+  {
+    label,
+    className,
+    value,
+    minValue,
+    maxValue,
+    defaultValue,
+    disabled,
+    onChange,
+    ...props
+  }: {
+    label?: ReactNode
+    className?: string
+    value?: number
+    minValue?: number
+    maxValue?: number
+    defaultValue?: number
+    disabled?: boolean
+    onChange?: (newValue: number) => void
+    props?: SliderProps
+  }
+  //  & Omit<ComponentProps<typeof Input>, "value" | "defaultValue" | "placeholder" | "type" | "className" | "disabled" | "onChange">
+  // & ComponentProps<typeof Input>
+) {
+  const theme = useTheme()
+  const isNumberInput =
+    typeof defaultValue === 'number' || typeof value === 'number'
+
+  const ref = useRef<HTMLInputElement>(null)
+
+  return (
+    <FormField label={label} className="flex flex-col space-y-1.5">
+      <Slider
+        ref={ref}
+        className={cn(`w-full`, `font-mono text-xs font-light`, className)}
+        disabled={disabled}
+        onValueChange={(range) => {
+          const value = range[0]
+          onChange?.(value)
+        }}
+        defaultValue={[defaultValue || 0]}
+        value={[value || 0]}
+        min={minValue}
+        max={maxValue}
+        step={0.01}
+        {...props}
+      />
+      <div className="flex w-full flex-row items-end justify-between text-xs">
+        {typeof minValue === 'number' && <div>{minValue}</div>}
+        {typeof maxValue === 'number' && <div>{maxValue}</div>}
+      </div>
+    </FormField>
+  )
+}