Skip to content

Commit 7540d34

Browse files
jbilcke-hfjbilcke-hf
committed
add flux (fal.ai)
1 parent 6aa6f65 commit 7540d34

File tree

26 files changed

+1340
-985
lines changed

26 files changed

+1340
-985
lines changed

‎package-lock.json‎

Lines changed: 18 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@
8383
"@react-three/uikit-lucide": "^0.3.4",
8484
"@tailwindcss/container-queries": "^0.1.1",
8585
"@types/dom-speech-recognition": "^0.0.4",
86+
"@types/pngjs": "^6.0.5",
8687
"@xenova/transformers": "github:xenova/transformers.js#v3",
8788
"@xyflow/react": "^12.0.3",
8889
"autoprefixer": "10.4.19",
@@ -103,6 +104,7 @@
103104
"monaco-editor": "^0.50.0",
104105
"next": "^14.2.5",
105106
"next-themes": "^0.3.0",
107+
"pngjs": "^7.0.0",
106108
"qs": "^6.12.1",
107109
"query-string": "^9.0.0",
108110
"react": "^18.3.1",

‎src/app/api/resolve/providers/falai/index.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,10 +45,10 @@ export async function resolveSegment(
4545
if (request.settings.imageGenerationModel === 'fal-ai/pulid') {
4646
if (!request.prompts.image.identity) {
4747
// throw new Error(`you selected model ${request.settings.falAiModelForImage}, but no character was found, so skipping`)
48-
// console.log(`warning: user selected model ${request.settings.falAiModelForImage}, but no character was found. Falling back to fal-ai/fast-sdxl`)
48+
// console.log(`warning: user selected model ${request.settings.falAiModelForImage}, but no character was found. Falling back to fal-ai/flux-pro`)
4949

5050
// dirty fix to fallback to a non-face model
51-
request.settings.imageGenerationModel = 'fal-ai/fast-sdxl'
51+
request.settings.imageGenerationModel = 'fal-ai/flux-pro'
5252
}
5353
}
5454

‎src/components/editors/EntityEditor/EntityViewer/index.tsx‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import { useEntityEditor, useIO } from '@/services'
1111
import { cn } from '@/lib/utils'
1212

1313
import { EntityList } from './EntityList'
14+
import { FormSlider } from '@/components/forms/FormSlider'
1415

1516
export function EntityViewer({
1617
className = '',
@@ -163,9 +164,12 @@ export function EntityViewer({
163164
value={draft.appearance || ''}
164165
onChange={(value) => handleInputChange('appearance', value)}
165166
/>
166-
<FormInput
167+
<FormSlider
167168
label="Age"
168-
value={draft.age?.toString() || ''}
169+
value={draft.age || 18}
170+
defaultValue={18}
171+
minValue={18}
172+
maxValue={110}
169173
onChange={(value) => handleInputChange('age', value)}
170174
/>
171175
<FormInput

‎src/components/editors/FilterEditor/FilterViewer/index.tsx‎

Lines changed: 150 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -2,18 +2,101 @@ import {
22
FormField,
33
FormInput,
44
FormSection,
5+
FormSelect,
56
FormSwitch,
67
} from '@/components/forms'
7-
import { useFilterEditor, useUI } from '@/services'
8+
import { FormSlider } from '@/components/forms/FormSlider'
9+
import { useFilterEditor, useRenderer, useUI } from '@/services'
10+
import { getValidNumber } from '@aitube/clap'
11+
import { FilterWithParams } from '@aitube/clapper-services'
12+
import { useEffect, useState } from 'react'
13+
14+
// TODO: move this to the renderer service
15+
// also since filters use WebGPU, I think one day we can run them in real-time
16+
// over the video as well (or maybe using WebGL)
17+
function useCurrentlyVisibleStoryboard(): string | undefined {
18+
const { activeStoryboardSegment } = useRenderer((s) => s.bufferedSegments)
19+
20+
// can't return something if there is nothing
21+
if (!activeStoryboardSegment?.assetUrl.startsWith('data:image/')) {
22+
return undefined
23+
}
24+
25+
return activeStoryboardSegment.assetUrl
26+
}
27+
28+
function useFilteredStoryboard(input?: string): string | undefined {
29+
const current = useFilterEditor((s) => s.current)
30+
const runFilterPipeline = useFilterEditor((s) => s.runFilterPipeline)
31+
const [result, setResult] = useState('')
32+
33+
const currentFiltersWithParams: FilterWithParams[] = current || []
34+
35+
console.log('current changed?', current)
36+
37+
useEffect(() => {
38+
const fn = async (input?: string) => {
39+
if (!input) {
40+
return undefined
41+
}
42+
try {
43+
console.log('running filter using WebGPU..')
44+
const res = await runFilterPipeline(input)
45+
setResult(res)
46+
} catch (err) {
47+
console.error(err)
48+
}
49+
}
50+
fn(input)
51+
}, [
52+
input,
53+
// whenever the pipeline, filter, input values.. change,
54+
// we re-generate the output as well
55+
currentFiltersWithParams,
56+
JSON.stringify(currentFiltersWithParams),
57+
])
58+
59+
return result ? result : undefined
60+
}
861

962
export function FilterViewer() {
1063
const current = useFilterEditor((s) => s.current)
1164
const setCurrent = useFilterEditor((s) => s.setCurrent)
1265
const undo = useFilterEditor((s) => s.undo)
1366
const redo = useFilterEditor((s) => s.redo)
1467

68+
const input = useCurrentlyVisibleStoryboard()
69+
const output = useFilteredStoryboard(input)
70+
1571
const hasBetaAccess = useUI((s) => s.hasBetaAccess)
1672

73+
const setFilterParamValue = (
74+
filterId: string,
75+
fieldId: string,
76+
value: string | number | boolean
77+
) => {
78+
console.log(`setFilterParamValue(${filterId}, ${fieldId}, ${value})`)
79+
setCurrent(
80+
(current || []).map((fwp) => {
81+
if (fwp.filter.id === filterId) {
82+
console.log('match!', fwp)
83+
return {
84+
...fwp,
85+
parameters: {
86+
...fwp.parameters,
87+
[fieldId]: value,
88+
},
89+
}
90+
} else {
91+
console.log('no match..', fwp)
92+
console.log('filterId:', filterId)
93+
console.log('fieldId:', fieldId)
94+
}
95+
return fwp
96+
})
97+
)
98+
}
99+
17100
if (!hasBetaAccess) {
18101
return (
19102
<FormSection label={'Filter Editor'} className="p-4">
@@ -31,40 +114,71 @@ export function FilterViewer() {
31114
}
32115

33116
return (
34-
<>
35-
{current.map(({ filter, parameters }) => (
36-
<FormSection key={filter.id} label={filter.label} className="p-4">
37-
{filter.parameters.map((filter) => (
38-
<FormField key={filter.id}>
39-
{filter.type === 'string' && (
40-
<FormInput
41-
type="text"
42-
label={filter.label}
43-
value={parameters[filter.id]}
44-
defaultValue={filter.defaultValue}
45-
/>
46-
)}
47-
{filter.type === 'number' && (
48-
<FormInput
49-
type="number"
50-
label={filter.label}
51-
value={parameters[filter.id]}
52-
defaultValue={filter.defaultValue}
53-
/>
54-
)}
55-
{filter.type === 'boolean' && (
56-
<FormSwitch
57-
label={filter.label}
58-
checked={!!parameters[filter.id]}
59-
onCheckedChange={() => {
60-
// TODO
61-
}}
62-
/>
63-
)}
64-
</FormField>
65-
))}
66-
</FormSection>
67-
))}
68-
</>
117+
<div className="flex flex-row space-x-2">
118+
<div className="flex w-1/3 flex-col">
119+
{current.map(({ filter, parameters }) => (
120+
<FormSection key={filter.id} label={filter.label} className="p-4">
121+
{filter.parameters.map((field) => (
122+
<FormField key={field.id}>
123+
{field.type === 'string' && (
124+
<FormSelect
125+
label={field.label}
126+
className=""
127+
selectedItemId={parameters[field.id] as string}
128+
selectedItemLabel={parameters[field.id] as string}
129+
defaultItemId={field.defaultValue}
130+
defaultItemLabel={field.defaultValue}
131+
items={field.allowedValues.map((value) => ({
132+
id: value,
133+
label: value,
134+
value,
135+
}))}
136+
onSelect={(value) => {
137+
setFilterParamValue(filter.id, field.id, value || '')
138+
}}
139+
/>
140+
)}
141+
{field.type === 'number' && (
142+
<FormSlider
143+
className="w-full"
144+
label={field.label}
145+
value={getValidNumber(
146+
parameters[field.id],
147+
field.minValue,
148+
field.maxValue,
149+
field.defaultValue
150+
)}
151+
defaultValue={field.defaultValue}
152+
minValue={field.minValue}
153+
maxValue={field.maxValue}
154+
onChange={(newValue) => {
155+
const value = getValidNumber(
156+
newValue,
157+
field.minValue,
158+
field.maxValue,
159+
field.defaultValue
160+
)
161+
setFilterParamValue(filter.id, field.id, value || '')
162+
}}
163+
/>
164+
)}
165+
{field.type === 'boolean' && (
166+
<FormSwitch
167+
label={field.label}
168+
checked={!!parameters[field.id]}
169+
onCheckedChange={(checked) => {
170+
setFilterParamValue(filter.id, field.id, checked || false)
171+
}}
172+
/>
173+
)}
174+
</FormField>
175+
))}
176+
</FormSection>
177+
))}
178+
</div>
179+
<div className="flex w-2/3 flex-col">
180+
{output ? <img src={output}></img> : null}
181+
</div>
182+
</div>
69183
)
70184
}

‎src/components/forms/FormField.tsx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@ export function FormField({
1717
<div
1818
className={cn(
1919
`flex flex-col items-center justify-center`,
20-
`full`,
21-
`opacity-60`,
20+
`w-full`,
21+
`opacity-80`,
2222
`font-thin text-neutral-200`,
2323
// note: the parent component needs @container for this to work
2424
`@md:flex-row @md:space-x-3`

‎src/components/forms/FormSlider.tsx‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
import {
2+
ChangeEvent,
3+
HTMLInputTypeAttribute,
4+
ReactNode,
5+
useMemo,
6+
useRef,
7+
} from 'react'
8+
9+
import { cn, getValidNumber, isValidNumber } from '@/lib/utils'
10+
11+
import { Input } from '../ui/input'
12+
13+
import { FormField } from './FormField'
14+
import { useTheme } from '@/services'
15+
import { Slider } from '../ui/slider'
16+
17+
type SliderProps = React.ComponentProps<typeof Slider>
18+
19+
export function FormSlider<T>(
20+
{
21+
label,
22+
className,
23+
value,
24+
minValue,
25+
maxValue,
26+
defaultValue,
27+
disabled,
28+
onChange,
29+
...props
30+
}: {
31+
label?: ReactNode
32+
className?: string
33+
value?: number
34+
minValue?: number
35+
maxValue?: number
36+
defaultValue?: number
37+
disabled?: boolean
38+
onChange?: (newValue: number) => void
39+
props?: SliderProps
40+
}
41+
// & Omit<ComponentProps<typeof Input>, "value" | "defaultValue" | "placeholder" | "type" | "className" | "disabled" | "onChange">
42+
// & ComponentProps<typeof Input>
43+
) {
44+
const theme = useTheme()
45+
const isNumberInput =
46+
typeof defaultValue === 'number' || typeof value === 'number'
47+
48+
const ref = useRef<HTMLInputElement>(null)
49+
50+
return (
51+
<FormField label={label} className="flex flex-col space-y-1.5">
52+
<Slider
53+
ref={ref}
54+
className={cn(`w-full`, `font-mono text-xs font-light`, className)}
55+
disabled={disabled}
56+
onValueChange={(range) => {
57+
const value = range[0]
58+
onChange?.(value)
59+
}}
60+
defaultValue={[defaultValue || 0]}
61+
value={[value || 0]}
62+
min={minValue}
63+
max={maxValue}
64+
step={0.01}
65+
{...props}
66+
/>
67+
<div className="flex w-full flex-row items-end justify-between text-xs">
68+
{typeof minValue === 'number' && <div>{minValue}</div>}
69+
{typeof maxValue === 'number' && <div>{maxValue}</div>}
70+
</div>
71+
</FormField>
72+
)
73+
}

0 commit comments

Comments
 (0)