diff --git a/src/app/api/resolve/providers/stabilityai/generateVideo.ts b/src/app/api/resolve/providers/stabilityai/generateVideo.ts index 4d97c5d9..714ed983 100644 --- a/src/app/api/resolve/providers/stabilityai/generateVideo.ts +++ b/src/app/api/resolve/providers/stabilityai/generateVideo.ts @@ -150,9 +150,10 @@ async function pollGenerationResult( throw new Error(errors?.join('\n')) } if ( - finish_reason != + finish_reason !== StabilityAIVImageToVideoFetchhGenerationFinishReason.SUCCESS ) { + console.log('finish_reason:', finish_reason) throw new Error('Content filtered') } resolve(`data:video/mp4;base64,${video}`) diff --git a/src/components/editors/EntityEditor/EntityTree/index.tsx b/src/components/editors/EntityEditor/EntityTree/index.tsx index d7dc5d33..71a15cd3 100644 --- a/src/components/editors/EntityEditor/EntityTree/index.tsx +++ b/src/components/editors/EntityEditor/EntityTree/index.tsx @@ -10,6 +10,7 @@ import { Tree } from '@/components/core/tree' import { useEntityTree } from './useEntityTree' import { ClapEntity } from '@aitube/clap' +import { useEntityEditor } from '@/services' export function EntityTree({ className = '', @@ -28,42 +29,29 @@ export function EntityTree({ setProjectEntities(entities) }, [entitiesChanged, entities.map((e) => e.id).join(',')]) - /** - * handle click on tree node - * yes, this is where the magic happens! - * - * @param id - * @param nodeType - * @param node - * @returns - */ - const handleOnChange = async ( - id: string | null, - nodeType?: LibraryNodeType, - nodeItem?: TreeNodeItem - ) => { - console.log(`calling selectTreeNodeById(id)`) - selectTreeNode(id, nodeType, nodeItem) + const setCurrent = useEntityEditor((s) => s.setCurrent) + const selectedNodeItem = useEntityTree((s) => s.selectedNodeItem) + const selectedNodeType = useEntityTree((s) => s.selectedNodeType) - if (!nodeType || !nodeItem) { - console.log('tree-browser: clicked on an undefined node') + useEffect(() => { + if (!selectedNodeType || !selectedNodeItem) { + setCurrent(undefined) return } - if (isClapEntity(nodeType, nodeItem)) { - // ClapEntity + + if (isClapEntity(selectedNodeType, selectedNodeItem)) { + const entity: ClapEntity = selectedNodeItem + + setCurrent(entity) } else { - console.log( - `tree-browser: no action attached to ${nodeType}, so skipping` - ) - return + // must be a different kind of node (eg. a collection, list or folder) } - console.log(`tree-browser: clicked on a ${nodeType}`, nodeItem) - } + }, [selectedNodeType, selectedNodeItem]) return ( value={selectedTreeNodeId} - onChange={handleOnChange} + onChange={selectTreeNode} className={cn(`not-prose h-full w-full px-2 pt-2`, className)} label="Entities" > diff --git a/src/components/editors/EntityEditor/EntityTree/useEntityTree.ts b/src/components/editors/EntityEditor/EntityTree/useEntityTree.ts index 22ae4893..d18b0d10 100644 --- a/src/components/editors/EntityEditor/EntityTree/useEntityTree.ts +++ b/src/components/editors/EntityEditor/EntityTree/useEntityTree.ts @@ -189,13 +189,14 @@ export const useEntityTree = create<{ */ selectedNodeItem: undefined, + /* selectEntity: (entity?: ClapEntity) => { if (entity) { console.log( - 'TODO julian: change this code to search in the entity collections' + 'TODO julian: change this code to search in the entity collections children instead' ) const selectedTreeNode = get().libraryTreeRoot.find( - (node) => node.data?.id === entity.id + (node) =>(node.data as any)?.id === entity.id ) // set({ selectedTreeNode }) @@ -207,6 +208,7 @@ export const useEntityTree = create<{ set({ selectedNodeItem: undefined }) } }, + */ // selectedTreeNode: undefined, selectedTreeNodeId: null, diff --git a/src/components/editors/FilterEditor/FilterTree/index.tsx b/src/components/editors/FilterEditor/FilterTree/index.tsx index b61d4eef..d3b1a844 100644 --- a/src/components/editors/FilterEditor/FilterTree/index.tsx +++ b/src/components/editors/FilterEditor/FilterTree/index.tsx @@ -3,12 +3,21 @@ import { useEffect } from 'react' import { cn } from '@/lib/utils' -import { isClapEntity } from '@/components/tree-browsers/utils/isSomething' +import { + isClapEntity, + isFilter, + isFilterWithParams, +} from '@/components/tree-browsers/utils/isSomething' import { TreeNodeItem, LibraryNodeType } from '@/components/tree-browsers/types' import { Tree } from '@/components/core/tree' import { useFilterTree } from './useFilterTree' import { useFilterEditor } from '@/services/editors/filter-editor/useFilterEditor' +import { + Filter, + FilterParams, + FilterWithParams, +} from '@aitube/clapper-services' export function FilterTree({ className = '', @@ -23,47 +32,59 @@ export function FilterTree({ const availableFilters = useFilterEditor((s) => s.availableFilters) const activeFilters = useFilterEditor((s) => s.activeFilters) const current = useFilterEditor((s) => s.current) + const setCurrent = useFilterEditor((s) => s.setCurrent) useEffect(() => { setAvailableFilters(availableFilters) }, [availableFilters.map((f) => f.id).join(',')]) - /** - * handle click on tree node - * yes, this is where the magic happens! - * - * @param id - * @param nodeType - * @param node - * @returns - */ - const handleOnChange = async ( - id: string | null, - nodeType?: LibraryNodeType, - nodeItem?: TreeNodeItem - ) => { - console.log(`calling selectTreeNodeById(id)`) - selectTreeNode(id, nodeType, nodeItem) - - if (!nodeType || !nodeItem) { - console.log('tree-browser: clicked on an undefined node') + const selectedNodeItem = useFilterTree((s) => s.selectedNodeItem) + const selectedNodeType = useFilterTree((s) => s.selectedNodeType) + + useEffect(() => { + console.log('FilterTree:', { + selectedNodeType, + selectedNodeItem, + }) + if (!selectedNodeType || !selectedNodeItem) { + setCurrent(undefined) return } - if (isClapEntity(nodeType, nodeItem)) { - // ClapEntity + + if (isFilter(selectedNodeType, selectedNodeItem)) { + console.log('is Filter!') + const filter: Filter = selectedNodeItem + + const parameters: FilterParams = {} + for (const field of filter.parameters) { + parameters[field.id] = field.defaultValue + } + + const filterWithParams: FilterWithParams = { + filter, + parameters, + } + + const pipeline = [filterWithParams] + + setCurrent(pipeline) + } else if (isFilterWithParams(selectedNodeType, selectedNodeItem)) { + console.log('is FilterWithParams!') + const filterWithParams: FilterWithParams = selectedNodeItem + + const pipeline = [filterWithParams] + + setCurrent(pipeline) } else { - console.log( - `tree-browser: no action attached to ${nodeType}, so skipping` - ) - return + console.log('is not a filter..') + // must be a different kind of node (eg. a collection, list or folder) } - console.log(`tree-browser: clicked on a ${nodeType}`, nodeItem) - } + }, [selectedNodeType, selectedNodeItem]) return ( value={selectedTreeNodeId} - onChange={handleOnChange} + onChange={selectTreeNode} className={cn(`not-prose h-full w-full px-2 pt-2`, className)} label="Filters" > diff --git a/src/components/editors/FilterEditor/FilterTree/useFilterTree.ts b/src/components/editors/FilterEditor/FilterTree/useFilterTree.ts index 609dc44c..b7eccc05 100644 --- a/src/components/editors/FilterEditor/FilterTree/useFilterTree.ts +++ b/src/components/editors/FilterEditor/FilterTree/useFilterTree.ts @@ -57,7 +57,7 @@ export const useFilterTree = create<{ children: [ { id: UUID(), - nodeType: 'DEFAULT_TREE_NODE_EMPTY', + nodeType: 'FILTER_TREE_NODE_ITEM_FILTER', label: 'Empty', icon: icons.project, className: collectionClassName, @@ -84,11 +84,12 @@ export const useFilterTree = create<{ className: libraryClassName, isExpanded: true, children: filters.map((filter) => ({ - id: UUID(), - nodeType: 'DEFAULT_TREE_NODE_ITEM', + id: filter.id, + nodeType: 'FILTER_TREE_NODE_ITEM_FILTER', label: filter.label, icon: icons.imageFilter, className: collectionClassName, + data: filter, })), } @@ -110,24 +111,26 @@ export const useFilterTree = create<{ */ selectedNodeItem: undefined, - selectEntity: (entity?: ClapEntity) => { - if (entity) { + /* + selectFilter: (filter?: Filter) => { + if (filter) { console.log( - 'TODO julian: change this code to search in the entity collections' + 'TODO julian: change this code to search in the filter collections children instead' ) const selectedTreeNode = get().libraryTreeRoot.find( - (node) => node.data?.id === entity.id + (node) => (node.data as any)?.id === filter.id ) // set({ selectedTreeNode }) set({ selectedTreeNodeId: selectedTreeNode?.id || null }) - set({ selectedNodeItem: entity }) + set({ selectedNodeItem: filter }) } else { // set({ selectedTreeNode: undefined }) set({ selectedTreeNodeId: null }) set({ selectedNodeItem: undefined }) } }, + */ // selectedTreeNode: undefined, selectedTreeNodeId: null, diff --git a/src/components/editors/FilterEditor/FilterViewer/index.tsx b/src/components/editors/FilterEditor/FilterViewer/index.tsx index bd6ffb71..3cd92f1a 100644 --- a/src/components/editors/FilterEditor/FilterViewer/index.tsx +++ b/src/components/editors/FilterEditor/FilterViewer/index.tsx @@ -1,4 +1,9 @@ -import { FormSection } from '@/components/forms' +import { + FormField, + FormInput, + FormSection, + FormSwitch, +} from '@/components/forms' import { useFilterEditor, useUI } from '@/services' export function FilterViewer() { @@ -26,8 +31,40 @@ export function FilterViewer() { } return ( - -

Put filter parameters

- + <> + {current.map(({ filter, parameters }) => ( + + {filter.parameters.map((filter) => ( + + {filter.type === 'string' && ( + + )} + {filter.type === 'number' && ( + + )} + {filter.type === 'boolean' && ( + { + // TODO + }} + /> + )} + + ))} + + ))} + ) } diff --git a/src/components/editors/WorkflowEditor/WorkflowTree/index.tsx b/src/components/editors/WorkflowEditor/WorkflowTree/index.tsx index 69ed1cca..ff90bd35 100644 --- a/src/components/editors/WorkflowEditor/WorkflowTree/index.tsx +++ b/src/components/editors/WorkflowEditor/WorkflowTree/index.tsx @@ -16,42 +16,12 @@ export function WorkflowTree({ const selectTreeNode = useWorkflowTree((s) => s.selectTreeNode) const selectedTreeNodeId = useWorkflowTree((s) => s.selectedTreeNodeId) - /** - * handle click on tree node - * yes, this is where the magic happens! - * - * @param id - * @param nodeType - * @param node - * @returns - */ - const handleOnChange = async ( - id: string | null, - nodeType?: LibraryNodeType, - nodeItem?: TreeNodeItem - ) => { - console.log(`calling selectTreeNodeById(id)`) - selectTreeNode(id, nodeType, nodeItem) - - if (!nodeType || !nodeItem) { - console.log('tree-browser: clicked on an undefined node') - return - } - if (isClapEntity(nodeType, nodeItem)) { - // ClapEntity - } else { - console.log( - `tree-browser: no action attached to ${nodeType}, so skipping` - ) - return - } - console.log(`tree-browser: clicked on a ${nodeType}`, nodeItem) - } + // TODO: allow selecting a workflow (see example of filter/entity tree) return ( value={selectedTreeNodeId} - onChange={handleOnChange} + onChange={selectTreeNode} className={cn(`not-prose h-full w-full px-2 pt-2`, className)} label="Workflows" > diff --git a/src/components/editors/WorkflowEditor/WorkflowTree/useWorkflowTree.ts b/src/components/editors/WorkflowEditor/WorkflowTree/useWorkflowTree.ts index cb8e927b..cf20b517 100644 --- a/src/components/editors/WorkflowEditor/WorkflowTree/useWorkflowTree.ts +++ b/src/components/editors/WorkflowEditor/WorkflowTree/useWorkflowTree.ts @@ -85,6 +85,8 @@ export const useWorkflowTree = create<{ ], } + // TODO: inject the workflow (don't foget to set the `data: field` as well) + const libraryTreeRoot = [builtinLibrary, communityLibrary] set({ @@ -97,24 +99,6 @@ export const useWorkflowTree = create<{ }, selectedNodeItem: undefined, - selectEntity: (entity?: ClapEntity) => { - if (entity) { - console.log( - 'TODO julian: change this code to search in the entity collections' - ) - const selectedTreeNode = get().libraryTreeRoot.find( - (node) => node.data?.id === entity.id - ) - - // set({ selectedTreeNode }) - set({ selectedTreeNodeId: selectedTreeNode?.id || null }) - set({ selectedNodeItem: entity }) - } else { - // set({ selectedTreeNode: undefined }) - set({ selectedTreeNodeId: null }) - set({ selectedNodeItem: undefined }) - } - }, // selectedTreeNode: undefined, selectedTreeNodeId: null, diff --git a/src/components/tree-browsers/types.ts b/src/components/tree-browsers/types.ts index efeba2c3..c25932cd 100644 --- a/src/components/tree-browsers/types.ts +++ b/src/components/tree-browsers/types.ts @@ -3,6 +3,7 @@ import { ScreenplaySequence } from '@aitube/broadway' import { ClapEntity, ClapSegment, ClapWorkflow } from '@aitube/clap' import { TreeNodeType } from '../core/tree/types' +import { Filter, FilterWithParams } from '@aitube/clapper-services' export type DefaultTreeNodeList = 'DEFAULT_TREE_NODE_LIST' export type DefaultTreeNodeItem = 'DEFAULT_TREE_NODE_ITEM' @@ -46,6 +47,17 @@ export type EntityTreeNode = EntityTreeNodeList | EntityTreeNodeItem // ------------------------------------------ +export type FilterTreeNodeList = 'FILTER_TREE_NODE_LIST_FILTERS' +export type FilterTreeNodeItem = 'FILTER_TREE_NODE_ITEM_FILTER' +export type FilterTreeNodeItemPreset = 'FILTER_TREE_NODE_ITEM_FILTER_PRESET' + +export type FilterTreeNode = + | FilterTreeNodeList + | FilterTreeNodeItem + | FilterTreeNodeItemPreset + +// ------------------------------------------ + // some specialized types // TODO @@ -180,6 +192,7 @@ export type LibraryNodeType = | DeviceTreeNode | FlowTreeNode | EntityTreeNode + | FilterTreeNode | TreeRoot // TODO unify this a bit, at least in the naming scheme @@ -201,6 +214,8 @@ export type TreeNodeItem = | DeviceFileOrFolder | CommunityEntityCollection | CommunityFileOrFolder + | Filter + | FilterWithParams // a model library is a collection of models // this collection can itself include sub-models diff --git a/src/components/tree-browsers/utils/isSomething.ts b/src/components/tree-browsers/utils/isSomething.ts index 90e2c2e9..e4c75c4c 100644 --- a/src/components/tree-browsers/utils/isSomething.ts +++ b/src/components/tree-browsers/utils/isSomething.ts @@ -1,9 +1,3 @@ -/* - as you can see, we try to make some data structure generic a bit, - for instance we have a single data structure for AI models ("clap model"), - and a single data structure for files ("item") -*/ - import { ClapEntity } from '@aitube/clap' import { CommunityEntityCollection, @@ -13,6 +7,15 @@ import { DeviceCollection, DeviceFileOrFolder, } from '../types' +import { Filter, FilterWithParams } from '@aitube/clapper-services' + +//////////////////////////////////// +// TYPEGUARDS // +//////////////////////////////////// + +// a tree can mix nodes of various nature (list, leaf) and type (segment, entity, filter, filter with preset params..) +// the purpose of all those type guards is to be able to rect the type of a node, +// and make sure we are within the right type context export const isFSCollection = ( nodeType: LibraryNodeType, @@ -54,3 +57,17 @@ export const isClapEntity = ( ): data is ClapEntity => { return nodeType === 'ENTITY_TREE_NODE_ITEM_ENTITY' } + +export const isFilter = ( + nodeType: LibraryNodeType, + data: TreeNodeItem +): data is Filter => { + return nodeType === 'FILTER_TREE_NODE_ITEM_FILTER' +} + +export const isFilterWithParams = ( + nodeType: LibraryNodeType, + data: TreeNodeItem +): data is FilterWithParams => { + return nodeType === 'FILTER_TREE_NODE_ITEM_FILTER_PRESET' +} diff --git a/src/services/monitor/useMonitor.ts b/src/services/monitor/useMonitor.ts index 797ce2db..a910976a 100644 --- a/src/services/monitor/useMonitor.ts +++ b/src/services/monitor/useMonitor.ts @@ -123,7 +123,7 @@ export const useMonitor = create((set, get) => ({ const timeline: TimelineStore = useTimeline.getState() const { isPlaying, mode, staticVideoRef } = monitor - const { renderLoop } = renderer + const { renderLoop, syncVideoToCurrentCursorPosition } = renderer const { setCursorTimestampAtInMs, cursorTimestampAtInMs } = timeline if (cursorTimestampAtInMs !== timeInMs) { @@ -138,13 +138,15 @@ export const useMonitor = create((set, get) => ({ if (!staticVideoRef) { return } - // console.log("resetting static video current time") + console.log('resetting static video current time') staticVideoRef.currentTime = timeInMs / 1000 } else if (mode === MonitoringMode.DYNAMIC) { // we force a full state recompute // and we also pass jumpedSomewhere=true to indicate that we // need a buffer transition renderLoop(true) + + syncVideoToCurrentCursorPosition() } }, diff --git a/src/services/renderer/useRenderer.ts b/src/services/renderer/useRenderer.ts index c0a200bd..1d9e0918 100644 --- a/src/services/renderer/useRenderer.ts +++ b/src/services/renderer/useRenderer.ts @@ -79,27 +79,22 @@ export const useRenderer = create((set, get) => ({ if (jumpedSomewhere) { // if we jumped somewhere we need to change the visible buffer - if (previousActiveBufferNumber == 2) { // visible buffer is #2 - if (newCurrentSegmentKey !== previousDataUriBuffer1Key) { + if (newCurrentSegmentKey !== previousDataUriBuffer2Key) { // we thus write to visible buffer (#1) newDataUriBuffer1 = maybeNewCurrentSegment - newDataUriBuffer1Key = `${newDataUriBuffer1?.assetUrl || ''}`.slice( - 0, - 1024 - ) + newDataUriBuffer1Key = + `${maybeNewCurrentSegment?.assetUrl || ''}`.slice(0, 1024) } } else { // visible buffer is #1 - if (newCurrentSegmentKey !== previousDataUriBuffer2Key) { + if (newCurrentSegmentKey !== previousDataUriBuffer1Key) { // we thus write to visible buffer (#2) newDataUriBuffer2 = maybeNewCurrentSegment - newDataUriBuffer2Key = `${newDataUriBuffer2?.assetUrl || ''}`.slice( - 0, - 1024 - ) + newDataUriBuffer2Key = + `${maybeNewCurrentSegment?.assetUrl || ''}`.slice(0, 1024) } } } @@ -259,8 +254,63 @@ export const useRenderer = create((set, get) => ({ }, syncVideoToCurrentCursorPosition: () => { + const { + dataUriBuffer1, + dataUriBuffer2, + dataUriBuffer1Key, + dataUriBuffer2Key, + activeBufferNumber, + } = get() const timeline: TimelineStore = useTimeline.getState() - // @TODO julian: make sure we play the video at the correct time - console.log(`syncing Video..`) + + const activeSegment = + activeBufferNumber === 1 ? dataUriBuffer1 : dataUriBuffer2 + + if (!activeSegment) { + return + } + + const elements = document.getElementsByTagName('video') + + const visibleVideoElements = Array.from(elements).filter((element) => + Array.from(element.classList).includes('opacity-100') + ) + + const segmentDurationInMs = + activeSegment.endTimeInMs - activeSegment.startTimeInMs + + const actualDurationInMs = Math.min( + segmentDurationInMs, + activeSegment.assetDurationInMs + ) + + // how much advanced into the video we should be + const deltaTimeInMs = + timeline.cursorTimestampAtInMs - activeSegment.startTimeInMs + + let progressTimeWithinVideoInMs = 0 + if (deltaTimeInMs < 0) { + progressTimeWithinVideoInMs = 0 + } else if (deltaTimeInMs > actualDurationInMs) { + progressTimeWithinVideoInMs = actualDurationInMs + } else { + progressTimeWithinVideoInMs = deltaTimeInMs + } + + for (const videoElement of visibleVideoElements) { + const currentTimeInMs = videoElement.currentTime * 1000 + + const diffInMs = Math.abs(currentTimeInMs - progressTimeWithinVideoInMs) + + // console.log(`diffInMs = ${diffInMs}; deltaTimeInMs = ${deltaTimeInMs}; currentTime = ${visibleVideoElements[0].currentTime * 1000};`) + + // we let the native video player controls the playback as much as possible + // normally we should be in sync, more or less 50ms + // but to be safe, we only kick-in the sync if we observe a discrepancy > 100ms + if (diffInMs > 100) { + // need to be converted to seconds + videoElement.currentTime = progressTimeWithinVideoInMs / 1000 + } + } }, }))