diff --git a/packages/app/src/app/main.tsx b/packages/app/src/app/main.tsx index 8f4ed351..d1cb04b5 100644 --- a/packages/app/src/app/main.tsx +++ b/packages/app/src/app/main.tsx @@ -89,7 +89,6 @@ function MainContent({ mode }: { mode: ClapperIntegrationMode }) { setHasBetaAccess(hasBetaAccess) }, [hasBetaAccess, setHasBetaAccess]) - useQueryStringLoader() const iframeLayout = ( diff --git a/packages/app/src/components/toolbars/top-menu/file/useQueryStringLoader.ts b/packages/app/src/components/toolbars/top-menu/file/useQueryStringLoader.ts index ea622d8f..cb3632f9 100644 --- a/packages/app/src/components/toolbars/top-menu/file/useQueryStringLoader.ts +++ b/packages/app/src/components/toolbars/top-menu/file/useQueryStringLoader.ts @@ -6,12 +6,13 @@ import { useQueryStringParams } from '@/lib/hooks' import { useAssistant, useIO } from '@/services' export function useQueryStringLoader() { - const { clapUrl, startAt, interactive, prompt, imageStrategy } = useQueryStringParams({ - // clapUrl: `/samples/test.clap`, - // clapUrl: `/samples/Afterglow%20v10%20X%20Rewrite%20Bryan%20E.%20Harris%202023.clap`, - }) + const { clapUrl, startAt, interactive, prompt, imageStrategy } = + useQueryStringParams({ + // clapUrl: `/samples/test.clap`, + // clapUrl: `/samples/Afterglow%20v10%20X%20Rewrite%20Bryan%20E.%20Harris%202023.clap`, + }) - const processUserMessage = useAssistant(s => s.processUserMessage) + const processUserMessage = useAssistant((s) => s.processUserMessage) const openClapUrl = useIO((s) => s.openClapUrl) useEffect(() => { diff --git a/packages/app/src/lib/hooks/useQueryStringParams.ts b/packages/app/src/lib/hooks/useQueryStringParams.ts index abcf502f..433c6987 100644 --- a/packages/app/src/lib/hooks/useQueryStringParams.ts +++ b/packages/app/src/lib/hooks/useQueryStringParams.ts @@ -2,34 +2,46 @@ import { useSearchParams } from 'next/navigation' import { getValidNumber } from '../utils' import { RenderingStrategy } from '@aitube/timeline' -export function useQueryStringParams({ - interactive: defaultInteractive = false, - startAt: defaultStartAt = 0, - prompt: defaultPrompt = '', - clapUrl: defaultClapUrl = '', - imageStrategy: defaultImageStrategy = RenderingStrategy.ON_DEMAND -}: { - interactive?: boolean - startAt?: number - prompt?: string - clapUrl?: string - imageStrategy?: RenderingStrategy -} = { - interactive: false, - startAt: 0, - prompt: '', - clapUrl: '', - imageStrategy: RenderingStrategy.ON_DEMAND -}) { +export function useQueryStringParams( + { + interactive: defaultInteractive = false, + startAt: defaultStartAt = 0, + prompt: defaultPrompt = '', + clapUrl: defaultClapUrl = '', + imageStrategy: defaultImageStrategy = RenderingStrategy.ON_DEMAND, + }: { + interactive?: boolean + startAt?: number + prompt?: string + clapUrl?: string + imageStrategy?: RenderingStrategy + } = { + interactive: false, + startAt: 0, + prompt: '', + clapUrl: '', + imageStrategy: RenderingStrategy.ON_DEMAND, + } +) { const searchParams = useSearchParams() const prompt = (searchParams?.get('prompt') as string) || defaultPrompt - const imageStrategy = (searchParams?.get('imageStrategy') as RenderingStrategy) || defaultImageStrategy + const imageStrategy = + (searchParams?.get('imageStrategy') as RenderingStrategy) || + defaultImageStrategy - const startAt = getValidNumber(`${(searchParams?.get('startAt') as string) || defaultStartAt}`.trim(), 0, Number.MAX_VALUE, 0) + const startAt = getValidNumber( + `${(searchParams?.get('startAt') as string) || defaultStartAt}`.trim(), + 0, + Number.MAX_VALUE, + 0 + ) - const interactive = `${(searchParams?.get('interactive') as string) || defaultInteractive}`.trim().toLowerCase() === 'true' + const interactive = + `${(searchParams?.get('interactive') as string) || defaultInteractive}` + .trim() + .toLowerCase() === 'true' const clapUrl = (searchParams?.get('clap') as string) || defaultClapUrl diff --git a/packages/app/src/services/api/resolve.ts b/packages/app/src/services/api/resolve.ts new file mode 100644 index 00000000..1478e85a --- /dev/null +++ b/packages/app/src/services/api/resolve.ts @@ -0,0 +1,80 @@ +'use client' + +import { + clapSegmentToTimelineSegment, + TimelineSegment, + TimelineStore, + useTimeline, +} from '@aitube/timeline' +import { + ResolveRequest, + ResolveRequestPrompts, + SettingsStore, +} from '@aitube/clapper-services' +import { useSettings } from '../settings' +import { ClapSegmentCategory, newSegment } from '@aitube/clap' +import { getDefaultResolveRequestPrompts } from '../resolver/getDefaultResolveRequestPrompts' + +export async function resolve( + req: Partial +): Promise { + const { getRequestSettings }: SettingsStore = useSettings.getState() + const { meta }: TimelineStore = useTimeline.getState() + + const defaultTimelineSegment: TimelineSegment = + await clapSegmentToTimelineSegment( + newSegment({ + category: ClapSegmentCategory.STORYBOARD, + }) + ) + + const segment: TimelineSegment = { + ...defaultTimelineSegment, + ...req.segment, + + // we omit things that cannot be serialized + scene: undefined, + audioBuffer: undefined, + textures: {}, + } + + const request: ResolveRequest = { + settings: getRequestSettings(), + segment, + + segments: Array.isArray(req.segments) + ? req.segments.map((s) => ({ + ...s, + + // we omit things that cannot be serialized + scene: undefined, + audioBuffer: undefined, + textures: {}, + })) + : [], + + entities: req.entities ? req.entities : {}, + speakingCharactersIds: Array.isArray(req.speakingCharactersIds) + ? req.speakingCharactersIds + : [], + generalCharactersIds: Array.isArray(req.generalCharactersIds) + ? req.generalCharactersIds + : [], + mainCharacterId: req.mainCharacterId || undefined, + mainCharacterEntity: req.mainCharacterEntity || undefined, + meta, + prompts: getDefaultResolveRequestPrompts(req.prompts), + } + + const res = await fetch('/api/resolve', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(request), + }) + + const newSegmentData = (await res.json()) as TimelineSegment + + return newSegmentData +} diff --git a/packages/app/src/services/monitor/getDefaultMonitorState.ts b/packages/app/src/services/monitor/getDefaultMonitorState.ts index 3adf1379..c56e9c64 100644 --- a/packages/app/src/services/monitor/getDefaultMonitorState.ts +++ b/packages/app/src/services/monitor/getDefaultMonitorState.ts @@ -6,6 +6,7 @@ export function getDefaultMonitorState(): MonitorState { lastTimelineUpdateAtInMs: 0, isPlaying: false, staticVideoRef: undefined, + isEmbedded: false, } return state diff --git a/packages/app/src/services/monitor/useMonitor.ts b/packages/app/src/services/monitor/useMonitor.ts index 701c19ec..2403d086 100644 --- a/packages/app/src/services/monitor/useMonitor.ts +++ b/packages/app/src/services/monitor/useMonitor.ts @@ -49,6 +49,12 @@ export const useMonitor = create((set, get) => ({ }) }, + setIsEmbedded: (isEmbedded: boolean) => { + set({ + isEmbedded, + }) + }, + checkIfPlaying: (): boolean => { return get().isPlaying }, diff --git a/packages/app/src/services/renderer/getDefaultRendererState.ts b/packages/app/src/services/renderer/getDefaultRendererState.ts index 358ca0ea..399b91ee 100644 --- a/packages/app/src/services/renderer/getDefaultRendererState.ts +++ b/packages/app/src/services/renderer/getDefaultRendererState.ts @@ -1,9 +1,38 @@ -import { RendererState } from '@aitube/clapper-services' +import { RenderingStrategy } from '@aitube/timeline' + +import { + RendererState, + RenderingBufferSizes, + RenderingStrategies, +} from '@aitube/clapper-services' import { getDefaultBufferedSegments } from './getDefaultBufferedSegments' export function getDefaultRendererState(): RendererState { + const renderingStrategies: RenderingStrategies = { + imageRenderingStrategy: RenderingStrategy.BUFFERED_PLAYBACK_STREAMING, + videoRenderingStrategy: RenderingStrategy.BUFFERED_PLAYBACK_STREAMING, + soundRenderingStrategy: RenderingStrategy.BUFFERED_PLAYBACK_STREAMING, + voiceRenderingStrategy: RenderingStrategy.BUFFERED_PLAYBACK_STREAMING, + musicRenderingStrategy: RenderingStrategy.BUFFERED_PLAYBACK_STREAMING, + } + + /** + * Tells how many segments should be renderer in advanced during playback, for each segment category + */ + const bufferSizes: RenderingBufferSizes = { + imageBufferSize: 32, + videoBufferSize: 32, + soundBufferSize: 32, + voiceBufferSize: 32, + musicBufferSize: 8, // music segments are longer, so no need to generate that many + } + const state: RendererState = { + ...bufferSizes, + + ...renderingStrategies, + bufferedSegments: getDefaultBufferedSegments(), dataUriBuffer1: undefined, diff --git a/packages/app/src/services/renderer/useRenderLoop.ts b/packages/app/src/services/renderer/useRenderLoop.ts index f0fe435a..61b06137 100644 --- a/packages/app/src/services/renderer/useRenderLoop.ts +++ b/packages/app/src/services/renderer/useRenderLoop.ts @@ -10,6 +10,8 @@ import { useRenderer } from './useRenderer' import { useAudio } from '@/services/audio/useAudio' import { useMonitor } from '../monitor/useMonitor' import { useEffect, useRef } from 'react' +import { useSettings } from '../settings' +import { set } from 'date-fns' /** * Runs a rendering loop @@ -34,6 +36,33 @@ export function useRenderLoop(): void { const timeoutRef = useRef() + const setUserDefinedRenderingStrategies = useRenderer( + (s) => s.setUserDefinedRenderingStrategies + ) + + // those are the currently active rendering strategies determined by the renderer + // this is different from the image rendering preferences (what the user has set) + const imageRenderingStrategy = useSettings((s) => s.imageRenderingStrategy) + const videoRenderingStrategy = useSettings((s) => s.videoRenderingStrategy) + const soundRenderingStrategy = useSettings((s) => s.soundRenderingStrategy) + const voiceRenderingStrategy = useSettings((s) => s.voiceRenderingStrategy) + const musicRenderingStrategy = useSettings((s) => s.musicRenderingStrategy) + useEffect(() => { + setUserDefinedRenderingStrategies({ + imageRenderingStrategy, + videoRenderingStrategy, + soundRenderingStrategy, + voiceRenderingStrategy, + musicRenderingStrategy, + }) + }, [ + imageRenderingStrategy, + videoRenderingStrategy, + soundRenderingStrategy, + voiceRenderingStrategy, + musicRenderingStrategy, + ]) + // used to control transitions between buffers useEffect(() => { clearTimeout(timeoutRef.current) diff --git a/packages/app/src/services/renderer/useRenderer.ts b/packages/app/src/services/renderer/useRenderer.ts index 1d9e0918..58872443 100644 --- a/packages/app/src/services/renderer/useRenderer.ts +++ b/packages/app/src/services/renderer/useRenderer.ts @@ -2,12 +2,23 @@ import { create } from 'zustand' import { ClapOutputType, ClapSegmentCategory } from '@aitube/clap' -import { BufferedSegments, RendererStore } from '@aitube/clapper-services' -import { TimelineStore, useTimeline, TimelineSegment } from '@aitube/timeline' +import { + BufferedSegments, + RendererStore, + RenderingStrategies, +} from '@aitube/clapper-services' +import { + TimelineStore, + useTimeline, + TimelineSegment, + RenderingStrategy, +} from '@aitube/timeline' import { getDefaultRendererState } from './getDefaultRendererState' import { getSegmentCacheKey } from './getSegmentCacheKey' import { getDefaultBufferedSegments } from './getDefaultBufferedSegments' +import { useMonitor } from '../monitor/useMonitor' +import { useSettings } from '../settings' export const useRenderer = create((set, get) => ({ ...getDefaultRendererState(), @@ -18,6 +29,42 @@ export const useRenderer = create((set, get) => ({ }) }, + setUserDefinedRenderingStrategies: ({ + imageRenderingStrategy: userDefinedImageRenderingStrategy, + videoRenderingStrategy: userDefinedVideoRenderingStrategy, + soundRenderingStrategy: userDefinedSoundRenderingStrategy, + voiceRenderingStrategy: userDefinedVoiceRenderingStrategy, + musicRenderingStrategy: userDefinedMusicRenderingStrategy, + }: RenderingStrategies) => { + // if the monitor is embedded, the imageRenderingStrategy is temporary bypassed + // to try to render all the segments in advance, to create a buffer + // this is a potentially expensive solution, so we might want to put + // some limits to that ex. the first 64 or something + const { isEmbedded } = useMonitor.getState() + + // What we are doing here is that when we are in "embedded" video player mode, + // we bypass the image rendering strategy to render all the segments in advance + const imageRenderingStrategy = isEmbedded + ? RenderingStrategy.BUFFERED_PLAYBACK_STREAMING + : userDefinedImageRenderingStrategy + const videoRenderingStrategy = + /* isEmbedded ? RenderingStrategy.ON_SCREEN_THEN_ALL : */ userDefinedVideoRenderingStrategy + const soundRenderingStrategy = + /* isEmbedded ? RenderingStrategy.ON_SCREEN_THEN_ALL : */ userDefinedSoundRenderingStrategy + const voiceRenderingStrategy = + /* isEmbedded ? RenderingStrategy.ON_SCREEN_THEN_ALL : */ userDefinedVoiceRenderingStrategy + const musicRenderingStrategy = + /* isEmbedded ? RenderingStrategy.ON_SCREEN_THEN_ALL : */ userDefinedMusicRenderingStrategy + + set({ + imageRenderingStrategy, + videoRenderingStrategy, + soundRenderingStrategy, + voiceRenderingStrategy, + musicRenderingStrategy, + }) + }, + // this will be called at 60 FPS - and yes, it is expensive // we could probably improve things by using a temporal tree index renderLoop: (jumpedSomewhere?: boolean): BufferedSegments => { diff --git a/packages/app/src/services/resolver/getDefaultResolveRequestPrompts.ts b/packages/app/src/services/resolver/getDefaultResolveRequestPrompts.ts new file mode 100644 index 00000000..cb344961 --- /dev/null +++ b/packages/app/src/services/resolver/getDefaultResolveRequestPrompts.ts @@ -0,0 +1,37 @@ +import { ResolveRequestPrompts } from '@aitube/clapper-services' + +type DeepPartial = T extends object + ? { + [P in keyof T]?: DeepPartial + } + : T + +export function getDefaultResolveRequestPrompts( + partials: DeepPartial = {} +): ResolveRequestPrompts { + const defaultPrompts: ResolveRequestPrompts = { + image: { + identity: `${partials?.image?.identity || ''}`, + positive: `${partials?.image?.positive || ''}`, + negative: `${partials?.image?.negative || ''}`, + }, + video: { + image: `${partials?.video?.image || ''}`, + voice: `${partials?.video?.voice || ''}`, + }, + voice: { + identity: `${partials?.voice?.identity || ''}`, + positive: `${partials?.voice?.positive || ''}`, + negative: `${partials?.voice?.negative || ''}`, + }, + audio: { + positive: `${partials?.audio?.positive || ''}`, + negative: `${partials?.audio?.negative || ''}`, + }, + music: { + positive: `${partials?.music?.positive || ''}`, + negative: `${partials?.music?.negative || ''}`, + }, + } + return defaultPrompts +} diff --git a/packages/app/src/services/resolver/useResolver.ts b/packages/app/src/services/resolver/useResolver.ts index b470fe5f..4a391368 100644 --- a/packages/app/src/services/resolver/useResolver.ts +++ b/packages/app/src/services/resolver/useResolver.ts @@ -2,12 +2,16 @@ import { create } from 'zustand' import { + ClapAssetSource, ClapEntity, ClapOutputType, ClapSegmentCategory, ClapSegmentFilteringMode, ClapSegmentStatus, filterSegments, + generateSeed, + newSegment, + UUID, } from '@aitube/clap' import { RenderingStrategy, @@ -17,6 +21,7 @@ import { SegmentVisibility, segmentVisibilityPriority, TimelineSegment, + clapSegmentToTimelineSegment, } from '@aitube/timeline' import { getBackgroundAudioPrompt, @@ -24,13 +29,23 @@ import { getMusicPrompt, getSpeechForegroundAudioPrompt, getVideoPrompt, + getCharacterReferencePrompt, } from '@aitube/engine' -import { ResolverStore } from '@aitube/clapper-services' +import { + RendererState, + RenderingBufferSizes, + RenderingStrategies, + ResolverStore, +} from '@aitube/clapper-services' import { getDefaultResolverState } from './getDefaultResolverState' import { useSettings } from '../settings' import { DEFAULT_WAIT_TIME_IF_NOTHING_TO_DO_IN_MS } from './constants' import { ResolveRequest, ResolveRequestPrompts } from '@aitube/clapper-services' +import { useMonitor } from '../monitor/useMonitor' +import { useRenderer } from '../renderer' +import { getDefaultResolveRequestPrompts } from './getDefaultResolveRequestPrompts' +import { resolve } from '../api/resolve' export const useResolver = create((set, get) => ({ ...getDefaultResolverState(), @@ -59,13 +74,32 @@ export const useResolver = create((set, get) => ({ * @returns */ runLoop: async (): Promise => { + const renderer: RendererState = useRenderer.getState() + const timeline: TimelineStore = useTimeline.getState() + + // note: we read the rendering strategies from the renderer, not from the settings + // that's because ultimately it is Clapper and the Renderer module which decide which + // strategy to use, and override user settings (eg. playback takes precedence over + // whatever the user set) const { imageRenderingStrategy, videoRenderingStrategy, soundRenderingStrategy, voiceRenderingStrategy, musicRenderingStrategy, - } = useSettings.getState() + }: RenderingStrategies = renderer + + // Tells how many segments should be renderer in advanced during playback, for each segment category + const { + imageBufferSize, + videoBufferSize, + soundBufferSize, + voiceBufferSize, + musicBufferSize, + }: RenderingBufferSizes = renderer + + // TODO @julian-hf: we have the buffer sizes, but we don't yet have a way to tell how much the buffer + // are filled. This could be done simply by counting each time we run the loop const runLoopAgain = ( waitTimeIfNothingToDoInMs = DEFAULT_WAIT_TIME_IF_NOTHING_TO_DO_IN_MS @@ -79,13 +113,13 @@ export const useResolver = create((set, get) => ({ // otherwise we won't be able to get the status of current tasks // console.log(`useResolver.runLoop()`) - const timelineState: TimelineStore = useTimeline.getState() + const { visibleSegments, loadedSegments, segments: allSegments, resolveSegment, - } = timelineState + } = timeline // ------------------------------------------------------------------------------------------------ // @@ -164,7 +198,9 @@ export const useResolver = create((set, get) => ({ if ( s.visibility === SegmentVisibility.HIDDEN && - videoRenderingStrategy !== RenderingStrategy.ON_SCREEN_THEN_ALL + videoRenderingStrategy !== RenderingStrategy.ON_SCREEN_THEN_ALL && + videoRenderingStrategy !== + RenderingStrategy.BUFFERED_PLAYBACK_STREAMING ) { continue } else if ( @@ -212,7 +248,9 @@ export const useResolver = create((set, get) => ({ if ( s.visibility === SegmentVisibility.HIDDEN && - imageRenderingStrategy !== RenderingStrategy.ON_SCREEN_THEN_ALL + imageRenderingStrategy !== RenderingStrategy.ON_SCREEN_THEN_ALL && + imageRenderingStrategy !== + RenderingStrategy.BUFFERED_PLAYBACK_STREAMING ) { continue } else if ( @@ -258,7 +296,9 @@ export const useResolver = create((set, get) => ({ if ( s.visibility === SegmentVisibility.HIDDEN && - voiceRenderingStrategy !== RenderingStrategy.ON_SCREEN_THEN_ALL + voiceRenderingStrategy !== RenderingStrategy.ON_SCREEN_THEN_ALL && + voiceRenderingStrategy !== + RenderingStrategy.BUFFERED_PLAYBACK_STREAMING ) { continue } else if ( @@ -301,7 +341,9 @@ export const useResolver = create((set, get) => ({ if ( s.visibility === SegmentVisibility.HIDDEN && - soundRenderingStrategy !== RenderingStrategy.ON_SCREEN_THEN_ALL + soundRenderingStrategy !== RenderingStrategy.ON_SCREEN_THEN_ALL && + soundRenderingStrategy !== + RenderingStrategy.BUFFERED_PLAYBACK_STREAMING ) { continue } else if ( @@ -344,7 +386,9 @@ export const useResolver = create((set, get) => ({ if ( s.visibility === SegmentVisibility.HIDDEN && - musicRenderingStrategy !== RenderingStrategy.ON_SCREEN_THEN_ALL + musicRenderingStrategy !== RenderingStrategy.ON_SCREEN_THEN_ALL && + musicRenderingStrategy !== + RenderingStrategy.BUFFERED_PLAYBACK_STREAMING ) { continue } else if ( @@ -432,11 +476,54 @@ export const useResolver = create((set, get) => ({ * This will generate for instance an image and a voice * for the entity, based on the entity description. * - * @param segment + * @param entity * @returns */ resolveEntity: async (entity: ClapEntity): Promise => { - throw new Error('Not implemented (TODO by @Julian)') + // note: if the entity has an image id or an audio id, we proceeed anyway. + // that way the parent function can decide to re-generate the entity at any time. + + // we create a segment that will only be used to create an identity + // picture of our character + const segment: TimelineSegment = await clapSegmentToTimelineSegment( + newSegment({ + category: ClapSegmentCategory.STORYBOARD, + prompt: getCharacterReferencePrompt(entity), + }) + ) + + let imageId = '' + + try { + const newSegmentData = await resolve({ + segment, + prompts: getDefaultResolveRequestPrompts({ + image: { positive: segment.prompt }, + }), + }) + imageId = `${newSegmentData.assetUrl || ''}` + } catch (err) { + console.error(`useResolver.resolveEntity(): error: ${err}`) + } + + /* + try { + const newSegmentData = await resolve({ + segment, + prompts: getDefaultResolveRequestPrompts({ + TODO : do the audio! + image: { positive: segment.prompt } + }), + }) + imageId = `${newSegmentData.assetUrl || ''}` + } catch (err) { + console.error(`useResolver.resolveEntity(): error: ${err}`) + } + */ + + Object.assign(entity, { imageId }) + + return entity }, /** @@ -463,7 +550,6 @@ export const useResolver = create((set, get) => ({ // that's because resolveSegment is 100% asynchronous, // meaning it might be called on invisible segments too! const { - meta, entityIndex, segments: allSegments, trackSilentChangeInSegment, @@ -542,67 +628,50 @@ export const useResolver = create((set, get) => ({ // note: not all AI models will support those parameters. // in 2024, even the "best" proprietary video models like Sora, Veo, Kling, Gen-3, Dream Machine etc.. // don't support voice input for lip syncing, for instance. - const prompts: ResolveRequestPrompts = { + const prompts: ResolveRequestPrompts = getDefaultResolveRequestPrompts({ image: { // the "identification picture" of the character, if available - identity: `${mainCharacterEntity?.imageId || ''}`, + identity: mainCharacterEntity?.imageId, positive: positiveImagePrompt, negative: negativeImagePrompt, }, video: { // image to animate - image: `${storyboard?.assetUrl || ''}`, + image: storyboard?.assetUrl, // dialogue line to lip-sync - voice: `${dialogue?.assetUrl || ''}`, + voice: dialogue?.assetUrl, }, voice: { - identity: `${mainCharacterEntity?.audioId || ''}`, + identity: mainCharacterEntity?.audioId, positive: positiveVoicePrompt, - negative: '', + // negative: '', }, audio: { positive: positiveAudioPrompt, - negative: '', + // negative: '', }, music: { positive: positiveMusicPrompt, - negative: '', + // negative: '', }, - } - - const serializableSegment = { ...segment } - // we delete things that cannot be serialized properly - delete serializableSegment.scene - delete serializableSegment.audioBuffer - serializableSegment.textures = {} - - const request: ResolveRequest = { - settings, - segment: serializableSegment, - segments, - entities, - speakingCharactersIds, - generalCharactersIds, - mainCharacterId, - mainCharacterEntity, - meta, - prompts, - } + }) try { - const res = await fetch('/api/resolve', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(request), - }) - // console.log(`useResolver.resolveSegment(): result from /api.render:`, res) - // note: this isn't really a "full" TimelineSegment, // it will miss some data that cannot be serialized - const newSegmentData = (await res.json()) as TimelineSegment + const newSegmentData = await resolve({ + segment, + segments, + entities, + speakingCharactersIds, + generalCharactersIds, + mainCharacterId, + mainCharacterEntity, + prompts, + }) + + // console.log(`useResolver.resolveSegment(): result from /api.render:`, res) // console.log(`useResolver.resolveSegment(): newSegmentData`, newSegmentData) diff --git a/packages/clap/src/types.ts b/packages/clap/src/types.ts index d030e1e0..b7d00a29 100644 --- a/packages/clap/src/types.ts +++ b/packages/clap/src/types.ts @@ -34,7 +34,12 @@ export enum ClapSegmentCategory { // otherwise some web browsers might not be able to display it VIDEO = "VIDEO", - // a storyboard image, eg. jpg, png, webp, heic (note: most of the apps compatible with .clap prefer to work with jpg and png) + // the name of this field is a bit confusing, + // we are going to rename it to IMAGE = "IMAGE" + // (because the storyboard is the board and timeline of drawing and picutes) + // + // format can currently be jpg, png, webp, heic + // (note: most of the apps compatible with .clap prefer to work with jpg and png) STORYBOARD = "STORYBOARD", // a transition between two shots (eg. a cross-fade) diff --git a/packages/clapper-services/src/index.ts b/packages/clapper-services/src/index.ts index 9dd88083..366109a9 100644 --- a/packages/clapper-services/src/index.ts +++ b/packages/clapper-services/src/index.ts @@ -153,6 +153,8 @@ export { ActiveSegments, UpcomingSegments, BufferedSegments, + RenderingStrategies, + RenderingBufferSizes, RendererState, RendererControls, RendererStore, diff --git a/packages/clapper-services/src/monitor.ts b/packages/clapper-services/src/monitor.ts index 15945c33..d688aa30 100644 --- a/packages/clapper-services/src/monitor.ts +++ b/packages/clapper-services/src/monitor.ts @@ -3,6 +3,7 @@ export type MonitorState = { lastTimelineUpdateAtInMs: number isPlaying: boolean staticVideoRef?: HTMLVideoElement + isEmbedded: boolean } export type MonitorControls = { @@ -10,6 +11,8 @@ export type MonitorControls = { setStaticVideoRef: (staticVideoRef: HTMLVideoElement) => void + setIsEmbedded: (isEmbedded: boolean) => void + checkIfPlaying: () => boolean /** * Play/pause the project timeline (video and audio) diff --git a/packages/clapper-services/src/renderer.ts b/packages/clapper-services/src/renderer.ts index 3ffe797b..f6b43dd4 100644 --- a/packages/clapper-services/src/renderer.ts +++ b/packages/clapper-services/src/renderer.ts @@ -1,4 +1,28 @@ -import { TimelineSegment } from "@aitube/timeline" +import { RenderingStrategy, TimelineSegment } from "@aitube/timeline" + +/** + * those are the currently active rendering strategies determined by the renderer + * this is different from the image rendering preferences (what the user has set) + */ +export type RenderingStrategies = { + imageRenderingStrategy: RenderingStrategy + videoRenderingStrategy: RenderingStrategy + soundRenderingStrategy: RenderingStrategy + voiceRenderingStrategy: RenderingStrategy + musicRenderingStrategy: RenderingStrategy +} + +/** + * Tells how many segments should be renderer in advanced during playback, + * for each segment category + */ +export type RenderingBufferSizes = { + imageBufferSize: number + videoBufferSize: number + soundBufferSize: number + voiceBufferSize: number + musicBufferSize: number +} export type ActiveSegments = { activeSegmentsCacheKey: string @@ -17,7 +41,8 @@ export type UpcomingSegments = { export type BufferedSegments = ActiveSegments & UpcomingSegments -export type RendererState = { +export type RendererState = RenderingStrategies & RenderingBufferSizes & { + bufferedSegments: BufferedSegments // various helpers to manage buffering, @@ -35,6 +60,8 @@ export type RendererState = { export type RendererControls = { + setUserDefinedRenderingStrategies: (strategies: RenderingStrategies) => void + // used to clear the renderer eg. when we load a new project clear: () => void diff --git a/packages/clapper-services/src/resolver.ts b/packages/clapper-services/src/resolver.ts index a0849d44..0cc19277 100644 --- a/packages/clapper-services/src/resolver.ts +++ b/packages/clapper-services/src/resolver.ts @@ -11,7 +11,7 @@ export type ResolverState = { // request have have already be sent to the API providers // will still be honored, which is why the number of pending // requests won't drop to 0 immediately - isPaused: boolean + isPaused: boolean defaultParallelismQuotas: { video: number diff --git a/packages/timeline/src/types/rendering.ts b/packages/timeline/src/types/rendering.ts index 58ec167b..616e6f91 100644 --- a/packages/timeline/src/types/rendering.ts +++ b/packages/timeline/src/types/rendering.ts @@ -23,6 +23,19 @@ export enum RenderingStrategy { // !! this is hardcore! only GPU-rich people shoud use this feature! !! // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ON_SCREEN_THEN_ALL = "ON_SCREEN_THEN_ALL", + + // a special mode which only renders what is under the current playback cursor, + // plus also pre-generates the following videos according to a given buffer size, + // to try to ensure smooth streaming (smooth loading is only possible of the hardware + // is fast other. Otherwise we will have to display a loader or progression bar). + // + // - if the user watches the whole video, then it will cost as much as the ON_SCREEN_THEN_ALL option + // + // - but if the user only watches 10%, then they only have to pay for 10% (+ the buffer) + // + // - if the user clicks somewhere in the timeline during playback, the it buffers again, + // but they also saves money (it won't compute the skipped segments) + BUFFERED_PLAYBACK_STREAMING = "BUFFERED_PLAYBACK_STREAMING" } export type SegmentResolver = (segment: TimelineSegment) => Promise diff --git a/packages/timeline/src/utils/parseRenderingStrategy.ts b/packages/timeline/src/utils/parseRenderingStrategy.ts index 204f9524..48bd73e2 100644 --- a/packages/timeline/src/utils/parseRenderingStrategy.ts +++ b/packages/timeline/src/utils/parseRenderingStrategy.ts @@ -24,6 +24,8 @@ export function parseRenderingStrategy(input: any, defaultStrategy?: RenderingSt } else if (unknownString === "on_screen_then_all") { strategy = RenderingStrategy.ON_SCREEN_THEN_ALL + } else if (unknownString === "buffered_playback_streaming") { + strategy = RenderingStrategy.BUFFERED_PLAYBACK_STREAMING } else { strategy = RenderingStrategy.ON_DEMAND }