work on live generation and entity creation

packages/app/src/app/main.tsx

@@ -89,7 +89,6 @@ function MainContent({ mode }: { mode: ClapperIntegrationMode }) {
     setHasBetaAccess(hasBetaAccess)
   }, [hasBetaAccess, setHasBetaAccess])
 
-
   useQueryStringLoader()
 
   const iframeLayout = (

packages/app/src/components/toolbars/top-menu/file/useQueryStringLoader.ts

@@ -6,12 +6,13 @@ import { useQueryStringParams } from '@/lib/hooks'
 import { useAssistant, useIO } from '@/services'
 
 export function useQueryStringLoader() {
-  const { clapUrl, startAt, interactive, prompt, imageStrategy } = useQueryStringParams({
-    // clapUrl: `/samples/test.clap`,
-    // clapUrl: `/samples/Afterglow%20v10%20X%20Rewrite%20Bryan%20E.%20Harris%202023.clap`,
-  })
+  const { clapUrl, startAt, interactive, prompt, imageStrategy } =
+    useQueryStringParams({
+      // clapUrl: `/samples/test.clap`,
+      // clapUrl: `/samples/Afterglow%20v10%20X%20Rewrite%20Bryan%20E.%20Harris%202023.clap`,
+    })
 
-  const processUserMessage = useAssistant(s => s.processUserMessage)
+  const processUserMessage = useAssistant((s) => s.processUserMessage)
   const openClapUrl = useIO((s) => s.openClapUrl)
 
   useEffect(() => {

packages/app/src/lib/hooks/useQueryStringParams.ts

@@ -2,34 +2,46 @@ import { useSearchParams } from 'next/navigation'
 import { getValidNumber } from '../utils'
 import { RenderingStrategy } from '@aitube/timeline'
 
-export function useQueryStringParams({
-  interactive: defaultInteractive = false,
-  startAt: defaultStartAt = 0,
-  prompt: defaultPrompt = '',
-  clapUrl: defaultClapUrl = '',
-  imageStrategy: defaultImageStrategy = RenderingStrategy.ON_DEMAND
-}: {
-  interactive?: boolean
-  startAt?: number
-  prompt?: string
-  clapUrl?: string
-  imageStrategy?: RenderingStrategy
-} = {
-  interactive: false,
-  startAt: 0,
-  prompt: '',
-  clapUrl: '',
-  imageStrategy: RenderingStrategy.ON_DEMAND
-}) {
+export function useQueryStringParams(
+  {
+    interactive: defaultInteractive = false,
+    startAt: defaultStartAt = 0,
+    prompt: defaultPrompt = '',
+    clapUrl: defaultClapUrl = '',
+    imageStrategy: defaultImageStrategy = RenderingStrategy.ON_DEMAND,
+  }: {
+    interactive?: boolean
+    startAt?: number
+    prompt?: string
+    clapUrl?: string
+    imageStrategy?: RenderingStrategy
+  } = {
+    interactive: false,
+    startAt: 0,
+    prompt: '',
+    clapUrl: '',
+    imageStrategy: RenderingStrategy.ON_DEMAND,
+  }
+) {
   const searchParams = useSearchParams()
 
   const prompt = (searchParams?.get('prompt') as string) || defaultPrompt
 
-  const imageStrategy = (searchParams?.get('imageStrategy') as RenderingStrategy) || defaultImageStrategy
+  const imageStrategy =
+    (searchParams?.get('imageStrategy') as RenderingStrategy) ||
+    defaultImageStrategy
 
-  const startAt = getValidNumber(`${(searchParams?.get('startAt') as string) || defaultStartAt}`.trim(), 0, Number.MAX_VALUE, 0)
+  const startAt = getValidNumber(
+    `${(searchParams?.get('startAt') as string) || defaultStartAt}`.trim(),
+    0,
+    Number.MAX_VALUE,
+    0
+  )
 
-  const interactive = `${(searchParams?.get('interactive') as string) || defaultInteractive}`.trim().toLowerCase() === 'true'
+  const interactive =
+    `${(searchParams?.get('interactive') as string) || defaultInteractive}`
+      .trim()
+      .toLowerCase() === 'true'
 
   const clapUrl = (searchParams?.get('clap') as string) || defaultClapUrl
 

packages/app/src/services/api/resolve.ts

@@ -0,0 +1,80 @@
+'use client'
+
+import {
+  clapSegmentToTimelineSegment,
+  TimelineSegment,
+  TimelineStore,
+  useTimeline,
+} from '@aitube/timeline'
+import {
+  ResolveRequest,
+  ResolveRequestPrompts,
+  SettingsStore,
+} from '@aitube/clapper-services'
+import { useSettings } from '../settings'
+import { ClapSegmentCategory, newSegment } from '@aitube/clap'
+import { getDefaultResolveRequestPrompts } from '../resolver/getDefaultResolveRequestPrompts'
+
+export async function resolve(
+  req: Partial<ResolveRequest>
+): Promise<TimelineSegment> {
+  const { getRequestSettings }: SettingsStore = useSettings.getState()
+  const { meta }: TimelineStore = useTimeline.getState()
+
+  const defaultTimelineSegment: TimelineSegment =
+    await clapSegmentToTimelineSegment(
+      newSegment({
+        category: ClapSegmentCategory.STORYBOARD,
+      })
+    )
+
+  const segment: TimelineSegment = {
+    ...defaultTimelineSegment,
+    ...req.segment,
+
+    // we omit things that cannot be serialized
+    scene: undefined,
+    audioBuffer: undefined,
+    textures: {},
+  }
+
+  const request: ResolveRequest = {
+    settings: getRequestSettings(),
+    segment,
+
+    segments: Array.isArray(req.segments)
+      ? req.segments.map((s) => ({
+          ...s,
+
+          // we omit things that cannot be serialized
+          scene: undefined,
+          audioBuffer: undefined,
+          textures: {},
+        }))
+      : [],
+
+    entities: req.entities ? req.entities : {},
+    speakingCharactersIds: Array.isArray(req.speakingCharactersIds)
+      ? req.speakingCharactersIds
+      : [],
+    generalCharactersIds: Array.isArray(req.generalCharactersIds)
+      ? req.generalCharactersIds
+      : [],
+    mainCharacterId: req.mainCharacterId || undefined,
+    mainCharacterEntity: req.mainCharacterEntity || undefined,
+    meta,
+    prompts: getDefaultResolveRequestPrompts(req.prompts),
+  }
+
+  const res = await fetch('/api/resolve', {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify(request),
+  })
+
+  const newSegmentData = (await res.json()) as TimelineSegment
+
+  return newSegmentData
+}

packages/app/src/services/monitor/getDefaultMonitorState.ts

@@ -6,6 +6,7 @@ export function getDefaultMonitorState(): MonitorState {
     lastTimelineUpdateAtInMs: 0,
     isPlaying: false,
     staticVideoRef: undefined,
+    isEmbedded: false,
   }
 
   return state

packages/app/src/services/monitor/useMonitor.ts

@@ -49,6 +49,12 @@ export const useMonitor = create<MonitorStore>((set, get) => ({
     })
   },
 
+  setIsEmbedded: (isEmbedded: boolean) => {
+    set({
+      isEmbedded,
+    })
+  },
+
   checkIfPlaying: (): boolean => {
     return get().isPlaying
   },

packages/app/src/services/renderer/getDefaultRendererState.ts

@@ -1,9 +1,38 @@
-import { RendererState } from '@aitube/clapper-services'
+import { RenderingStrategy } from '@aitube/timeline'
+
+import {
+  RendererState,
+  RenderingBufferSizes,
+  RenderingStrategies,
+} from '@aitube/clapper-services'
 
 import { getDefaultBufferedSegments } from './getDefaultBufferedSegments'
 
 export function getDefaultRendererState(): RendererState {
+  const renderingStrategies: RenderingStrategies = {
+    imageRenderingStrategy: RenderingStrategy.BUFFERED_PLAYBACK_STREAMING,
+    videoRenderingStrategy: RenderingStrategy.BUFFERED_PLAYBACK_STREAMING,
+    soundRenderingStrategy: RenderingStrategy.BUFFERED_PLAYBACK_STREAMING,
+    voiceRenderingStrategy: RenderingStrategy.BUFFERED_PLAYBACK_STREAMING,
+    musicRenderingStrategy: RenderingStrategy.BUFFERED_PLAYBACK_STREAMING,
+  }
+
+  /**
+   * Tells how many segments should be renderer in advanced during playback, for each segment category
+   */
+  const bufferSizes: RenderingBufferSizes = {
+    imageBufferSize: 32,
+    videoBufferSize: 32,
+    soundBufferSize: 32,
+    voiceBufferSize: 32,
+    musicBufferSize: 8, // music segments are longer, so no need to generate that many
+  }
+
   const state: RendererState = {
+    ...bufferSizes,
+
+    ...renderingStrategies,
+
     bufferedSegments: getDefaultBufferedSegments(),
 
     dataUriBuffer1: undefined,

packages/app/src/services/renderer/useRenderLoop.ts

@@ -10,6 +10,8 @@ import { useRenderer } from './useRenderer'
 import { useAudio } from '@/services/audio/useAudio'
 import { useMonitor } from '../monitor/useMonitor'
 import { useEffect, useRef } from 'react'
+import { useSettings } from '../settings'
+import { set } from 'date-fns'
 
 /**
  * Runs a rendering loop
@@ -34,6 +36,33 @@ export function useRenderLoop(): void {
 
   const timeoutRef = useRef<NodeJS.Timeout>()
 
+  const setUserDefinedRenderingStrategies = useRenderer(
+    (s) => s.setUserDefinedRenderingStrategies
+  )
+
+  // those are the currently active rendering strategies determined by the renderer
+  // this is different from the image rendering preferences (what the user has set)
+  const imageRenderingStrategy = useSettings((s) => s.imageRenderingStrategy)
+  const videoRenderingStrategy = useSettings((s) => s.videoRenderingStrategy)
+  const soundRenderingStrategy = useSettings((s) => s.soundRenderingStrategy)
+  const voiceRenderingStrategy = useSettings((s) => s.voiceRenderingStrategy)
+  const musicRenderingStrategy = useSettings((s) => s.musicRenderingStrategy)
+  useEffect(() => {
+    setUserDefinedRenderingStrategies({
+      imageRenderingStrategy,
+      videoRenderingStrategy,
+      soundRenderingStrategy,
+      voiceRenderingStrategy,
+      musicRenderingStrategy,
+    })
+  }, [
+    imageRenderingStrategy,
+    videoRenderingStrategy,
+    soundRenderingStrategy,
+    voiceRenderingStrategy,
+    musicRenderingStrategy,
+  ])
+
   // used to control transitions between buffers
   useEffect(() => {
     clearTimeout(timeoutRef.current)

packages/app/src/services/renderer/useRenderer.ts

@@ -2,12 +2,23 @@
 
 import { create } from 'zustand'
 import { ClapOutputType, ClapSegmentCategory } from '@aitube/clap'
-import { BufferedSegments, RendererStore } from '@aitube/clapper-services'
-import { TimelineStore, useTimeline, TimelineSegment } from '@aitube/timeline'
+import {
+  BufferedSegments,
+  RendererStore,
+  RenderingStrategies,
+} from '@aitube/clapper-services'
+import {
+  TimelineStore,
+  useTimeline,
+  TimelineSegment,
+  RenderingStrategy,
+} from '@aitube/timeline'
 
 import { getDefaultRendererState } from './getDefaultRendererState'
 import { getSegmentCacheKey } from './getSegmentCacheKey'
 import { getDefaultBufferedSegments } from './getDefaultBufferedSegments'
+import { useMonitor } from '../monitor/useMonitor'
+import { useSettings } from '../settings'
 
 export const useRenderer = create<RendererStore>((set, get) => ({
   ...getDefaultRendererState(),
@@ -18,6 +29,42 @@ export const useRenderer = create<RendererStore>((set, get) => ({
     })
   },
 
+  setUserDefinedRenderingStrategies: ({
+    imageRenderingStrategy: userDefinedImageRenderingStrategy,
+    videoRenderingStrategy: userDefinedVideoRenderingStrategy,
+    soundRenderingStrategy: userDefinedSoundRenderingStrategy,
+    voiceRenderingStrategy: userDefinedVoiceRenderingStrategy,
+    musicRenderingStrategy: userDefinedMusicRenderingStrategy,
+  }: RenderingStrategies) => {
+    // if the monitor is embedded, the imageRenderingStrategy is temporary bypassed
+    // to try to render all the segments in advance, to create a buffer
+    // this is a potentially expensive solution, so we might want to put
+    // some limits to that ex. the first 64 or something
+    const { isEmbedded } = useMonitor.getState()
+
+    // What we are doing here is that when we are in "embedded" video player mode,
+    // we bypass the image rendering strategy to render all the segments in advance
+    const imageRenderingStrategy = isEmbedded
+      ? RenderingStrategy.BUFFERED_PLAYBACK_STREAMING
+      : userDefinedImageRenderingStrategy
+    const videoRenderingStrategy =
+      /* isEmbedded ? RenderingStrategy.ON_SCREEN_THEN_ALL : */ userDefinedVideoRenderingStrategy
+    const soundRenderingStrategy =
+      /* isEmbedded ? RenderingStrategy.ON_SCREEN_THEN_ALL : */ userDefinedSoundRenderingStrategy
+    const voiceRenderingStrategy =
+      /* isEmbedded ? RenderingStrategy.ON_SCREEN_THEN_ALL : */ userDefinedVoiceRenderingStrategy
+    const musicRenderingStrategy =
+      /* isEmbedded ? RenderingStrategy.ON_SCREEN_THEN_ALL : */ userDefinedMusicRenderingStrategy
+
+    set({
+      imageRenderingStrategy,
+      videoRenderingStrategy,
+      soundRenderingStrategy,
+      voiceRenderingStrategy,
+      musicRenderingStrategy,
+    })
+  },
+
   // this will be called at 60 FPS - and yes, it is expensive
   // we could probably improve things by using a temporal tree index
   renderLoop: (jumpedSomewhere?: boolean): BufferedSegments => {

packages/app/src/services/resolver/getDefaultResolveRequestPrompts.ts

@@ -0,0 +1,37 @@
+import { ResolveRequestPrompts } from '@aitube/clapper-services'
+
+type DeepPartial<T> = T extends object
+  ? {
+      [P in keyof T]?: DeepPartial<T[P]>
+    }
+  : T
+
+export function getDefaultResolveRequestPrompts(
+  partials: DeepPartial<ResolveRequestPrompts> = {}
+): ResolveRequestPrompts {
+  const defaultPrompts: ResolveRequestPrompts = {
+    image: {
+      identity: `${partials?.image?.identity || ''}`,
+      positive: `${partials?.image?.positive || ''}`,
+      negative: `${partials?.image?.negative || ''}`,
+    },
+    video: {
+      image: `${partials?.video?.image || ''}`,
+      voice: `${partials?.video?.voice || ''}`,
+    },
+    voice: {
+      identity: `${partials?.voice?.identity || ''}`,
+      positive: `${partials?.voice?.positive || ''}`,
+      negative: `${partials?.voice?.negative || ''}`,
+    },
+    audio: {
+      positive: `${partials?.audio?.positive || ''}`,
+      negative: `${partials?.audio?.negative || ''}`,
+    },
+    music: {
+      positive: `${partials?.music?.positive || ''}`,
+      negative: `${partials?.music?.negative || ''}`,
+    },
+  }
+  return defaultPrompts
+}