From 553016cbf96fdb1c80b2b7c0ab8ebc4e2662127d Mon Sep 17 00:00:00 2001
From: Julian Bilcke <julian.bilcke@huggingface.co>
Date: Sat, 24 Aug 2024 19:48:45 +0200
Subject: [PATCH] work on the entity resolver

---
 .../app/src/services/resolver/useResolver.ts  | 95 +++++++++++--------
 packages/clapper-services/src/resolver.ts     | 12 ++-
 2 files changed, 65 insertions(+), 42 deletions(-)

diff --git a/packages/app/src/services/resolver/useResolver.ts b/packages/app/src/services/resolver/useResolver.ts
index 4a391368..4cf6e769 100644
--- a/packages/app/src/services/resolver/useResolver.ts
+++ b/packages/app/src/services/resolver/useResolver.ts
@@ -479,49 +479,68 @@ export const useResolver = create<ResolverStore>((set, get) => ({
    * @param entity
    * @returns
    */
-  resolveEntity: async (entity: ClapEntity): Promise<ClapEntity> => {
-    // note: if the entity has an image id or an audio id, we proceeed anyway.
+  resolveEntity: async (
+    entity: ClapEntity,
+    field?: 'face' | 'voice'
+  ): Promise<ClapEntity> => {
+    // note: if the entity has an image id or an audio id, we proceed anyway.
+    
     // that way the parent function can decide to re-generate the entity at any time.
 
-    // we create a segment that will only be used to create an identity
-    // picture of our character
-    const segment: TimelineSegment = await clapSegmentToTimelineSegment(
-      newSegment({
-        category: ClapSegmentCategory.STORYBOARD,
-        prompt: getCharacterReferencePrompt(entity),
-      })
-    )
-
-    let imageId = ''
-
-    try {
-      const newSegmentData = await resolve({
-        segment,
-        prompts: getDefaultResolveRequestPrompts({
-          image: { positive: segment.prompt },
-        }),
-      })
-      imageId = `${newSegmentData.assetUrl || ''}`
-    } catch (err) {
-      console.error(`useResolver.resolveEntity(): error: ${err}`)
+    if (!field || field === 'face') {
+      try {
+        const prompt = getCharacterReferencePrompt(entity)
+        const newSegmentData = await resolve({
+          segment: await clapSegmentToTimelineSegment(
+            newSegment({
+              category: ClapSegmentCategory.STORYBOARD,
+              prompt,
+            })
+          ),
+          prompts: getDefaultResolveRequestPrompts({
+            image: { positive: prompt },
+          }),
+        })
+        entity.imageId = `${newSegmentData.assetUrl || ''}`
+      } catch (err) {
+        console.error(
+          `useResolver.resolveEntity(): error when generating the face: ${err}`
+        )
+      }
     }
 
-    /*
-    try {
-      const newSegmentData = await resolve({
-        segment,
-        prompts: getDefaultResolveRequestPrompts({
-          TODO : do the audio!
-          image: { positive: segment.prompt }
-        }),
-      })
-      imageId = `${newSegmentData.assetUrl || ''}`
-    } catch (err) {
-      console.error(`useResolver.resolveEntity(): error: ${err}`)
+    if (!field || field === 'voice') {
+      try {
+        // we generate a random, novel voice
+        // TODO use the gender
+        const characterVoicePrompt = `A ${entity.age} years old ${entity.gender} is talking`
+
+        const sampleDialoguePrompt = `The quick brown fox jumps over the lazy dog.
+A number of panicked Europeans appear to have reckoned the
+wildly volatile, vulnerable, and tiny bitcoin market a
+preferable alternative to their own banking system, even
+temporarily, signals a serious widening of the cracks
+between the northern and southern E.U. countries in the
+wake of the euro-zone debt crisis.`
+
+        const newSegmentData = await resolve({
+          segment: await clapSegmentToTimelineSegment(
+            newSegment({
+              category: ClapSegmentCategory.DIALOGUE,
+              prompt: sampleDialoguePrompt,
+            })
+          ),
+          prompts: getDefaultResolveRequestPrompts({
+            voice: { positive: characterVoicePrompt },
+          }),
+        })
+        entity.audioId = `${newSegmentData.assetUrl || ''}`
+      } catch (err) {
+        console.error(
+          `useResolver.resolveEntity(): error when generating the voice: ${err}`
+        )
+      }
     }
-    */
-
-    Object.assign(entity, { imageId })
 
     return entity
   },
diff --git a/packages/clapper-services/src/resolver.ts b/packages/clapper-services/src/resolver.ts
index 0cc19277..acea6fcd 100644
--- a/packages/clapper-services/src/resolver.ts
+++ b/packages/clapper-services/src/resolver.ts
@@ -1,4 +1,4 @@
-import { ClapEntity } from "@aitube/clap"
+import { ClapEntity, ClapOutputType, ClapSegmentCategory } from "@aitube/clap"
 import { TimelineSegment } from "@aitube/timeline"
 
 export type ResolverState = {
@@ -48,13 +48,17 @@ export type ResolverControls = {
   /**
    * This resolve an entity (eg. a character or a location)
    * 
-   * This will generate for instance an image and a voice 
+   * This will generate for instance an image for the face and/or an audio file for the voice 
    * for the entity, based on the entity description.
    * 
-   * @param segment 
+   * By default, both the face and the voice will be generated.
+   * But this can be override
+   * 
+   * @param entity
+   * @param field (optional) 
    * @returns 
    */
-  resolveEntity: (entity: ClapEntity) => Promise<ClapEntity>
+  resolveEntity: (entity: ClapEntity, field?: 'face' | 'voice') => Promise<ClapEntity>
 
   /**
    * This resolve a segment