fix storybuilding

Author: jbilcke-hf

package-lock.json

@@ -38,7 +38,7 @@
   "dependencies": {
     "@aitube/broadway": "0.1.3-1",
     "@aitube/clap": "0.1.2",
-    "@aitube/clapper-services": "0.1.5-1",
+    "@aitube/clapper-services": "0.1.5-2",
     "@aitube/engine": "0.1.2",
     "@aitube/timeline": "0.1.2-4",
     "@fal-ai/serverless-client": "^0.13.0",

package.json

@@ -27,7 +27,7 @@ import {
   AssistantMessage,
   AssistantRequest,
   AssistantSceneSegment,
-  AssistantStorySentence,
+  AssistantStoryBlock,
   ComputeProvider,
   ChatEventVisibility,
 } from '@aitube/clapper-services'
@@ -124,17 +124,17 @@ export async function askAnyAssistant({
     ['human', humanTemplate],
   ])
 
-  //const storySentences: AssistantStorySentence[] = fullScene.split(/(?:. |\n)/).map(storySentence => {
+  //const storyBlocks: AssistantStorySentence[] = fullScene.split(/(?:. |\n)/).map(storySentence => {
   //})
 
-  const storySentences: AssistantStorySentence[] = [
+  const storyBlocks: AssistantStoryBlock[] = [
     {
-      sentenceId: 0,
-      sentence: fullScene,
+      blockId: 0,
+      block: fullScene,
     },
     {
-      sentenceId: 1,
-      sentence: actionLine,
+      blockId: 1,
+      block: actionLine,
     },
   ]
 
@@ -151,7 +151,7 @@ export async function askAnyAssistant({
   // TODO put this into a type
   const inputData: AssistantInput = {
     directorRequest: prompt,
-    storySentences,
+    storyBlocks,
     sceneSegments,
   }
 
@@ -162,7 +162,7 @@ export async function askAnyAssistant({
   let assistantMessage: AssistantMessage = {
     comment: '',
     action: AssistantAction.NONE,
-    updatedStorySentences: [],
+    updatedStoryBlocks: [],
     updatedSceneSegments: [],
   }
   try {
@@ -199,11 +199,14 @@ export async function askAnyAssistant({
           }
         ),
     })
+    // console.log('Lanchain success on the first time! rawResponse:', rawResponse)
 
     assistantMessage = parseLangChainResponse(rawResponse)
+    // console.log('assistantMessage:', assistantMessage)
   } catch (err) {
     // LangChain failure (this happens quite often, actually)
 
+    // console.log(`Langchain error:\n${err}`)
     let errorPlainText = `${err}`
 
     // Markdown formatting failure
@@ -229,7 +232,7 @@ export async function askAnyAssistant({
         assistantMessage.comment = errorPlainText || ''
         assistantMessage.action = AssistantAction.NONE
         assistantMessage.updatedSceneSegments = []
-        assistantMessage.updatedStorySentences = []
+        assistantMessage.updatedStoryBlocks = []
         if (!errorPlainText) {
           throw new Error(
             `failed to repair the output from LangChain (empty string)`

src/app/api/assistant/askAnyAssistant.ts

@@ -8,7 +8,7 @@ export function parseLangChainResponse(
   const assistantMessage: AssistantMessage = {
     comment: '',
     action: AssistantAction.NONE,
-    updatedStorySentences: [],
+    updatedStoryBlocks: [],
     updatedSceneSegments: [],
   }
   /*
@@ -74,6 +74,22 @@ export function parseLangChainResponse(
         })
       }
     }
+
+    i = 0
+    for (const block of langChainResponse.updatedStoryBlocks || []) {
+      i++
+      const blockId = isValidNumber(block.blockId) ? block.blockId! : i
+
+      // TODO: rename block.block to block.text or block.content it would be better
+      const textBlock = `${block.block || ''}`
+      // we assume no prompt is an error
+      if (textBlock) {
+        assistantMessage.updatedStoryBlocks.push({
+          blockId,
+          block: textBlock,
+        })
+      }
+    }
   }
 
   return assistantMessage

src/app/api/assistant/parseLangChainResponse.ts

@@ -45,20 +45,22 @@ export const zAssistantSceneSegment = z.object({
 })
 
 export const zAssistantStorySentence = z.object({
-  sentenceId: z.number().describe('unique identifier'),
-  sentence: z
+  blockId: z.number().describe('unique identifier for the story block'),
+  block: z
     .string()
-    .describe('A sentence extracted from the story plain-text.'),
+    .describe(
+      'A text block extracted from the story plain text. Can be about the general story, a specific scene like the current one.'
+    ),
 })
 
 export const zAssistantMessage = z.object({
   comment: z
     .string()
     .describe(
-      'A free-form comment and chat message, allowing you to answer to the user directly.'
+      'A free-form comment and chat message, allowing the assistant (aka you) to directly address, answer or question the director (aka user).'
     ),
   action: zAssistantAction,
-  updatedStorySentences: z.array(zAssistantStorySentence),
+  updatedStoryBlocks: z.array(zAssistantStorySentence),
   updatedSceneSegments: z.array(zAssistantSceneSegment),
 })
 

src/app/api/assistant/parser.ts

@@ -1,6 +1,6 @@
 import { NextResponse, NextRequest } from 'next/server'
 
-import { AssistantRequest, AssistantMessage } from '@aitube/clapper-services'
+import { AssistantRequest } from '@aitube/clapper-services'
 import { askAnyAssistant } from './askAnyAssistant'
 
 export async function POST(req: NextRequest) {