add new providers

packages/app/src/app/api/resolve/providers/bigmodel/callCogVideoX.ts

@@ -0,0 +1,75 @@
+import {
+  TaskResultResponse,
+  VideoGenerationParams,
+  VideoGenerationResponse,
+} from './types'
+
+/**
+ * Asynchronous function to generate a video using the CogVideoX API and retrieve the result.
+ * @param apiKey - The API key for authentication.
+ * @param params - The parameters for video generation.
+ * @returns A promise that resolves to the video generation result.
+ */
+export async function callCogVideoX(
+  apiKey: string,
+  params: VideoGenerationParams
+): Promise<TaskResultResponse> {
+  const baseUrl = 'https://open.bigmodel.cn/api/paas/v4'
+  const headers = {
+    'Content-Type': 'application/json',
+    Authorization: `Bearer ${apiKey}`,
+  }
+
+  try {
+    // Step 1: Initialize video generation
+    const generationResponse = await fetch(`${baseUrl}/videos/generations`, {
+      method: 'POST',
+      headers: headers,
+      body: JSON.stringify(params),
+    })
+
+    if (!generationResponse.ok) {
+      throw new Error(`HTTP error! status: ${generationResponse.status}`)
+    }
+
+    const generationData: VideoGenerationResponse =
+      await generationResponse.json()
+    const { id } = generationData
+
+    // Step 2: Poll for the task result
+    let taskResult: TaskResultResponse
+    do {
+      const resultResponse = await fetch(`${baseUrl}/async-result/${id}`, {
+        method: 'GET',
+        headers: headers,
+      })
+
+      if (!resultResponse.ok) {
+        throw new Error(`HTTP error! status: ${resultResponse.status}`)
+      }
+
+      taskResult = await resultResponse.json()
+
+      if (taskResult.task_status === 'PROCESSING') {
+        // Wait for 5 seconds before polling again
+        await new Promise((resolve) => setTimeout(resolve, 5000))
+      }
+    } while (taskResult.task_status === 'PROCESSING')
+
+    return taskResult
+  } catch (error) {
+    console.error('Error in video generation:', error)
+    throw error
+  }
+}
+
+// Example usage
+// const apiKey = 'your-api-key-here';
+// const params: VideoGenerationParams = {
+//   model: 'cogvideox',
+//   prompt: 'Peter Rabbit drives a small car, wandering on the road, with a face full of happiness and joy.'
+// };
+//
+// generateVideo(apiKey, params)
+//   .then(result => console.log(result))
+//   .catch(error => console.error(error));

packages/app/src/app/api/resolve/providers/bigmodel/index.ts

@@ -0,0 +1,56 @@
+import { TimelineSegment } from '@aitube/timeline'
+import { ResolveRequest } from '@aitube/clapper-services'
+import { ClapSegmentCategory } from '@aitube/clap'
+
+import { callCogVideoX } from './callCogVideoX'
+
+export async function resolveSegment(
+  request: ResolveRequest
+): Promise<TimelineSegment> {
+  if (!request.settings.bigModelApiKey) {
+    throw new Error(`Missing API key for "BigModel.cn"`)
+  }
+
+  const segment: TimelineSegment = request.segment
+
+  let model = request.settings.imageGenerationWorkflow.data || ''
+
+  if (request.segment.category === ClapSegmentCategory.VIDEO) {
+    model = request.settings.videoGenerationWorkflow.data || ''
+
+    /*
+    note 
+    if (!request.prompts.image.positive) {
+      console.error(
+        `resolveSegment: cannot resolve a storyboard with an empty prompt`
+      )
+      return segment
+    }
+    */
+
+    if (!request.prompts.video.image) {
+      throw new Error(
+        `cannot generate a video without a storyboard (the concept of Clapper is to use storyboards)`
+      )
+    }
+
+    // https://bigmodel.cn/dev/api#cogvideox
+    const result = await callCogVideoX(request.settings.bigModelApiKey, {
+      model: request.settings.videoGenerationWorkflow,
+      image_url: request.prompts.video.image,
+    })
+
+    const video = result.video_result.at(0)
+    if (!video) {
+      throw new Error(`Failed to generate at least one video`)
+    }
+
+    segment.assetUrl = video.url
+  } else {
+    throw new Error(
+      `Clapper doesn't support ${request.segment.category} generation for provider "Fal.ai". Please open a pull request with (working code) to solve this!`
+    )
+  }
+
+  return segment
+}

packages/app/src/app/api/resolve/providers/bigmodel/types.ts

@@ -0,0 +1,44 @@
+/**
+ * Represents the parameters for the video generation request.
+ */
+export type VideoGenerationParams = {
+  model: string
+  request_id?: string
+  user_id?: string
+} & (
+  | {
+      prompt?: string
+    }
+  | {
+      image_url?: string
+    }
+)
+
+/**
+ * Represents the response from the video generation request.
+ */
+export type VideoGenerationResponse = {
+  request_id: string
+  id: string
+  model: string
+  task_status: 'PROCESSING' | 'SUCCESS' | 'FAIL'
+}
+
+/**
+ * Represents the video result in the task result query.
+ */
+export type VideoResult = {
+  url: string
+  cover_image_url: string
+}
+
+/**
+ * Represents the response from the task result query.
+ */
+export type TaskResultResponse = {
+  model: string
+  video_result: VideoResult[]
+  task_status: 'PROCESSING' | 'SUCCESS' | 'FAIL'
+  request_id: string
+  id: string
+}

packages/app/src/app/api/resolve/providers/falai/index.ts

@@ -136,10 +136,7 @@ export async function resolveSegment(
       )
     }
 
-    const storyboard = request.segments.find(
-      (s) => s.category === ClapSegmentCategory.STORYBOARD
-    )
-    if (!storyboard) {
+    if (!request.prompts.video.image) {
       throw new Error(
         `cannot generate a video without a storyboard (the concept of Clapper is to use storyboards)`
       )
@@ -149,7 +146,7 @@ export async function resolveSegment(
       input: {
         ...getWorkflowInputValues(request.settings.videoGenerationWorkflow),
 
-        image_url: storyboard.assetUrl,
+        image_url: request.prompts.video.image,
 
         sync_mode: true,
         enable_safety_checker: request.settings.censorNotForAllAudiencesContent,

packages/app/src/app/api/resolve/providers/index.ts

@@ -1,10 +1,13 @@
-export { resolveSegment as resolveSegmentUsingHuggingFace } from './huggingface'
+export { resolveSegment as resolveSegmentUsingAiTube } from './aitube'
+export { resolveSegment as resolveSegmentUsingBigModel } from './stabilityai'
+export { resolveSegment as resolveSegmentUsingComfyDeploy } from './comfy-comfydeploy'
+export { resolveSegment as resolveSegmentUsingComfyIcu } from './comfy-comfyicu'
 export { resolveSegment as resolveSegmentUsingComfyReplicate } from './comfy-replicate'
-export { resolveSegment as resolveSegmentUsingReplicate } from './replicate'
 export { resolveSegment as resolveSegmentUsingComfyUI } from './comfyui'
-export { resolveSegment as resolveSegmentUsingComfyIcu } from './comfy-comfyicu'
-export { resolveSegment as resolveSegmentUsingComfyDeploy } from './comfy-comfydeploy'
 export { resolveSegment as resolveSegmentUsingFalAi } from './falai'
-export { resolveSegment as resolveSegmentUsingAiTube } from './aitube'
+export { resolveSegment as resolveSegmentUsingHuggingFace } from './huggingface'
+export { resolveSegment as resolveSegmentUsingLetzAi } from './letzai'
 export { resolveSegment as resolveSegmentUsingModelsLab } from './modelslab'
+export { resolveSegment as resolveSegmentUsingPiApi } from './piapi'
+export { resolveSegment as resolveSegmentUsingReplicate } from './replicate'
 export { resolveSegment as resolveSegmentUsingStabilityAi } from './stabilityai'

packages/app/src/app/api/resolve/providers/letzai/callCreateImage.ts

@@ -0,0 +1,52 @@
+import { ImageCreationParams, ImageCreationResponse } from './types'
+
+/**
+ * Asynchronous function to create an image using the LetzAI API.
+ * @param apiKey - The API key for authentication.
+ * @param params - The parameters for image creation.
+ * @returns A promise that resolves to the image creation result.
+ */
+export async function callCreateImage(
+  apiKey: string,
+  params: ImageCreationParams
+): Promise<ImageCreationResponse> {
+  const baseUrl = 'https://api.letz.ai'
+  const headers = {
+    'Content-Type': 'application/json',
+    Authorization: `Bearer ${apiKey}`,
+  }
+
+  try {
+    const response = await fetch(`${baseUrl}/images`, {
+      method: 'POST',
+      headers: headers,
+      body: JSON.stringify(params),
+    })
+
+    if (!response.ok) {
+      throw new Error(`HTTP error! status: ${response.status}`)
+    }
+
+    const data: ImageCreationResponse = await response.json()
+    return data
+  } catch (error) {
+    console.error('Error in image creation:', error)
+    throw error
+  }
+}
+
+// Example usage
+// const apiKey = 'your-api-key-here';
+// const params: ImageCreationParams = {
+//   prompt: "A beautiful sunset over a calm ocean",
+//   model: "sd_xl_base_1.0",
+//   width: 1024,
+//   height: 1024,
+//   steps: 30,
+//   guidance: 7.5,
+//   outputFormat: "png"
+// };
+//
+// callCreateImage(apiKey, params)
+//   .then(result => console.log(result))
+//   .catch(error => console.error(error));

packages/app/src/app/api/resolve/providers/letzai/index.ts

@@ -0,0 +1,72 @@
+import { TimelineSegment } from '@aitube/timeline'
+import { ResolveRequest } from '@aitube/clapper-services'
+import { ClapSegmentCategory, generateSeed } from '@aitube/clap'
+
+import { getWorkflowInputValues } from '../getWorkflowInputValues'
+import { callCreateImage } from './callCreateImage'
+
+export async function resolveSegment(
+  request: ResolveRequest
+): Promise<TimelineSegment> {
+  if (!request.settings.letzAiApiKey) {
+    throw new Error(`Missing API key for "LetzAi"`)
+  }
+
+  const segment: TimelineSegment = request.segment
+
+  let model = request.settings.imageGenerationWorkflow.data || ''
+
+  if (request.segment.category === ClapSegmentCategory.STORYBOARD) {
+    model = request.settings.imageGenerationWorkflow.data || ''
+
+    if (!request.prompts.image.positive) {
+      console.error(
+        `resolveSegment: cannot resolve a storyboard with an empty prompt`
+      )
+      return segment
+    }
+
+    const { workflowDefaultValues, workflowValues } = getWorkflowInputValues(
+      request.settings.imageGenerationWorkflow
+    )
+
+    const result = await callCreateImage(request.settings.letzAiApiKey, {
+      prompt: request.prompts.image.positive,
+      negativePrompt: request.prompts.image.negative,
+      // model: string;
+      width:
+        request.meta.width ||
+        workflowValues.width ||
+        workflowDefaultValues.width,
+      height:
+        request.meta.width ||
+        workflowValues.height ||
+        workflowDefaultValues.height,
+      // steps: number;
+      // guidance: number;
+      seed: generateSeed(),
+      // scheduler: string;
+      // outputFormat: string;
+    })
+
+    if (request.settings.censorNotForAllAudiencesContent) {
+      if (result.nsfw) {
+        throw new Error(
+          `The generated content has been filtered according to your safety settings`
+        )
+      }
+    }
+
+    if (!result.output) {
+      throw new Error(`Failed to generate at least one video`)
+    }
+
+    segment.assetUrl = `${result.output || ''}`
+  } else {
+    throw new Error(
+      `Clapper doesn't support ${request.segment.category} generation for provider "Fal.ai". Please open a pull request with (working code) to solve this!`
+    )
+  }
+
+  return segment
+}

packages/app/src/app/api/resolve/providers/letzai/types.ts

@@ -0,0 +1,43 @@
+/**
+ * Represents the parameters for the image creation request.
+ */
+export type ImageCreationParams = {
+  prompt: string
+  negativePrompt?: string
+  model?: string
+  width?: number
+  height?: number
+  steps?: number
+  guidance?: number
+  seed?: number
+  scheduler?: string
+  outputFormat?: string
+}
+
+/**
+ * Represents the response from the image creation request.
+ */
+export type ImageCreationResponse = {
+  id: string
+  createdAt: string
+  updatedAt: string
+  prompt: string
+  negativePrompt: string | null
+  model: string
+  width: number
+  height: number
+  steps: number
+  guidance: number
+  seed: number
+  scheduler: string
+  status: string
+  error: string | null
+  progress: number
+  outputFormat: string
+  output: string | null
+  nsfw: boolean
+  user: {
+    id: string
+    username: string
+  }
+}