diff --git a/bun.lockb b/bun.lockb index a1a595c8..28e1c125 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/package.json b/package.json index 89c74b67..29dfc931 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,5 @@ "packages/app" ], "dependencies": { - "civitai": "^0.1.15" } } \ No newline at end of file diff --git a/packages/app/package.json b/packages/app/package.json index f10d3c01..54f352f7 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -89,6 +89,8 @@ "@xyflow/react": "^12.0.3", "autoprefixer": "10.4.19", "base64-arraybuffer": "^1.0.2", + "bellhop-iframe": "^3.5.0", + "civitai": "^0.1.15", "class-variance-authority": "^0.7.0", "clsx": "^2.1.1", "cmdk": "^0.2.1", diff --git a/packages/app/src/app/api/resolve/providers/comfyui/index.ts b/packages/app/src/app/api/resolve/providers/comfyui/index.ts index 0d455aa0..45aaafcf 100644 --- a/packages/app/src/app/api/resolve/providers/comfyui/index.ts +++ b/packages/app/src/app/api/resolve/providers/comfyui/index.ts @@ -1,5 +1,9 @@ import { ResolveRequest } from '@aitube/clapper-services' -import { ClapAssetSource, ClapSegmentCategory, generateSeed } from '@aitube/clap' +import { + ClapAssetSource, + ClapSegmentCategory, + generateSeed, +} from '@aitube/clap' import { TimelineSegment } from '@aitube/timeline' import { BasicCredentials, CallWrapper, ComfyApi } from '@saintno/comfyui-sdk' import { decodeOutput } from '@/lib/utils/decodeOutput' diff --git a/packages/app/src/app/embed/EmbeddedPlayer.tsx b/packages/app/src/app/embed/EmbeddedPlayer.tsx new file mode 100644 index 00000000..ec1c0cd2 --- /dev/null +++ b/packages/app/src/app/embed/EmbeddedPlayer.tsx @@ -0,0 +1,110 @@ +import { useEffect, useRef, useState } from 'react' + +import { cn } from '@/lib/utils' + +import { TogglePlayback } from './TogglePlayback' +import { StaticOrInteractiveTag } from './StaticOrInteractive' +import { useMonitor } from '@/services' +import { useTimeline } from '@aitube/timeline' + +export function EmbeddedPlayer() { + const isPlaying = useMonitor((s) => s.isPlaying) + const togglePlayback = useMonitor((s) => s.togglePlayback) + + const meta = useTimeline((s) => s.meta) + + const [isOverlayVisible, setOverlayVisible] = useState(true) + + const overlayTimerRef = useRef() + // const videoLayerRef = useRef(null) + // const segmentationLayerRef = useRef(null) + + const isPlayingRef = useRef(isPlaying) + isPlayingRef.current = isPlaying + + const scheduleOverlayInvisibility = () => { + clearTimeout(overlayTimerRef.current) + overlayTimerRef.current = setTimeout(() => { + if (isPlayingRef.current) { + setOverlayVisible(!isPlayingRef.current) + } + clearTimeout(overlayTimerRef.current) + }, 3000) + } + + return ( + <> + {/* content overlay, with the gradient, buttons etc */} +
{ + setOverlayVisible(true) + scheduleOverlayInvisibility() + }} + style={{ + // width, + // height, + boxShadow: 'rgba(0, 0, 0, 1) 0px -77px 100px 15px inset', + }} + > + {/* bottom slider and button bar */} +
+ {/* the (optional) timeline slider bar */} +
+
+
+ + {/* button bar */} +
+ {/* left-side buttons */} +
+ + +
+ + {/* right-side buttons */} +
+
+
+
+ + ) +} diff --git a/packages/app/src/app/embed/StaticOrInteractive.tsx b/packages/app/src/app/embed/StaticOrInteractive.tsx new file mode 100644 index 00000000..4371643b --- /dev/null +++ b/packages/app/src/app/embed/StaticOrInteractive.tsx @@ -0,0 +1,39 @@ +import { PiLightningFill } from 'react-icons/pi' + +import { cn } from '@/lib/utils/cn' + +export function StaticOrInteractiveTag({ + isInteractive = false, + size = 'md', + className = '', +}: { + isInteractive?: boolean + size?: 'sm' | 'md' + className?: string +}) { + const isStatic = !isInteractive + + return ( +
+ + + {isInteractive + ? 'Interactive' + : // : isLive ? "Live" + 'Static content'} + +
+ ) +} diff --git a/packages/app/src/app/embed/TogglePlayback.tsx b/packages/app/src/app/embed/TogglePlayback.tsx new file mode 100644 index 00000000..d5e58146 --- /dev/null +++ b/packages/app/src/app/embed/TogglePlayback.tsx @@ -0,0 +1,28 @@ +import React from 'react' +import { IoMdPause, IoMdPlay } from 'react-icons/io' + +import { IconSwitch } from '@/components/monitor/icons/icon-switch' + +export function TogglePlayback({ + className = '', + isToggledOn, + onClick, +}: { + className?: string + isToggledOn?: boolean + onClick?: () => void +}) { + return ( + + ) +} diff --git a/packages/app/src/app/embed/embed.tsx b/packages/app/src/app/embed/embed.tsx deleted file mode 100644 index 2a185eb6..00000000 --- a/packages/app/src/app/embed/embed.tsx +++ /dev/null @@ -1,45 +0,0 @@ -'use client' - -import React, { useRef } from 'react' -import { useTimeline } from '@aitube/timeline' - -import { Toaster } from '@/components/ui/sonner' -import { cn } from '@/lib/utils' -import { TooltipProvider } from '@/components/ui/tooltip' -import { Monitor } from '@/components/monitor' - -import { SettingsDialog } from '@/components/settings' -import { LoadingDialog } from '@/components/dialogs/loader/LoadingDialog' -import { TopBar } from '@/components/toolbars/top-bar' -import { useTheme } from '@/services' - -export function Embed() { - const ref = useRef(null) - const isEmpty = useTimeline((s) => s.isEmpty) - const theme = useTheme() - - return ( - -
- -
- -
- - - -
-
- ) -} diff --git a/packages/app/src/app/embed/page.tsx b/packages/app/src/app/embed/page.tsx index 9a8fe302..fa9eebff 100644 --- a/packages/app/src/app/embed/page.tsx +++ b/packages/app/src/app/embed/page.tsx @@ -4,7 +4,7 @@ import { useEffect, useState } from 'react' import Head from 'next/head' import Script from 'next/script' -import { Embed } from './embed' +import { ClapperIntegrationMode, Main } from '../main' // https://nextjs.org/docs/pages/building-your-application/optimizing/fonts @@ -44,7 +44,7 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= style={{ display: 'none', visibility: 'hidden' }} > -
{isLoaded && }
+
{isLoaded &&
}
) } diff --git a/packages/app/src/app/embed/useParentController.ts b/packages/app/src/app/embed/useParentController.ts new file mode 100644 index 00000000..54af4161 --- /dev/null +++ b/packages/app/src/app/embed/useParentController.ts @@ -0,0 +1,59 @@ +'use client' + +import { create } from 'zustand' +import { Bellhop } from 'bellhop-iframe' + +export const useParentController = create<{ + bellhop: Bellhop + + /** + * Whether the communication pipeline seems to be working or not + * + * Initially we assume that it is, but this can be invalidated + * if we know for certain it is not, or in case of exception + */ + canUseBellhop: boolean + + /** + * Whether Clapper is ready to receive instructions + */ + hasLoadedBellhop: boolean + + setCanUseBellhop: (canUseBellhop: boolean) => void + setHasLoadedBellhop: (hasLoadedBellhop: boolean) => void + onMessage: (name: string, callback: Function, priority?: number) => void + sendMessage: (type: string, data?: any) => void +}>((set, get) => ({ + bellhop: undefined as unknown as Bellhop, + canUseBellhop: true, + hasLoadedBellhop: false, + setCanUseBellhop: (canUseBellhop: boolean) => { + set({ + canUseBellhop, + }) + }, + setHasLoadedBellhop: (hasLoadedBellhop: boolean) => { + set({ + bellhop: hasLoadedBellhop + ? new Bellhop() + : (undefined as unknown as Bellhop), + hasLoadedBellhop, + }) + }, + onMessage: (name: string, callback: Function, priority?: number) => { + const { bellhop } = get() + try { + bellhop.on(name, callback, priority) + } catch (err) { + console.log(`failed to subscribe to parent iframe messages:`, err) + } + }, + sendMessage: (type: string, data?: any) => { + const { bellhop } = get() + try { + bellhop.send(type, data) + } catch (err) { + console.log(`failed to send a message to parent iframe:`, err) + } + }, +})) diff --git a/packages/app/src/app/embed/useSetupIframeOnce.ts b/packages/app/src/app/embed/useSetupIframeOnce.ts new file mode 100644 index 00000000..811daa10 --- /dev/null +++ b/packages/app/src/app/embed/useSetupIframeOnce.ts @@ -0,0 +1,79 @@ +'use client' + +import { useEffect } from 'react' + +import { useMonitor } from '@/services' +import { useParentController } from './useParentController' + +/** + * You should only call this once, at the root of the react tree + */ +export function useSetupIframeOnce(isEnabled = false) { + const canUseBellhop = useParentController((s) => s.canUseBellhop) + const setCanUseBellhop = useParentController((s) => s.setCanUseBellhop) + const hasLoadedBellhop = useParentController((s) => s.hasLoadedBellhop) + const setHasLoadedBellhop = useParentController((s) => s.setHasLoadedBellhop) + const onMessage = useParentController((s) => s.onMessage) + const sendMessage = useParentController((s) => s.sendMessage) + const isPlaying = useMonitor((s) => s.isPlaying) + const togglePlayback = useMonitor((s) => s.togglePlayback) + + // TODO: maybe we should add a JWT token to secure this, make it only embeddable + // on a certain website (eg. AiTube.at), and if people want to + // embed the player somewhere's else they will have to deploy their own + + useEffect(() => { + if (!isEnabled) { + // when we are detecting that we are not in an iframe + + if (canUseBellhop) { + setCanUseBellhop(false) + } + return + } + + if (!canUseBellhop) { + setCanUseBellhop(true) + } + + if (hasLoadedBellhop) { + // no need to connect twice + // but we can update the "isPlaying" status + sendMessage('status', { isPlaying }) + return + } else { + // we only try this once + try { + setHasLoadedBellhop(true) + + onMessage('togglePlayback', function (event) { + const { isPlaying } = event.data as { isPlaying: boolean } + togglePlayback(isPlaying) + }) + + onMessage('loadPrompt', function (event) { + console.log('loadPrompt:', event) + // generate the first scene of an OpenClap file from the prompt + }) + + onMessage('loadClap', function (event) { + // we need to be careful here in how to transmit the ClapProject + // to the iframe + // we are going to want to encode it somehow, eg using: + // function base64_encode(s) { + // return btoa(unescape(encodeURIComponent(s))); + // } + // function base64_decode(s) { + // return decodeURIComponent(escape(atob(s))); + // } + }) + + sendMessage('status', { isReady: true }) + } catch (err) { + console.error(`failed to initialize bellhop`) + setHasLoadedBellhop(false) + setCanUseBellhop(false) + } + } + }, [canUseBellhop, isEnabled, hasLoadedBellhop, isPlaying]) +} diff --git a/packages/app/src/app/main.tsx b/packages/app/src/app/main.tsx index 2e669623..33c1e0f5 100644 --- a/packages/app/src/app/main.tsx +++ b/packages/app/src/app/main.tsx @@ -7,6 +7,7 @@ import { DndProvider, useDrop } from 'react-dnd' import { HTML5Backend, NativeTypes } from 'react-dnd-html5-backend' import { UIWindowLayout } from '@aitube/clapper-services' import { ErrorBoundary, FallbackProps } from 'react-error-boundary' +import { Bellhop } from 'bellhop-iframe' import { Toaster } from '@/components/ui/sonner' import { cn } from '@/lib/utils' @@ -27,13 +28,19 @@ import { EntityEditor } from '@/components/editors/EntityEditor' import { WorkflowEditor } from '@/components/editors/WorkflowEditor' import { FilterEditor } from '@/components/editors/FilterEditor' -import { useUI, useIO, useTheme } from '@/services' +import { useUI, useIO, useTheme, useMonitor } from '@/services' import { useRenderLoop } from '@/services/renderer' import { useDynamicWorkflows } from '@/services/editors/workflow-editor/useDynamicWorkflows' +import { useSetupIframeOnce } from './embed/useSetupIframeOnce' -type DroppableThing = { files: File[] } +export enum ClapperIntegrationMode { + APP = 'APP', + IFRAME = 'IFRAME', +} + +export type DroppableThing = { files: File[] } -function MainContent() { +function MainContent({ mode }: { mode: ClapperIntegrationMode }) { const ref = useRef(null) const showWelcomeScreen = useUI((s) => s.showWelcomeScreen) const showExplorer = useUI((s) => s.showExplorer) @@ -45,6 +52,8 @@ function MainContent() { const isTopMenuOpen = useUI((s) => s.isTopMenuOpen) const windowLayout = useUI((s) => s.windowLayout) + const isIframe = mode === ClapperIntegrationMode.IFRAME + // this has to be done at the root of the app, that way it can // perform its routine even when the monitor component is hidden useRenderLoop() @@ -53,6 +62,9 @@ function MainContent() { // sync workflows even when the workflow component is hidden useDynamicWorkflows() + // also has to be done here + useSetupIframeOnce(isIframe) + const [{ isOver, canDrop }, connectFileDrop] = useDrop({ accept: [NativeTypes.FILE], drop: (item: DroppableThing): void => { @@ -75,6 +87,13 @@ function MainContent() { setHasBetaAccess(hasBetaAccess) }, [hasBetaAccess, setHasBetaAccess]) + const iframeLayout = ( + <> + + + + ) + const gridLayout = ( @@ -289,29 +308,35 @@ function MainContent() { `dark fixed flex h-screen w-screen select-none flex-col overflow-hidden font-light text-neutral-900/90 dark:text-neutral-100/90` )} > - +
- {windowLayout === UIWindowLayout.FLYING ? flyingLayout : gridLayout} + {isIframe + ? iframeLayout + : windowLayout === UIWindowLayout.FLYING + ? flyingLayout + : gridLayout}
- {welcomeScreen} + {!isIframe && welcomeScreen} - {windowLayout === UIWindowLayout.GRID && } + {!isIframe && windowLayout === UIWindowLayout.GRID && } ) } @@ -327,12 +352,20 @@ function fallbackRender({ error, resetErrorBoundary }: FallbackProps) { ) } -export function Main() { +export function Main( + { + mode = ClapperIntegrationMode.APP, + }: { + mode: ClapperIntegrationMode + } = { + mode: ClapperIntegrationMode.APP, + } +) { return ( - + diff --git a/packages/app/src/app/page.tsx b/packages/app/src/app/page.tsx index 2cbcc73a..4a6aa6f3 100644 --- a/packages/app/src/app/page.tsx +++ b/packages/app/src/app/page.tsx @@ -4,7 +4,7 @@ import { useEffect, useState } from 'react' import Head from 'next/head' import Script from 'next/script' -import { Main } from './main' +import { ClapperIntegrationMode, Main } from './main' // https://nextjs.org/docs/pages/building-your-application/optimizing/fonts @@ -44,7 +44,7 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= style={{ display: 'none', visibility: 'hidden' }} > -
{isLoaded &&
}
+
{isLoaded &&
}
) } diff --git a/packages/app/src/components/core/timeline/index.tsx b/packages/app/src/components/core/timeline/index.tsx index 1be0aec6..a7ac9cd5 100644 --- a/packages/app/src/components/core/timeline/index.tsx +++ b/packages/app/src/components/core/timeline/index.tsx @@ -5,7 +5,15 @@ import { useMonitor } from '@/services/monitor/useMonitor' import { useResolver } from '@/services/resolver/useResolver' import { useUI } from '@/services/ui' -export function Timeline() { +export function Timeline( + { + className = '', + }: { + className?: string + } = { + className: '', + } +) { const isReady = useTimeline((s) => s.isReady) const resolveSegment: SegmentResolver = useResolver((s) => s.resolveSegment) @@ -49,5 +57,13 @@ export function Timeline() { togglePlayback, ]) - return + if (className) { + return ( +
+ +
+ ) + } + + return } diff --git a/packages/app/src/components/toolbars/top-bar/index.tsx b/packages/app/src/components/toolbars/top-bar/index.tsx index ec407e85..a26917cb 100644 --- a/packages/app/src/components/toolbars/top-bar/index.tsx +++ b/packages/app/src/components/toolbars/top-bar/index.tsx @@ -5,12 +5,20 @@ import { cn } from '@/lib/utils' import { TopMenu } from '../top-menu' -export function TopBar() { +export function TopBar( + { + className = '', + }: { + className: string + } = { + className: '', + } +) { const theme = useTheme() return (