Skip to content

Commit 119f192

Browse files
flo-bitflo-bit
authored
Merge pull request #6 from flo-bit/main
deploy
2 parents 742c30b + 616e37a commit 119f192

File tree

3 files changed

+109
-32
lines changed

3 files changed

+109
-32
lines changed

‎README.md

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,17 @@ it allows you to easily use the openai realtime api in your svelte(kit) project.
66

77
work in progress, but it should be functional.
88

9-
demo here (enter your own api key, as always be careful with stuff like that and e.g. use a 2$ limited api key): https://flo-bit.dev/svelte-openai-realtime-api/
9+
demos here (enter your own api key, as always be careful with stuff like that and e.g. use a 2$ limited api key):
10+
11+
basic chat: https://flo-bit.dev/svelte-openai-realtime-api/
12+
13+
chat with visualizations: https://flo-bit.dev/svelte-openai-realtime-api/visualizations-chat
14+
15+
visualizations only: https://flo-bit.dev/svelte-openai-realtime-api/visualizations-input
16+
17+
18+
https://github.com/user-attachments/assets/81473395-5811-475d-81ac-13c0e7811eff
1019

11-
https://github.com/user-attachments/assets/c1e96dac-98b5-4e16-95e8-9cbc6e60865d
1220

1321
## how to use
1422

@@ -65,6 +73,10 @@ $ npm i openai/openai-realtime-api-beta
6573

6674
see `src/routes/+page.svelte` for a full example.
6775

76+
## visualization
77+
78+
see `src/routes/visulizations-chat/+page.svelte` for an example of how to visualize the audio input and output.
79+
6880
## relay server
6981

7082
for production use, you will need to use a relay server to use the realtime api.

‎src/routes/visualizations-chat/+page.svelte

Lines changed: 91 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import InnerGlowVisualizer from '$lib/realtime/visualizations/InnerGlowVisualizer.svelte';
99
import CircleCirclesVisualizer from '$lib/realtime/visualizations/CircleCirclesVisualizer.svelte';
1010
import BarVisualizer from '$lib/realtime/visualizations/BarVisualizer.svelte';
11+
import type { ItemType } from '@openai/realtime-api-beta/dist/lib/client';
1112
1213
export let wavRecorder: WavRecorder;
1314
let wavStreamPlayer: WavStreamPlayer;
@@ -21,10 +22,40 @@
2122
let current = 0;
2223
2324
$: shownVisualizer = visualizations[current];
25+
26+
let items: ItemType[] = [];
27+
28+
let lastAssistantText = '';
29+
let lastUserText = '';
30+
31+
$: if (items && items.length > 0) {
32+
// last item with role "assistant"
33+
34+
const lastAssistant = items
35+
.slice()
36+
.reverse()
37+
.find((item) => item.role === 'assistant');
38+
39+
if (
40+
lastAssistant?.formatted.transcript &&
41+
lastAssistant.formatted.transcript !== lastAssistantText
42+
) {
43+
lastAssistantText = lastAssistant.formatted.transcript;
44+
}
45+
46+
const lastUser = items
47+
.slice()
48+
.reverse()
49+
.find((item) => item.role === 'user');
50+
51+
if (lastUser?.formatted.transcript && lastUser.formatted.transcript !== lastUserText) {
52+
lastUserText = lastUser.formatted.transcript;
53+
}
54+
}
2455
</script>
2556

2657
<button
27-
class="absolute inset-0 h-[100dvh] w-screen z-10 text-white flex items-end justify-center p-8 text-sm font-semibold"
58+
class="absolute inset-0 h-[100dvh] w-screen z-10 text-white flex items-end justify-center p-4 text-sm font-semibold"
2859
on:click={() => {
2960
current = (current + 1) % visualizations.length;
3061
}}
@@ -33,52 +64,69 @@
3364
</button>
3465

3566
{#if apiKey}
36-
<Realtime bind:startConversation bind:wavRecorder bind:wavStreamPlayer {apiKey} />
67+
<Realtime bind:startConversation bind:items bind:wavRecorder bind:wavStreamPlayer {apiKey} />
3768
{/if}
3869

3970
{#if shownVisualizer === 'circle-bars'}
40-
<div
41-
class="absolute flex flex-col md:flex-row items-center justify-center h-[100dvh] w-screen gap-16"
42-
>
43-
<div class="h-64 w-96">
44-
<CircleBarVisualizer detail={128} wavInput={wavStreamPlayer} startHue={150} endHue={220} />
71+
<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
72+
<div class="h-96 w-96">
73+
<CircleBarVisualizer
74+
detail={128}
75+
wavInput={wavRecorder}
76+
startHue={0}
77+
endHue={50}
78+
varyBrightness
79+
/>
4580
</div>
81+
</div>
4682

47-
<div class="h-64 w-96">
48-
<CircleBarVisualizer detail={128} wavInput={wavRecorder} startHue={0} endHue={50} />
83+
<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
84+
<div class="h-96 w-96">
85+
<CircleBarVisualizer
86+
detail={128}
87+
wavInput={wavStreamPlayer}
88+
startHue={150}
89+
endHue={220}
90+
varyBrightness
91+
/>
4992
</div>
5093
</div>
5194
{:else if shownVisualizer === 'bars'}
52-
<div
53-
class="absolute flex flex-col md:flex-row items-center justify-center h-[100dvh] w-screen gap-16"
54-
>
55-
<div class="h-64 w-96 -scale-y-100 md:scale-y-100">
56-
<BarVisualizer barSpacing={8} wavInput={wavStreamPlayer} startHue={150} endHue={220} />
57-
</div>
95+
<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
5896
<div class="h-64 w-96">
5997
<BarVisualizer barSpacing={8} wavInput={wavRecorder} startHue={0} endHue={50} />
6098
</div>
6199
</div>
100+
<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
101+
<div class="h-64 w-96 -scale-y-100 md:scale-y-100">
102+
<BarVisualizer barSpacing={8} wavInput={wavStreamPlayer} startHue={150} endHue={220} />
103+
</div>
104+
</div>
62105
{:else if shownVisualizer === 'circle-circles'}
63-
<div
64-
class="absolute flex flex-col md:flex-row items-center justify-center h-[100dvh] w-screen gap-16"
65-
>
66-
<div class="h-64 w-96">
67-
<CircleCirclesVisualizer wavInput={wavStreamPlayer} startHue={150} endHue={220} />
106+
<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
107+
<div class="h-96 w-96">
108+
<CircleCirclesVisualizer wavInput={wavRecorder} startHue={0} endHue={50} varyBrightness />
68109
</div>
69-
<div class="h-64 w-96">
70-
<CircleCirclesVisualizer wavInput={wavRecorder} startHue={0} endHue={50} />
110+
</div>
111+
<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
112+
<div class="h-96 w-96">
113+
<CircleCirclesVisualizer
114+
wavInput={wavStreamPlayer}
115+
startHue={150}
116+
endHue={220}
117+
varyBrightness
118+
/>
71119
</div>
72120
</div>
73121
{:else if shownVisualizer === 'deformed-circle'}
74-
<div
75-
class="absolute flex flex-col md:flex-row items-center justify-center h-[100dvh] w-screen gap-16"
76-
>
122+
<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
77123
<div class="h-96 w-96">
78-
<DeformedCircleVisualizer wavInput={wavStreamPlayer} startHue={150} endHue={220} />
124+
<DeformedCircleVisualizer wavInput={wavRecorder} startHue={0} endHue={50} />
79125
</div>
126+
</div>
127+
<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
80128
<div class="h-96 w-96">
81-
<DeformedCircleVisualizer wavInput={wavRecorder} startHue={0} endHue={50} />
129+
<DeformedCircleVisualizer wavInput={wavStreamPlayer} startHue={150} endHue={220} />
82130
</div>
83131
</div>
84132
{:else if shownVisualizer === 'innerglow'}
@@ -103,6 +151,22 @@
103151
</div>
104152
{/if}
105153

154+
<div
155+
class="absolute flex flex-col items-center justify-between h-[100dvh] w-screen py-16 px-4 text-center"
156+
>
157+
<div
158+
class="bg-gradient-to-r from-blue-400 via-cyan-400 to-green-400 inline-block text-transparent bg-clip-text font-semibold text-xl max-w-xl"
159+
>
160+
{lastAssistantText}
161+
</div>
162+
<div class="h-96 w-96"></div>
163+
<div
164+
class="bg-gradient-to-r from-red-400 to-amber-400 inline-block text-transparent bg-clip-text font-semibold text-xl max-w-xl"
165+
>
166+
{lastUserText}
167+
</div>
168+
</div>
169+
106170
<Modal
107171
onStart={() => {
108172
startConversation();

‎src/routes/visualizations-input/+page.svelte

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,9 @@
2121

2222
recording = true;
2323
}}
24-
class="rounded-full px-5 py-3 text-base font-semibold focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-amber-500 text-amber-500 bg-amber-500/10 border border-amber-500/20 hover:bg-amber-500/20 {recording ? 'cursor-not-allowed opacity-50' : ''}"
25-
>start microphone</button
24+
class="rounded-full px-5 py-3 text-base font-semibold focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-amber-500 text-amber-500 bg-amber-500/10 border border-amber-500/20 hover:bg-amber-500/20 {recording
25+
? 'cursor-not-allowed opacity-50'
26+
: ''}">start microphone</button
2627
>
2728

2829
<div class="grid grid-cols-1 sm:grid-cols-3 gap-4 mt-8">
@@ -39,7 +40,7 @@
3940
<DeformedCircleVisualizer wavInput={wavRecorder} startHue={0} endHue={50} />
4041
</div>
4142
<div class="h-64 w-full rounded-xl border border-white/15 overflow-hidden">
42-
<InnerGlowVisualizer wavInput={wavRecorder} startHue={0} endHue={50}/>
43+
<InnerGlowVisualizer wavInput={wavRecorder} startHue={0} endHue={50} />
4344
</div>
4445
</div>
4546
</div>

0 commit comments

Comments
 (0)