Skip to content

Commit 219a51f

Browse files
paulpvpaulpv
committed
Refactor Client & API to specify either WebRTC or WebSocket
1 parent a5cb948 commit 219a51f

File tree

5 files changed

+393
-129
lines changed

5 files changed

+393
-129
lines changed

‎lib/api.js

+60-127
Original file line numberDiff line numberDiff line change
@@ -1,44 +1,61 @@
11
import { RealtimeEventHandler } from './event_handler.js';
22
import { RealtimeUtils } from './utils.js';
3+
import { RealtimeTransportType } from './transport.js';
4+
import { RealtimeTransportWebRTC } from './webrtc.js';
5+
import { RealtimeTransportWebSocket } from './websocket.js';
36

47
export class RealtimeAPI extends RealtimeEventHandler {
58
/**
69
* Create a new RealtimeAPI instance
710
* @param {{url?: string, apiKey?: string, dangerouslyAllowAPIKeyInBrowser?: boolean, debug?: boolean}} [settings]
811
* @returns {RealtimeAPI}
912
*/
10-
constructor({ url, apiKey, dangerouslyAllowAPIKeyInBrowser, debug } = {}) {
13+
constructor({ transportType, url, apiKey, dangerouslyAllowAPIKeyInBrowser, debug } = {}) {
1114
super();
12-
this.defaultUrl = 'wss://api.openai.com/v1/realtime';
13-
this.url = url || this.defaultUrl;
14-
this.apiKey = apiKey || null;
1515
this.debug = !!debug;
16-
this.ws = null;
17-
if (globalThis.document && this.apiKey) {
18-
if (!dangerouslyAllowAPIKeyInBrowser) {
19-
throw new Error(
20-
`Can not provide API key in the browser without "dangerouslyAllowAPIKeyInBrowser" set to true`,
21-
);
16+
transportType = transportType?.toUpperCase() || RealtimeTransportType.WEBRTC;
17+
switch (transportType) {
18+
case RealtimeTransportType.WEBRTC: {
19+
this.transport = new RealtimeTransportWebRTC({ url, apiKey, dangerouslyAllowAPIKeyInBrowser, debug });
20+
break;
21+
}
22+
case RealtimeTransportType.WEBSOCKET: {
23+
this.transport = new RealtimeTransportWebSocket({ url, apiKey, dangerouslyAllowAPIKeyInBrowser, debug });
24+
break;
25+
}
26+
default: {
27+
throw new Error(`Invalid transportType: "${transportType}"`);
2228
}
2329
}
30+
this.transport.on('close', () => {
31+
this.disconnect();
32+
});
33+
this.transport.on('message', (event) => {
34+
const message = JSON.parse(event.data);
35+
this._receive(message.type, message)
36+
});
37+
}
38+
39+
get transportType() {
40+
return this.transport.transportType;
2441
}
2542

2643
/**
27-
* Tells us whether or not the WebSocket is connected
44+
* Tells us whether or not the transport is connected
2845
* @returns {boolean}
2946
*/
30-
isConnected() {
31-
return !!this.ws;
47+
get isConnected() {
48+
return this.transport.isConnected;
3249
}
3350

3451
/**
35-
* Writes WebSocket logs to console
52+
* Writes log to console
3653
* @param {...any} args
3754
* @returns {true}
3855
*/
3956
log(...args) {
4057
const date = new Date().toISOString();
41-
const logs = [`[Websocket/${date}]`].concat(args).map((arg) => {
58+
const logs = [`[RealtimeAPI/${date}]`].concat(args).map((arg) => {
4259
if (typeof arg === 'object' && arg !== null) {
4360
return JSON.stringify(arg, null, 2);
4461
} else {
@@ -52,142 +69,51 @@ export class RealtimeAPI extends RealtimeEventHandler {
5269
}
5370

5471
/**
55-
* Connects to Realtime API Websocket Server
72+
* Connects to Realtime API Server
5673
* @param {{model?: string}} [settings]
5774
* @returns {Promise<true>}
5875
*/
59-
async connect({ model } = { model: 'gpt-4o-realtime-preview-2024-10-01' }) {
60-
if (!this.apiKey && this.url === this.defaultUrl) {
61-
console.warn(`No apiKey provided for connection to "${this.url}"`);
62-
}
63-
if (this.isConnected()) {
64-
throw new Error(`Already connected`);
65-
}
66-
if (globalThis.WebSocket) {
67-
/**
68-
* Web browser
69-
*/
70-
if (globalThis.document && this.apiKey) {
71-
console.warn(
72-
'Warning: Connecting using API key in the browser, this is not recommended',
73-
);
74-
}
75-
const WebSocket = globalThis.WebSocket;
76-
const ws = new WebSocket(`${this.url}${model ? `?model=${model}` : ''}`, [
77-
'realtime',
78-
`openai-insecure-api-key.${this.apiKey}`,
79-
'openai-beta.realtime-v1',
80-
]);
81-
ws.addEventListener('message', (event) => {
82-
const message = JSON.parse(event.data);
83-
this.receive(message.type, message);
84-
});
85-
return new Promise((resolve, reject) => {
86-
const connectionErrorHandler = () => {
87-
this.disconnect(ws);
88-
reject(new Error(`Could not connect to "${this.url}"`));
89-
};
90-
ws.addEventListener('error', connectionErrorHandler);
91-
ws.addEventListener('open', () => {
92-
this.log(`Connected to "${this.url}"`);
93-
ws.removeEventListener('error', connectionErrorHandler);
94-
ws.addEventListener('error', () => {
95-
this.disconnect(ws);
96-
this.log(`Error, disconnected from "${this.url}"`);
97-
this.dispatch('close', { error: true });
98-
});
99-
ws.addEventListener('close', () => {
100-
this.disconnect(ws);
101-
this.log(`Disconnected from "${this.url}"`);
102-
this.dispatch('close', { error: false });
103-
});
104-
this.ws = ws;
105-
resolve(true);
106-
});
107-
});
108-
} else {
109-
/**
110-
* Node.js
111-
*/
112-
const moduleName = 'ws';
113-
const wsModule = await import(/* webpackIgnore: true */ moduleName);
114-
const WebSocket = wsModule.default;
115-
const ws = new WebSocket(
116-
'wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview-2024-10-01',
117-
[],
118-
{
119-
finishRequest: (request) => {
120-
// Auth
121-
request.setHeader('Authorization', `Bearer ${this.apiKey}`);
122-
request.setHeader('OpenAI-Beta', 'realtime=v1');
123-
request.end();
124-
},
125-
},
126-
);
127-
ws.on('message', (data) => {
128-
const message = JSON.parse(data.toString());
129-
this.receive(message.type, message);
130-
});
131-
return new Promise((resolve, reject) => {
132-
const connectionErrorHandler = () => {
133-
this.disconnect(ws);
134-
reject(new Error(`Could not connect to "${this.url}"`));
135-
};
136-
ws.on('error', connectionErrorHandler);
137-
ws.on('open', () => {
138-
this.log(`Connected to "${this.url}"`);
139-
ws.removeListener('error', connectionErrorHandler);
140-
ws.on('error', () => {
141-
this.disconnect(ws);
142-
this.log(`Error, disconnected from "${this.url}"`);
143-
this.dispatch('close', { error: true });
144-
});
145-
ws.on('close', () => {
146-
this.disconnect(ws);
147-
this.log(`Disconnected from "${this.url}"`);
148-
this.dispatch('close', { error: false });
149-
});
150-
this.ws = ws;
151-
resolve(true);
152-
});
153-
});
154-
}
76+
async connect({ sessionConfig, getMicrophoneCallback, setAudioOutputCallback }) {
77+
return this.transport.connect({ sessionConfig, getMicrophoneCallback, setAudioOutputCallback });
15578
}
15679

15780
/**
15881
* Disconnects from Realtime API server
159-
* @param {WebSocket} [ws]
16082
* @returns {true}
16183
*/
162-
disconnect(ws) {
163-
if (!ws || this.ws === ws) {
164-
this.ws && this.ws.close();
165-
this.ws = null;
166-
return true;
167-
}
84+
async disconnect() {
85+
await this.transport.disconnect();
86+
return true;
16887
}
16988

17089
/**
171-
* Receives an event from WebSocket and dispatches as "server.{eventName}" and "server.*" events
90+
* Receives an event from transport and dispatches as "server.{eventName}" and "server.*" events
17291
* @param {string} eventName
17392
* @param {{[key: string]: any}} event
17493
* @returns {true}
17594
*/
176-
receive(eventName, event) {
177-
this.log(`received:`, eventName, event);
95+
_receive(eventName, event) {
96+
if (this.debug) {
97+
if (eventName === 'response.audio.delta') {
98+
const delta = event.delta;
99+
this.log(`received:`, eventName, { ...event, delta: delta.slice(0, 10) + '...' + delta.slice(-10) });
100+
} else {
101+
this.log(`received:`, eventName, event);
102+
}
103+
}
178104
this.dispatch(`server.${eventName}`, event);
179105
this.dispatch('server.*', event);
180106
return true;
181107
}
182108

183109
/**
184-
* Sends an event to WebSocket and dispatches as "client.{eventName}" and "client.*" events
110+
* Sends an event to transport and dispatches as "client.{eventName}" and "client.*" events
185111
* @param {string} eventName
186112
* @param {{[key: string]: any}} event
187113
* @returns {true}
188114
*/
189-
send(eventName, data) {
190-
if (!this.isConnected()) {
115+
async send(eventName, data) {
116+
if (!this.isConnected) {
191117
throw new Error(`RealtimeAPI is not connected`);
192118
}
193119
data = data || {};
@@ -201,8 +127,15 @@ export class RealtimeAPI extends RealtimeEventHandler {
201127
};
202128
this.dispatch(`client.${eventName}`, event);
203129
this.dispatch('client.*', event);
204-
this.log(`sent:`, eventName, event);
205-
this.ws.send(JSON.stringify(event));
130+
if (this.debug) {
131+
if (eventName === 'input_audio_buffer.append') {
132+
const audio = event.audio;
133+
this.log(`sending:`, eventName, { ...event, audio: audio.slice(0, 10) + '...' + audio.slice(-10) });
134+
} else {
135+
this.log(`sending:`, eventName, event);
136+
}
137+
}
138+
await this.transport.send(event);
206139
return true;
207140
}
208141
}

‎lib/client.js

+5-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
1-
import { RealtimeEventHandler } from './event_handler.js';
21
import { RealtimeAPI } from './api.js';
32
import { RealtimeConversation } from './conversation.js';
3+
import { RealtimeEventHandler } from './event_handler.js';
4+
import { RealtimeTransportType } from './transport.js';
45
import { RealtimeUtils } from './utils.js';
56

67
/**
@@ -191,8 +192,9 @@ export class RealtimeClient extends RealtimeEventHandler {
191192
* Create a new RealtimeClient instance
192193
* @param {{url?: string, apiKey?: string, dangerouslyAllowAPIKeyInBrowser?: boolean, debug?: boolean}} [settings]
193194
*/
194-
constructor({ url, apiKey, dangerouslyAllowAPIKeyInBrowser, debug } = {}) {
195+
constructor({ transportType, url, apiKey, dangerouslyAllowAPIKeyInBrowser, debug } = {}) {
195196
super();
197+
transportType = transportType?.toUpperCase() || RealtimeTransportType.WEBRTC;
196198
this.defaultSessionConfig = {
197199
modalities: ['text', 'audio'],
198200
instructions: '',
@@ -219,6 +221,7 @@ export class RealtimeClient extends RealtimeEventHandler {
219221
silence_duration_ms: 200, // How long to wait to mark the speech as stopped.
220222
};
221223
this.realtime = new RealtimeAPI({
224+
transportType,
222225
url,
223226
apiKey,
224227
dangerouslyAllowAPIKeyInBrowser,

‎lib/transport.js

+84
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
import { RealtimeEventHandler } from './event_handler.js';
2+
3+
/**
4+
* Enum representing the transport types.
5+
* @readonly
6+
* @enum {string}
7+
*/
8+
export const RealtimeTransportType = {
9+
WEBRTC: "WEBRTC",
10+
WEBSOCKET: "WEBSOCKET",
11+
};
12+
13+
/**
14+
* An abstract base class representing a RealtimeTransport.
15+
* Subclasses must implement all of these methods.
16+
*
17+
* @interface
18+
*/
19+
export class RealtimeTransport extends RealtimeEventHandler {
20+
get transportType() {
21+
throw new Error("Not implemented: transportType getter");
22+
}
23+
24+
get defaultUrl() {
25+
throw new Error("Not implemented: defaultUrl getter");
26+
}
27+
28+
log(...args) {
29+
if (this.debug) {
30+
const date = new Date().toISOString();
31+
const logs = [`[${this.transportType}/${date}]`].concat(args).map((arg) => {
32+
if (typeof arg === 'object' && arg !== null) {
33+
return JSON.stringify(arg, null, 2);
34+
} else {
35+
return arg;
36+
}
37+
});
38+
console.log(...logs);
39+
}
40+
return true;
41+
}
42+
43+
constructor({ url, apiKey, dangerouslyAllowAPIKeyInBrowser, debug } = {}) {
44+
super();
45+
this.url = url || this.defaultUrl;
46+
this.apiKey = apiKey || null;
47+
this.debug = !!debug;
48+
if (globalThis.document && this.apiKey) {
49+
if (!dangerouslyAllowAPIKeyInBrowser) {
50+
throw new Error(
51+
`Can not provide API key in the browser without "dangerouslyAllowAPIKeyInBrowser" set to true`,
52+
);
53+
}
54+
}
55+
}
56+
57+
get isConnected() {
58+
throw new Error("Not implemented: isConnected getter");
59+
}
60+
61+
async connect(options = {}) {
62+
if (!this.apiKey && this.url === this.defaultUrl) {
63+
console.warn(`No apiKey provided for connection to "${this.url}"`);
64+
}
65+
if (this.isConnected) {
66+
throw new Error(`Already connected`);
67+
}
68+
if (globalThis.document && this.apiKey) {
69+
console.warn(
70+
'Warning: Connecting using API key in the browser, this is not recommended',
71+
);
72+
}
73+
}
74+
75+
async disconnect(options = {}) {
76+
throw new Error("Not implemented: disconnect");
77+
}
78+
79+
async send(data) {
80+
if (!this.isConnected) {
81+
throw new Error(`RealtimeAPI is not connected`);
82+
}
83+
}
84+
}

0 commit comments

Comments
 (0)