Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor RealtimeClient & RealtimeAPI to specify transport of either WebRTC (default) or WebSocket #99

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+421 −145
Open

Refactor RealtimeClient & RealtimeAPI to specify transport of either WebRTC (default) or WebSocket #99

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
192 changes: 63 additions & 129 deletions lib/api.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,44 +1,62 @@
import { RealtimeEventHandler } from './event_handler.js';
import { RealtimeUtils } from './utils.js';
import { RealtimeTransportType } from './transport.js';
import { RealtimeTransportWebRTC } from './transport_webrtc.js';
import { RealtimeTransportWebSocket } from './transport_websocket.js';

export class RealtimeAPI extends RealtimeEventHandler {
/**
* Create a new RealtimeAPI instance
* @param {{url?: string, apiKey?: string, dangerouslyAllowAPIKeyInBrowser?: boolean, debug?: boolean}} [settings]
* @param {{transportType?: string, url?: string, apiKey?: string, dangerouslyAllowAPIKeyInBrowser?: boolean, debug?: boolean}} [settings]
* @returns {RealtimeAPI}
*/
constructor({ url, apiKey, dangerouslyAllowAPIKeyInBrowser, debug } = {}) {
constructor({ transportType, url, apiKey, dangerouslyAllowAPIKeyInBrowser, debug } = {}) {
super();
this.defaultUrl = 'wss://api.openai.com/v1/realtime';
this.url = url || this.defaultUrl;
this.apiKey = apiKey || null;
this.debug = !!debug;
this.ws = null;
if (globalThis.document && this.apiKey) {
if (!dangerouslyAllowAPIKeyInBrowser) {
throw new Error(
`Can not provide API key in the browser without "dangerouslyAllowAPIKeyInBrowser" set to true`,
);
transportType = transportType?.toUpperCase() || RealtimeTransportType.WEBRTC;
switch (transportType) {
case RealtimeTransportType.WEBRTC: {
this.transport = new RealtimeTransportWebRTC({ url, apiKey, dangerouslyAllowAPIKeyInBrowser, debug });
break;
}
case RealtimeTransportType.WEBSOCKET: {
this.transport = new RealtimeTransportWebSocket({ url, apiKey, dangerouslyAllowAPIKeyInBrowser, debug });
break;
}
default: {
throw new Error(`Invalid transportType: "${transportType}"`);
}
}
this.transport.on('close', (data) => {
this.disconnect();
this.dispatch('close', data);
});
this.transport.on('message', (event) => {
const message = JSON.parse(event.data);
this._receive(message.type, message)
});
}

get transportType() {
return this.transport.transportType;
}

/**
* Tells us whether or not the WebSocket is connected
* Tells us whether or not the Realtime API server is connected
* @returns {boolean}
*/
isConnected() {
return !!this.ws;
get isConnected() {
return this.transport.isConnected;
}

/**
* Writes WebSocket logs to console
* Writes log to console
* @param {...any} args
* @returns {true}
*/
log(...args) {
const date = new Date().toISOString();
const logs = [`[Websocket/${date}]`].concat(args).map((arg) => {
const logs = [`[RealtimeAPI/${date}]`].concat(args).map((arg) => {
if (typeof arg === 'object' && arg !== null) {
return JSON.stringify(arg, null, 2);
} else {
Expand All @@ -52,142 +70,51 @@ export class RealtimeAPI extends RealtimeEventHandler {
}

/**
* Connects to Realtime API Websocket Server
* @param {{model?: string}} [settings]
* Connects to Realtime API Server
* @param {{sessionConfig?: SessionConfig, setAudioOutputCallback?: Function, getMicrophoneCallback?: Function}} [settings]
* @returns {Promise<true>}
*/
async connect({ model } = { model: 'gpt-4o-realtime-preview-2024-10-01' }) {
if (!this.apiKey && this.url === this.defaultUrl) {
console.warn(`No apiKey provided for connection to "${this.url}"`);
}
if (this.isConnected()) {
throw new Error(`Already connected`);
}
if (globalThis.WebSocket) {
/**
* Web browser
*/
if (globalThis.document && this.apiKey) {
console.warn(
'Warning: Connecting using API key in the browser, this is not recommended',
);
}
const WebSocket = globalThis.WebSocket;
const ws = new WebSocket(`${this.url}${model ? `?model=${model}` : ''}`, [
'realtime',
`openai-insecure-api-key.${this.apiKey}`,
'openai-beta.realtime-v1',
]);
ws.addEventListener('message', (event) => {
const message = JSON.parse(event.data);
this.receive(message.type, message);
});
return new Promise((resolve, reject) => {
const connectionErrorHandler = () => {
this.disconnect(ws);
reject(new Error(`Could not connect to "${this.url}"`));
};
ws.addEventListener('error', connectionErrorHandler);
ws.addEventListener('open', () => {
this.log(`Connected to "${this.url}"`);
ws.removeEventListener('error', connectionErrorHandler);
ws.addEventListener('error', () => {
this.disconnect(ws);
this.log(`Error, disconnected from "${this.url}"`);
this.dispatch('close', { error: true });
});
ws.addEventListener('close', () => {
this.disconnect(ws);
this.log(`Disconnected from "${this.url}"`);
this.dispatch('close', { error: false });
});
this.ws = ws;
resolve(true);
});
});
} else {
/**
* Node.js
*/
const moduleName = 'ws';
const wsModule = await import(/* webpackIgnore: true */ moduleName);
const WebSocket = wsModule.default;
const ws = new WebSocket(
'wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview-2024-10-01',
[],
{
finishRequest: (request) => {
// Auth
request.setHeader('Authorization', `Bearer ${this.apiKey}`);
request.setHeader('OpenAI-Beta', 'realtime=v1');
request.end();
},
},
);
ws.on('message', (data) => {
const message = JSON.parse(data.toString());
this.receive(message.type, message);
});
return new Promise((resolve, reject) => {
const connectionErrorHandler = () => {
this.disconnect(ws);
reject(new Error(`Could not connect to "${this.url}"`));
};
ws.on('error', connectionErrorHandler);
ws.on('open', () => {
this.log(`Connected to "${this.url}"`);
ws.removeListener('error', connectionErrorHandler);
ws.on('error', () => {
this.disconnect(ws);
this.log(`Error, disconnected from "${this.url}"`);
this.dispatch('close', { error: true });
});
ws.on('close', () => {
this.disconnect(ws);
this.log(`Disconnected from "${this.url}"`);
this.dispatch('close', { error: false });
});
this.ws = ws;
resolve(true);
});
});
}
async connect({ sessionConfig, setAudioOutputCallback, getMicrophoneCallback }) {
return this.transport.connect({ sessionConfig, setAudioOutputCallback, getMicrophoneCallback });
}

/**
* Disconnects from Realtime API server
* @param {WebSocket} [ws]
* @returns {true}
*/
disconnect(ws) {
if (!ws || this.ws === ws) {
this.ws && this.ws.close();
this.ws = null;
return true;
}
async disconnect() {
await this.transport.disconnect();
return true;
}

/**
* Receives an event from WebSocket and dispatches as "server.{eventName}" and "server.*" events
* Receives an event from Realtime API server and dispatches as "server.{eventName}" and "server.*" events
* @param {string} eventName
* @param {{[key: string]: any}} event
* @returns {true}
*/
receive(eventName, event) {
this.log(`received:`, eventName, event);
_receive(eventName, event) {
if (this.debug) {
if (eventName === 'response.audio.delta') {
const delta = event.delta;
this.log(`received:`, eventName, { ...event, delta: delta.slice(0, 10) + '...' + delta.slice(-10) });
} else {
this.log(`received:`, eventName, event);
}
}
this.dispatch(`server.${eventName}`, event);
this.dispatch('server.*', event);
return true;
}

/**
* Sends an event to WebSocket and dispatches as "client.{eventName}" and "client.*" events
* Sends an event to Realtime API server and dispatches as "client.{eventName}" and "client.*" events
* @param {string} eventName
* @param {{[key: string]: any}} event
* @returns {true}
*/
send(eventName, data) {
if (!this.isConnected()) {
async send(eventName, data) {
if (!this.isConnected) {
throw new Error(`RealtimeAPI is not connected`);
}
data = data || {};
Expand All @@ -201,8 +128,15 @@ export class RealtimeAPI extends RealtimeEventHandler {
};
this.dispatch(`client.${eventName}`, event);
this.dispatch('client.*', event);
this.log(`sent:`, eventName, event);
this.ws.send(JSON.stringify(event));
if (this.debug) {
if (eventName === 'input_audio_buffer.append') {
const audio = event.audio;
this.log(`sending:`, eventName, { ...event, audio: audio.slice(0, 10) + '...' + audio.slice(-10) });
} else {
this.log(`sending:`, eventName, event);
}
}
await this.transport.send(event);
return true;
}
}
Loading