Typed events

lib/api.js

@@ -5,7 +5,6 @@ export class RealtimeAPI extends RealtimeEventHandler {
   /**
    * Create a new RealtimeAPI instance
    * @param {{url?: string, apiKey?: string, dangerouslyAllowAPIKeyInBrowser?: boolean, debug?: boolean}} [settings]
-   * @returns {RealtimeAPI}
    */
   constructor({ url, apiKey, dangerouslyAllowAPIKeyInBrowser, debug } = {}) {
     super();
@@ -14,6 +13,7 @@ export class RealtimeAPI extends RealtimeEventHandler {
     this.apiKey = apiKey || null;
     this.debug = !!debug;
     this.ws = null;
+
     if (globalThis.document && this.apiKey) {
       if (!dangerouslyAllowAPIKeyInBrowser) {
         throw new Error(
@@ -156,7 +156,7 @@ export class RealtimeAPI extends RealtimeEventHandler {
 
   /**
    * Disconnects from Realtime API server
-   * @param {WebSocket} [ws]
+   * @param {typeof globalThis.WebSocket} [ws]
    * @returns {true}
    */
   disconnect(ws) {
@@ -182,15 +182,12 @@ export class RealtimeAPI extends RealtimeEventHandler {
 
   /**
    * Sends an event to WebSocket and dispatches as "client.{eventName}" and "client.*" events
-   * @param {string} eventName
-   * @param {{[key: string]: any}} event
-   * @returns {true}
+   * @type {import('./types').SendEvent} 
    */
   send(eventName, data) {
     if (!this.isConnected()) {
       throw new Error(`RealtimeAPI is not connected`);
     }
-    data = data || {};
     if (typeof data !== 'object') {
       throw new Error(`data must be an object`);
     }

lib/client.js

@@ -59,7 +59,7 @@ import { RealtimeUtils } from './utils.js';
 /**
  * @typedef {Object} InputAudioContentType
  * @property {"input_audio"} type
- * @property {string} [audio] base64-encoded audio data
+ * @property {string|ArrayBuffer|Int16Array} [audio] base64-encoded audio data
  * @property {string|null} [transcript]
  */
 
@@ -118,6 +118,7 @@ import { RealtimeUtils } from './utils.js';
  * @property {string|null} [previous_item_id]
  * @property {"function_call_output"} type
  * @property {string} call_id
+ * @property {string} status
  * @property {string} output
  */
 
@@ -143,6 +144,7 @@ import { RealtimeUtils } from './utils.js';
  * @typedef {Object} FormattedItemType
  * @property {string} id
  * @property {string} object
+ * @property {string} status
  * @property {"user"|"assistant"|"system"} [role]
  * @property {FormattedPropertyType} formatted
  */
@@ -193,6 +195,7 @@ export class RealtimeClient extends RealtimeEventHandler {
    */
   constructor({ url, apiKey, dangerouslyAllowAPIKeyInBrowser, debug } = {}) {
     super();
+    /* @type { import('./types').SessionConfig }*/
     this.defaultSessionConfig = {
       modalities: ['text', 'audio'],
       instructions: '',
@@ -295,6 +298,7 @@ export class RealtimeClient extends RealtimeEventHandler {
           throw new Error(`Tool "${tool.name}" has not been added`);
         }
         const result = await toolConfig.handler(jsonArguments);
+
         this.realtime.send('conversation.item.create', {
           item: {
             type: 'function_call_output',
@@ -344,6 +348,7 @@ export class RealtimeClient extends RealtimeEventHandler {
       'server.response.audio_transcript.delta',
       handlerWithDispatch,
     );
+
     this.realtime.on('server.response.audio.delta', handlerWithDispatch);
     this.realtime.on('server.response.text.delta', handlerWithDispatch);
     this.realtime.on(
@@ -533,7 +538,7 @@ export class RealtimeClient extends RealtimeEventHandler {
         };
       }),
     );
-    const session = { ...this.sessionConfig };
+    const session = { ...this.sessionConfig, tools: useTools };
     session.tools = useTools;
     if (this.realtime.isConnected()) {
       this.realtime.send('session.update', { session });
@@ -559,6 +564,7 @@ export class RealtimeClient extends RealtimeEventHandler {
         item: {
           type: 'message',
           role: 'user',
+          //@ts-ignore TODO fix
           content,
         },
       });
@@ -594,11 +600,11 @@ export class RealtimeClient extends RealtimeEventHandler {
       this.getTurnDetectionType() === null &&
       this.inputAudioBuffer.byteLength > 0
     ) {
-      this.realtime.send('input_audio_buffer.commit');
+      this.realtime.send('input_audio_buffer.commit', null);
       this.conversation.queueInputAudio(this.inputAudioBuffer);
       this.inputAudioBuffer = new Int16Array(0);
     }
-    this.realtime.send('response.create');
+    this.realtime.send('response.create', null);
     return true;
   }
 
@@ -611,7 +617,7 @@ export class RealtimeClient extends RealtimeEventHandler {
    */
   cancelResponse(id, sampleCount = 0) {
     if (!id) {
-      this.realtime.send('response.cancel');
+      this.realtime.send('response.cancel', null);
       return { item: null };
     } else if (id) {
       const item = this.conversation.getItem(id);
@@ -625,7 +631,7 @@ export class RealtimeClient extends RealtimeEventHandler {
           `Can only cancelResponse messages with role "assistant"`,
         );
       }
-      this.realtime.send('response.cancel');
+      this.realtime.send('response.cancel', null);
       const audioIndex = item.content.findIndex((c) => c.type === 'audio');
       if (audioIndex === -1) {
         throw new Error(`Could not find audio on item to cancel`);
@@ -643,7 +649,6 @@ export class RealtimeClient extends RealtimeEventHandler {
 
   /**
    * Utility for waiting for the next `conversation.item.appended` event to be triggered by the server
-   * @returns {Promise<{item: ItemType}>}
    */
   async waitForNextItem() {
     const event = await this.waitForNext('conversation.item.appended');
@@ -653,7 +658,6 @@ export class RealtimeClient extends RealtimeEventHandler {
 
   /**
    * Utility for waiting for the next `conversation.item.completed` event to be triggered by the server
-   * @returns {Promise<{item: ItemType}>}
    */
   async waitForNextCompletedItem() {
     const event = await this.waitForNext('conversation.item.completed');

lib/conversation.js

@@ -18,7 +18,8 @@ import { RealtimeUtils } from './utils.js';
 export class RealtimeConversation {
   defaultFrequency = 24_000; // 24,000 Hz
 
-  EventProcessors = {
+  /** @type { import('./types').EventProcessors} */
+  eventProcessors = {
     'conversation.item.created': (event) => {
       const { item } = event;
       // deep copy values
@@ -240,7 +241,6 @@ export class RealtimeConversation {
 
   /**
    * Create a new RealtimeConversation instance
-   * @returns {RealtimeConversation}
    */
   constructor() {
     this.clear();
@@ -275,7 +275,7 @@ export class RealtimeConversation {
    * Process an event from the WebSocket server and compose items
    * @param {Object} event
    * @param  {...any} args
-   * @returns {item: import('./client.js').ItemType | null, delta: ItemContentDeltaType | null}
+   * @returns {{item: import('./client.js').ItemType | null, delta: ItemContentDeltaType | null}}
    */
   processEvent(event, ...args) {
     if (!event.event_id) {
@@ -286,7 +286,7 @@ export class RealtimeConversation {
       console.error(event);
       throw new Error(`Missing "type" on event`);
     }
-    const eventProcessor = this.EventProcessors[event.type];
+    const eventProcessor = this.eventProcessors[event.type];
     if (!eventProcessor) {
       throw new Error(
         `Missing conversation event processor for "${event.type}"`,

lib/event_handler.js

@@ -1,6 +1,10 @@
 /**
- * EventHandler callback
- * @typedef {(event: {[key: string]: any}): void} EventHandlerCallbackType
+ * @typedef {import('./types').Listener} Listener
+ * @typedef {import('./types').ListenerBool} ListenerBool
+ * @typedef {import('./types').WaitForNext} WaitForNext
+ * @typedef {import('./types').EventNames} EventNames
+ * @typedef {Object.<EventNames, Listener[]>} EventHandlers
+
  */
 
 const sleep = (t) => new Promise((r) => setTimeout(() => r(), t));
@@ -13,10 +17,11 @@ const sleep = (t) => new Promise((r) => setTimeout(() => r(), t));
 export class RealtimeEventHandler {
   /**
    * Create a new RealtimeEventHandler instance
-   * @returns {RealtimeEventHandler}
    */
   constructor() {
+     /** @type {EventHandlers} */
     this.eventHandlers = {};
+     /** @type {EventHandlers} */
     this.nextEventHandlers = {};
   }
 
@@ -30,11 +35,9 @@ export class RealtimeEventHandler {
     return true;
   }
 
-  /**
-   * Listen to specific events
-   * @param {string} eventName The name of the event to listen to
-   * @param {EventHandlerCallbackType} callback Code to execute on event
-   * @returns {EventHandlerCallbackType}
+ /**
+   * Register an event listener
+   * @type {Listener}
    */
   on(eventName, callback) {
     this.eventHandlers[eventName] = this.eventHandlers[eventName] || [];
@@ -44,9 +47,7 @@ export class RealtimeEventHandler {
 
   /**
    * Listen for the next event of a specified type
-   * @param {string} eventName The name of the event to listen to
-   * @param {EventHandlerCallbackType} callback Code to execute on event
-   * @returns {EventHandlerCallbackType}
+   * @type {Listener}
    */
   onNext(eventName, callback) {
     this.nextEventHandlers[eventName] = this.nextEventHandlers[eventName] || [];
@@ -57,9 +58,7 @@ export class RealtimeEventHandler {
   /**
    * Turns off event listening for specific events
    * Calling without a callback will remove all listeners for the event
-   * @param {string} eventName
-   * @param {EventHandlerCallbackType} [callback]
-   * @returns {true}
+   * @type {ListenerBool}
    */
   off(eventName, callback) {
     const handlers = this.eventHandlers[eventName] || [];
@@ -80,9 +79,7 @@ export class RealtimeEventHandler {
   /**
    * Turns off event listening for the next event of a specific type
    * Calling without a callback will remove all listeners for the next event
-   * @param {string} eventName
-   * @param {EventHandlerCallbackType} [callback]
-   * @returns {true}
+   * @type {ListenerBool}
    */
   offNext(eventName, callback) {
     const nextHandlers = this.nextEventHandlers[eventName] || [];
@@ -102,9 +99,7 @@ export class RealtimeEventHandler {
 
   /**
    * Waits for next event of a specific type and returns the payload
-   * @param {string} eventName
-   * @param {number|null} [timeout]
-   * @returns {Promise<{[key: string]: any}|null>}
+   * @type {WaitForNext}
    */
   async waitForNext(eventName, timeout = null) {
     const t0 = Date.now();