Add Streaming support

Author: Graden Rea

Diff for: src/core.ts

@@ -1,4 +1,5 @@
 import { VERSION } from './version';
+import { Stream } from './lib/streaming';
 import {
   GroqError,
   APIError,
@@ -38,6 +39,19 @@ type APIResponseProps = {
 
 async function defaultParseResponse<T>(props: APIResponseProps): Promise<T> {
   const { response } = props;
+  if (props.options.stream) {
+    debug('response', response.status, response.url, response.headers, response.body);
+
+    // Note: there is an invariant here that isn't represented in the type system
+    // that if you set `stream: true` the response type must also be `Stream<T>`
+
+    if (props.options.__streamClass) {
+      return props.options.__streamClass.fromSSEResponse(response, props.controller) as any;
+    }
+
+    return Stream.fromSSEResponse(response, props.controller) as any;
+  }
+
   // fetch refuses to read the body when the status code is 204.
   if (response.status === 204) {
     return null as T;
@@ -736,6 +750,7 @@ export type RequestOptions<Req = unknown | Record<string, unknown> | Readable> =
   idempotencyKey?: string;
 
   __binaryResponse?: boolean | undefined;
+  __streamClass?: typeof Stream;
 };
 
 // This is required so that we can determine if a given object matches the RequestOptions
@@ -756,6 +771,7 @@ const requestOptionsKeys: KeysEnum<RequestOptions> = {
   idempotencyKey: true,
 
   __binaryResponse: true,
+  __streamClass: true,
 };
 
 export const isRequestOptions = (obj: unknown): obj is RequestOptions => {

Diff for: src/lib/chat_completions_ext.ts

@@ -0,0 +1,89 @@
+// Manually curated models for streaming chat completions.
+
+export interface ChatCompletionChunk {
+  id: string;
+
+  choices: Array<ChatCompletionChunk.Choice>;
+
+  created: number;
+
+  model: string;
+
+  object: 'chat.completion.chunk';
+
+  system_fingerprint?: string;
+}
+
+export namespace ChatCompletionChunk {
+  export interface Choice {
+    delta: Choice.Delta;
+
+    finish_reason: 'stop' | 'length' | 'tool_calls' | 'content_filter' | 'function_call' | null;
+
+    index: number;
+
+    logprobs?: Choice.Logprobs | null;
+  }
+
+  export namespace Choice {
+    export interface Delta {
+      content?: string | null;
+
+      function_call?: Delta.FunctionCall;
+
+      role?: 'system' | 'user' | 'assistant' | 'tool';
+
+      tool_calls?: Array<Delta.ToolCall>;
+    }
+
+    export namespace Delta {
+      export interface FunctionCall {
+        arguments?: string;
+
+        name?: string;
+      }
+
+      export interface ToolCall {
+        index: number;
+
+        id?: string;
+
+        function?: ToolCall.Function;
+
+        type?: 'function';
+      }
+
+      export namespace ToolCall {
+        export interface Function {
+          arguments?: string;
+
+          name?: string;
+        }
+      }
+    }
+
+    export interface Logprobs {
+      content: Array<ChatCompletionTokenLogprob> | null;
+    }
+  }
+}
+
+export interface ChatCompletionTokenLogprob {
+  token: string;
+
+  bytes: Array<number> | null;
+
+  logprob: number;
+
+  top_logprobs: Array<ChatCompletionTokenLogprob.TopLogprob>;
+}
+
+export namespace ChatCompletionTokenLogprob {
+  export interface TopLogprob {
+    token: string;
+
+    bytes: Array<number> | null;
+
+    logprob: number;
+  }
+}