Merge pull request #112 from PAIR-code/iislucas-playing-with-npm-ts-llmt

moved to use ts-llmt templates library

llm-recs/appengine-server/package.json

@@ -13,6 +13,7 @@
   "dependencies": {
     "express": "^4.18.2",
     "google-auth-library": "^9.2.0",
+    "ts-llmt": "^1.0.1",
     "ts-node": "^10.9.1",
     "underscore": "^1.13.6"
   },

llm-recs/appengine-server/server.ts

@@ -9,39 +9,39 @@
 import express, { Express, Request, Response } from 'express';
 import { GoogleAuth } from 'google-auth-library';
 import { VertexEmbedder } from './src/text-embeddings/embedder_gaxois_vertexapi';
-import { VertexPalm2LLM, Palm2ApiOptions, Palm2ApiParams } from './src/text-templates/llm_gaxois_vertexapi_palm2';
-import * as path from 'path';
+import { VertexGeminiLLM } from 'ts-llmt/src/llm_vertexapi_gemini_1_5_googleauth';
+import {
+  GeminiApiOptions,
+  GeminiRequestOptions,
+} from 'ts-llmt/src/llm_vertexapi_gemini_lib';
 
 export const app: Express = express();
 app.use(express.static('./static'));
-app.use(express.json());  // for POST requests
-app.use(express.urlencoded({ extended: true }));  // for PUT requests
+app.use(express.json()); // for POST requests
+app.use(express.urlencoded({ extended: true })); // for PUT requests
 
 export interface LlmOptions {
   modelId?: string; // e.g. text-bison
-  candidateCount?: number, // e.g. 1 to 8 = number of completions
-  maxOutputTokens?: number, // e.g. 256, 1024
-  stopSequences?: string[], // e.g. ']
-  temperature?: number,  // e.g. 0.8 (0=deterministic, 0.7-0.9=normal, x>1=wild)
-  topP?: number,  // e.g. 0.8 (0-1, smaller = restricts crazyiness)
-  topK?: number  // e.g. 40 (0-numOfTokens, smaller = restricts crazyiness)
+  candidateCount?: number; // e.g. 1 to 8 = number of completions
+  maxOutputTokens?: number; // e.g. 256, 1024
+  stopSequences?: string[]; // e.g. ']
+  temperature?: number; // e.g. 0.8 (0=deterministic, 0.7-0.9=normal, x>1=wild)
+  topP?: number; // e.g. 0.8 (0-1, smaller = restricts crazyiness)
+  topK?: number; // e.g. 40 (0-numOfTokens, smaller = restricts crazyiness)
 }
 
 export interface LlmRequest {
-  text: string
-  params?: LlmOptions
+  text: string;
+  params?: LlmOptions;
 }
 
 export interface SimpleEmbedRequest {
   text: string;
 }
 
 async function main() {
-
   const auth = new GoogleAuth({
-    scopes: [
-      'https://www.googleapis.com/auth/cloud-platform',
-    ]
+    scopes: ['https://www.googleapis.com/auth/cloud-platform'],
   });
   const projectId = await auth.getProjectId();
   const client = await auth.getClient();
@@ -53,7 +53,7 @@ async function main() {
 
   app.post('/api/embed', async (req: Request, res: Response) => {
     console.log(`${new Date()}: /api/embed`);
-    const query = (req.body as SimpleEmbedRequest).text
+    const query = (req.body as SimpleEmbedRequest).text;
     if (query.trim() === '') {
       return res.send({ error: 'Empty string does not have an embedding.' });
     }
@@ -63,24 +63,21 @@ async function main() {
 
   app.post('/api/llm', async (req: Request, res: Response) => {
     console.log(`${new Date()}: /api/llm`);
-    const request = (req.body as LlmRequest)
+    const request = req.body as LlmRequest;
     // Convert from LlmRequest to VertexAI LLM request.
     const inputRequestParameters = { ...request.params };
     delete inputRequestParameters.modelId;
-    const requestParameters = inputRequestParameters as Palm2ApiParams;
+    const requestOptions = inputRequestParameters as GeminiRequestOptions;
 
     // TODO: we now have duplicated definitions of default options, we might want to
     // remove one of them.
-    const options: Palm2ApiOptions = {
-      modelId: (request.params && request.params.modelId) || 'text-bison',
-      apiEndpoint: 'us-central1-aiplatform.googleapis.com',
-      requestParameters,
+    const options: GeminiApiOptions = {
+      modelId:
+        (request.params && request.params.modelId) || 'gemini-1.5-flash-002',
+      location: 'us-central1',
+      requestOptions,
     };
-    const llm = new VertexPalm2LLM(
-      projectId,
-      client,
-      options,
-    );
+    const llm = new VertexGeminiLLM(projectId, client, options);
     const response = await llm.predict(request.text, options);
     res.send(response);
   });

llm-recs/appengine-server/src/text-embeddings/embedder_gaxois_vertexapi.ts

@@ -6,8 +6,8 @@
  * found in the LICENSE file and http://www.apache.org/licenses/LICENSE-2.0
 ==============================================================================*/
 
-import { AuthClient } from "google-auth-library";
-import { Embedder, EmbedResponse as EmbedResponse } from "./embedder";
+import { AuthClient } from 'google-auth-library';
+import { Embedder, EmbedResponse as EmbedResponse } from './embedder';
 
 /*
 Google Cloud Vertex Embedding API
@@ -17,10 +17,10 @@ See: https://cloud.google.com/vertex-ai/docs/generative-ai/embeddings/get-text-e
 Uses Google-auth-client so that it handles refreshing tokens for API use.
 */
 
-export interface EmbedRequestParams { };
+export interface EmbedRequestParams {}
 
 export interface EmbedRequest {
-  instances: { content: string }[]
+  instances: { content: string }[];
 }
 
 export interface VertexEmbedding {
@@ -31,11 +31,11 @@ export interface VertexEmbedding {
         token_count: number;
       };
       values: number[];
-    }
+    };
   }[];
   metadata: {
     billableCharacterCount: number;
-  }
+  };
 }
 
 export interface VertexEmbedError {
@@ -44,20 +44,22 @@ export interface VertexEmbedError {
     details: unknown[];
     message: string;
     status: string;
-  }
+  };
 }
 
 export type VertexEmbedResponse = VertexEmbedding | VertexEmbedError;
 
-function isErrorResponse(response: VertexEmbedResponse
+function isErrorResponse(
+  response: VertexEmbedResponse
 ): response is VertexEmbedError {
   if ((response as VertexEmbedError).error) {
     return true;
   }
   return false;
 }
 
-export function prepareEmbedRequest(text: string,
+export function prepareEmbedRequest(
+  text: string,
   options?: EmbedRequestParams
 ): EmbedRequest {
   return {
@@ -85,21 +87,21 @@ export async function sendEmbedRequest(
   projectId: string,
   client: AuthClient,
   req: EmbedRequest,
-  modelId = 'textembedding-gecko', // e.g. textembedding-gecko embedding model
-  apiEndpoint = 'us-central1-aiplatform.googleapis.com',
+  modelId = 'text-embedding-004', // e.g. textembedding-gecko embedding model
+  location = 'us-central1'
 ): Promise<VertexEmbedResponse> {
   return await postDataToLLM(
     // TODO: it may be that the url part 'us-central1' has to match
     // apiEndpoint.
-    `https://${apiEndpoint}/v1/projects/${projectId}/locations/us-central1/publishers/google/models/${modelId}:predict`,
+    `https://${location}-aiplatform.googleapis.com/v1/projects/${projectId}/locations/${location}/publishers/google/models/${modelId}:predict`,
     client,
     req
   );
 }
 
 interface EmbedApiOptions {
-  modelId: string,
-  apiEndpoint: string,
+  modelId: string;
+  apiEndpoint: string;
 }
 
 export class VertexEmbedder implements Embedder<EmbedApiOptions> {
@@ -109,24 +111,19 @@ export class VertexEmbedder implements Embedder<EmbedApiOptions> {
     apiEndpoint: 'us-central1-aiplatform.googleapis.com',
   };
 
-  constructor(
-    public client: AuthClient,
-    public projectId: string,
-  ) {
+  constructor(public client: AuthClient, public projectId: string) {
     this.name = `VertexEmbedder:` + this.defaultOptions.modelId;
   }
 
-  async embed(
-    query: string, params?: EmbedApiOptions
-  ): Promise<EmbedResponse> {
-
+  async embed(query: string, params?: EmbedApiOptions): Promise<EmbedResponse> {
     const apiRequest = prepareEmbedRequest(query);
     const apiResponse = await sendEmbedRequest(
       this.projectId,
       this.client,
       apiRequest,
       params ? params.modelId : this.defaultOptions.modelId,
-      params ? params.apiEndpoint : this.defaultOptions.apiEndpoint);
+      params ? params.apiEndpoint : this.defaultOptions.apiEndpoint
+    );
     if (isErrorResponse(apiResponse)) {
       return { error: apiResponse.error.message };
     }