feat(generative-ai): Add function calling snippets (#3673)

* feat(generative-ai): Add sample for basic text generation.

* chore: fix lint issues

* feat: Add basic multimodal example

* chore: add header

* feat: Add streaming inference example

* feat: Add example for multimodal streaming response

* fix: Fix some tests that were not running correctly

* chore: Clarify test names

* fix: Adjust cloud storage bucket and argurment order for consistency

* feat(vertexai): Add Function calling snippets

Move existing snippets to new folder, and update tests

* feat(generative-ai): Add basic samples for Gemini / VertexAI inference (#3670)

* feat(generative-ai): Add sample for basic text generation.

* chore: fix lint issues

* feat: Add basic multimodal example

* chore: add header

* feat: Add streaming inference example

* feat: Add example for multimodal streaming response

* fix: Fix some tests that were not running correctly

* chore: Clarify test names

* fix: Adjust cloud storage bucket and argurment order for consistency

* chore: fix lint errors

* chore: Update some tests and prompt text to be clearer

* Update model to the correct multimodal model
* Clarify multimodal prompt based on order

* chore: Update function calling stream model to address flaky test.

* chore: Update advanced sample to match CURL

* fix: Fix function name

* chore: lint

* fix: Update safety settings generation settings to make a passing test _more likely_

* fix: Update copyright year

* chore: fix lint

* fix: Settle on model for funciton tests

generative-ai/snippets/function-calling/functionCallingAdvanced.js

@@ -0,0 +1,99 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// [START generativeaionvertexai_function_calling_advanced]
+const {
+  VertexAI,
+  FunctionDeclarationSchemaType,
+} = require('@google-cloud/vertexai');
+
+const functionDeclarations = [
+  {
+    function_declarations: [
+      {
+        name: 'get_product_sku',
+        description:
+          'Get the available inventory for a Google products, e.g: Pixel phones, Pixel Watches, Google Home etc',
+        parameters: {
+          type: FunctionDeclarationSchemaType.OBJECT,
+          properties: {
+            productName: {type: FunctionDeclarationSchemaType.STRING},
+          },
+        },
+      },
+      {
+        name: 'get_store_location',
+        description: 'Get the location of the closest store',
+        parameters: {
+          type: FunctionDeclarationSchemaType.OBJECT,
+          properties: {
+            location: {type: FunctionDeclarationSchemaType.STRING},
+          },
+        },
+      },
+    ],
+  },
+];
+
+const toolConfig = {
+  function_calling_config: {
+    mode: 'ANY',
+    allowed_function_names: ['get_product_sku'],
+  },
+};
+
+const generationConfig = {
+  temperature: 0.95,
+  topP: 1.0,
+  maxOutputTokens: 8192,
+};
+
+/**
+ * TODO(developer): Update these variables before running the sample.
+ */
+async function functionCallingAdvanced(
+  projectId = 'PROJECT_ID',
+  location = 'us-central1',
+  model = 'gemini-1.0-pro-001'
+) {
+  // Initialize Vertex with your Cloud project and location
+  const vertexAI = new VertexAI({project: projectId, location: location});
+
+  // Instantiate the model
+  const generativeModel = vertexAI.preview.getGenerativeModel({
+    model: model,
+  });
+
+  const request = {
+    contents: [
+      {
+        role: 'user',
+        parts: [
+          {text: 'Do you have the White Pixel 8 Pro 128GB in stock in the US?'},
+        ],
+      },
+    ],
+    tools: functionDeclarations,
+    tool_config: toolConfig,
+    generation_config: generationConfig,
+  };
+  const result = await generativeModel.generateContent(request);
+  console.log(JSON.stringify(result.response.candidates[0].content));
+}
+// [END generativeaionvertexai_function_calling_advanced]
+
+functionCallingAdvanced(...process.argv.slice(2)).catch(err => {
+  console.error(err.message);
+  process.exitCode = 1;
+});

generative-ai/snippets/function-calling/functionCallingBasic.js

@@ -0,0 +1,73 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// [START generativeaionvertexai_function_calling_basic]
+const {
+  VertexAI,
+  FunctionDeclarationSchemaType,
+} = require('@google-cloud/vertexai');
+
+const functionDeclarations = [
+  {
+    function_declarations: [
+      {
+        name: 'get_current_weather',
+        description: 'get weather in a given location',
+        parameters: {
+          type: FunctionDeclarationSchemaType.OBJECT,
+          properties: {
+            location: {type: FunctionDeclarationSchemaType.STRING},
+            unit: {
+              type: FunctionDeclarationSchemaType.STRING,
+              enum: ['celsius', 'fahrenheit'],
+            },
+          },
+          required: ['location'],
+        },
+      },
+    ],
+  },
+];
+
+/**
+ * TODO(developer): Update these variables before running the sample.
+ */
+async function functionCallingBasic(
+  projectId = 'PROJECT_ID',
+  location = 'us-central1',
+  model = 'gemini-1.0-pro-001'
+) {
+  // Initialize Vertex with your Cloud project and location
+  const vertexAI = new VertexAI({project: projectId, location: location});
+
+  // Instantiate the model
+  const generativeModel = vertexAI.preview.getGenerativeModel({
+    model: model,
+  });
+
+  const request = {
+    contents: [
+      {role: 'user', parts: [{text: 'What is the weather in Boston?'}]},
+    ],
+    tools: functionDeclarations,
+  };
+  const result = await generativeModel.generateContent(request);
+  console.log(JSON.stringify(result.response.candidates[0].content));
+}
+// [END generativeaionvertexai_function_calling_basic]
+
+functionCallingBasic(...process.argv.slice(2)).catch(err => {
+  console.error(err.message);
+  process.exitCode = 1;
+});

generative-ai/snippets/safetySettings.js

@@ -41,7 +41,12 @@ async function setSafetySettings(
         threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE,
       },
     ],
-    generation_config: {max_output_tokens: 256},
+    generation_config: {
+      max_output_tokens: 256,
+      temperature: 0.4,
+      top_p: 1,
+      top_k: 16,
+    },
   });
 
   const request = {
@@ -64,6 +69,7 @@ async function setSafetySettings(
       process.stdout.write(item.candidates[0].content.parts[0].text);
     }
   }
+  console.log('This response stream terminated due to safety concerns.');
 }
 // [END aiplatform_gemini_safety_settings]
 

generative-ai/snippets/test/function-calling/functionCallingAdvanced.test.js

@@ -0,0 +1,43 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+'use strict';
+
+const {assert} = require('chai');
+const {describe, it} = require('mocha');
+const cp = require('child_process');
+const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'});
+
+const projectId = process.env.CAIP_PROJECT_ID;
+const location = process.env.LOCATION;
+const model = 'gemini-1.0-pro-001';
+
+describe('Generative AI Function Calling Advanced', () => {
+  /**
+   * TODO(developer): Uncomment these variables before running the sample.\
+   * (Not necessary if passing values as arguments)
+   */
+  // const projectId = 'YOUR_PROJECT_ID';
+  // const location = 'YOUR_LOCATION';
+  // const model = 'gemini-1.0-pro';
+
+  it('should define multiple functions and have the model invoke the specified one', async () => {
+    const output = execSync(
+      `node ./function-calling/functionCallingAdvanced.js ${projectId} ${location} ${model}`
+    );
+
+    // Assert that the response is what we expect
+    assert(output.match(/get_product_sku/));
+  });
+});

generative-ai/snippets/test/function-calling/functionCallingBasic.test.js

@@ -0,0 +1,43 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+'use strict';
+
+const {assert} = require('chai');
+const {describe, it} = require('mocha');
+const cp = require('child_process');
+const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'});
+
+const projectId = process.env.CAIP_PROJECT_ID;
+const location = process.env.LOCATION;
+const model = 'gemini-1.0-pro-001';
+
+describe('Generative AI Function Calling', () => {
+  /**
+   * TODO(developer): Uncomment these variables before running the sample.\
+   * (Not necessary if passing values as arguments)
+   */
+  // const projectId = 'YOUR_PROJECT_ID';
+  // const location = 'YOUR_LOCATION';
+  // const model = 'gemini-1.0-pro';
+
+  it('should define a function and have the model invoke it', async () => {
+    const output = execSync(
+      `node ./function-calling/functionCallingBasic.js ${projectId} ${location} ${model}`
+    );
+
+    // Assert that the response is what we expect
+    assert(output.length > 0);
+  });
+});

generative-ai/snippets/test/functionCallingStreamChat.test.js → generative-ai/snippets/test/function-calling/functionCallingStreamChat.test.js

@@ -34,7 +34,7 @@ describe('Generative AI Function Calling Stream Chat', () => {
 
   it('should create stream chat and begin the conversation the same in each instance', async () => {
     const output = execSync(
-      `node ./functionCallingStreamChat.js ${projectId} ${location} ${model}`
+      `node ./function-calling/functionCallingStreamChat.js ${projectId} ${location} ${model}`
     );
 
     // Assert that the response is what we expect

generative-ai/snippets/test/functionCallingStreamContent.test.js → generative-ai/snippets/test/function-calling/functionCallingStreamContent.test.js

@@ -34,7 +34,7 @@ describe('Generative AI Function Calling Stream Content', () => {
 
   it('should create stream chat and begin the conversation the same in each instance', async () => {
     const output = execSync(
-      `node ./functionCallingStreamContent.js ${projectId} ${location} ${model}`
+      `node ./function-calling/functionCallingStreamContent.js ${projectId} ${location} ${model}`
     );
 
     // Assert that the response is what we expect