Skip to content

Commit 6683aff

Browse files
crickmancrickman
authored
.Net - Update and Fixes for AzureAIAgent (#12348)
### Motivation and Context <!-- Thank you for your contribution to the semantic-kernel repo! Please help reviewers and future users, providing the following information: 1. Why is this change required? 2. What problem does it solve? 3. What scenario does it contribute to? 4. If it fixes an open issue, please link to the issue here. --> Address customer input and discovered issues for `AzureAIAgent` ### Description <!-- Describe your changes, the overall approach, the underlying design. These notes will help understanding how your code works. Thanks! --> Address several issues: - Support image input - Null reference when function calling parameter is null - Null reference when using BingGroundingTool with streaming - Add samples for response-format - Fix connection resolution for declarative bing tool Fixes: #12288 ### Contribution Checklist <!-- Before submitting this PR, please make sure: --> - [X] The code builds clean without any errors or warnings - [X] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [X] All unit tests pass, and I have added new tests where possible - [X] I didn't break anyone 😄
1 parent dda17e4 commit 6683aff

24 files changed

+434
-192
lines changed

‎dotnet/Directory.Packages.props

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
<PackageVersion Include="Azure.AI.ContentSafety" Version="1.0.0" />
2222
<PackageVersion Include="Azure.AI.Inference" Version="1.0.0-beta.5" />
2323
<PackageVersion Include="Azure.AI.OpenAI" Version="[2.2.0-beta.4]" />
24+
<PackageVersion Include="Azure.AI.Projects" Version="1.0.0-beta.9" />
2425
<PackageVersion Include="Azure.Identity" Version="1.13.2" />
2526
<PackageVersion Include="Azure.Monitor.OpenTelemetry.Exporter" Version="1.3.0" />
2627
<PackageVersion Include="Azure.Search.Documents" Version="11.6.0" />

‎dotnet/samples/Concepts/Agents/AzureAIAgent_Streaming.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -199,7 +199,7 @@ public string GetSpecials()
199199
[KernelFunction, Description("Provides the price of the requested menu item.")]
200200
public string GetItemPrice(
201201
[Description("The name of the menu item.")]
202-
string menuItem)
202+
string menuItem)
203203
{
204204
return "$9.99";
205205
}

‎dotnet/samples/Concepts/Concepts.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
3030
<PrivateAssets>all</PrivateAssets>
3131
</PackageReference>
32+
<PackageReference Include="Azure.AI.Projects" />
3233
<PackageReference Include="Azure.Identity" />
3334
<PackageReference Include="Microsoft.Extensions.Configuration" />
3435
<PackageReference Include="Microsoft.Extensions.AI.AzureAIInference" />

‎dotnet/samples/GettingStartedWithAgents/AzureAIAgent/Step03_AzureAIAgent_Chat.cs

Lines changed: 0 additions & 103 deletions
This file was deleted.

‎dotnet/samples/GettingStartedWithAgents/AzureAIAgent/Step03_AzureAIAgent_Vision.cs

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
using Azure.AI.Agents.Persistent;
3+
using Microsoft.SemanticKernel;
4+
using Microsoft.SemanticKernel.Agents.AzureAI;
5+
using Microsoft.SemanticKernel.ChatCompletion;
6+
using Resources;
7+
8+
namespace GettingStarted.AzureAgents;
9+
10+
/// <summary>
11+
/// Demonstrate using code-interpreter on <see cref="AzureAIAgent"/> .
12+
/// </summary>
13+
public class Step03_AzureAIAgent_Vision(ITestOutputHelper output) : BaseAzureAgentTest(output)
14+
{
15+
[Fact]
16+
public async Task UseImageContentWithAgent()
17+
{
18+
// Upload an image
19+
await using Stream imageStream = EmbeddedResource.ReadStream("cat.jpg")!;
20+
PersistentAgentFileInfo fileInfo = await this.Client.Files.UploadFileAsync(imageStream, PersistentAgentFilePurpose.Agents, "cat.jpg");
21+
22+
// Define the agent
23+
PersistentAgent definition = await this.Client.Administration.CreateAgentAsync(TestConfiguration.AzureAI.ChatModelId);
24+
AzureAIAgent agent = new(definition, this.Client);
25+
26+
// Create a thread for the agent conversation.
27+
AzureAIAgentThread thread = new(this.Client, metadata: SampleMetadata);
28+
29+
// Respond to user input
30+
try
31+
{
32+
// Refer to public image by url
33+
await InvokeAgentAsync(CreateMessageWithImageUrl("Describe this image.", "https://upload.wikimedia.org/wikipedia/commons/thumb/4/47/New_york_times_square-terabass.jpg/1200px-New_york_times_square-terabass.jpg"));
34+
await InvokeAgentAsync(CreateMessageWithImageUrl("What are is the main color in this image?", "https://upload.wikimedia.org/wikipedia/commons/5/56/White_shark.jpg"));
35+
// Refer to uploaded image by file-id.
36+
await InvokeAgentAsync(CreateMessageWithImageReference("Is there an animal in this image?", fileInfo.Id));
37+
}
38+
finally
39+
{
40+
await thread.DeleteAsync();
41+
await this.Client.Administration.DeleteAgentAsync(agent.Id);
42+
await this.Client.Files.DeleteFileAsync(fileInfo.Id);
43+
}
44+
45+
// Local function to invoke agent and display the conversation messages.
46+
async Task InvokeAgentAsync(ChatMessageContent input)
47+
{
48+
this.WriteAgentChatMessage(input);
49+
50+
await foreach (ChatMessageContent response in agent.InvokeAsync(input, thread))
51+
{
52+
this.WriteAgentChatMessage(response);
53+
}
54+
}
55+
}
56+
57+
private ChatMessageContent CreateMessageWithImageUrl(string input, string url)
58+
=> new(AuthorRole.User, [new TextContent(input), new ImageContent(new Uri(url))]);
59+
60+
private ChatMessageContent CreateMessageWithImageReference(string input, string fileId)
61+
=> new(AuthorRole.User, [new TextContent(input), new FileReferenceContent(fileId)]);
62+
}

‎dotnet/samples/GettingStartedWithAgents/AzureAIAgent/Step08_AzureAIAgent_Declarative.cs

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -144,20 +144,20 @@ public async Task AzureAIAgentWithFunctions()
144144
public async Task AzureAIAgentWithBingGrounding()
145145
{
146146
var text =
147-
$"""
147+
"""
148148
type: foundry_agent
149149
name: BingAgent
150150
instructions: Answer questions using Bing to provide grounding context.
151151
description: This agent answers questions using Bing to provide grounding context.
152152
model:
153-
id: ${TestConfiguration.AzureAI:ChatModelId}
153+
id: ${AzureAI:ChatModelId}
154154
options:
155155
temperature: 0.4
156156
tools:
157157
- type: bing_grounding
158158
options:
159159
tool_connections:
160-
- {TestConfiguration.AzureAI.BingConnectionId}
160+
- ${AzureAI:BingConnectionId}
161161
""";
162162
AzureAIAgentFactory factory = new();
163163

@@ -173,21 +173,21 @@ public async Task AzureAIAgentWithBingGrounding()
173173
public async Task AzureAIAgentWithFileSearch()
174174
{
175175
var text =
176-
$"""
176+
"""
177177
type: foundry_agent
178178
name: FileSearchAgent
179179
instructions: Answer questions using available files to provide grounding context.
180180
description: This agent answers questions using available files to provide grounding context.
181181
model:
182-
id: ${TestConfiguration.AzureAI:ChatModelId}
183-
options:
182+
id: ${AzureAI:ChatModelId}
183+
optisons:
184184
temperature: 0.4
185185
tools:
186186
- type: file_search
187187
description: Grounding with available files.
188188
options:
189189
vector_store_ids:
190-
- {TestConfiguration.AzureAI.VectorStoreId}
190+
- ${AzureAI.VectorStoreId}
191191
""";
192192
AzureAIAgentFactory factory = new();
193193

@@ -418,6 +418,7 @@ public Step08_AzureAIAgent_Declarative(ITestOutputHelper output) : base(output)
418418
{
419419
var builder = Kernel.CreateBuilder();
420420
builder.Services.AddSingleton(this.Client);
421+
builder.Services.AddSingleton(this.CreateFoundryProjectClient());
421422
this._kernel = builder.Build();
422423
}
423424

@@ -438,10 +439,6 @@ private async Task InvokeAgentAsync(Agent agent, string input, bool? deleteAgent
438439
WriteAgentChatMessage(response);
439440
}
440441
}
441-
catch (Exception e)
442-
{
443-
Console.WriteLine($"Error invoking agent: {e.Message}");
444-
}
445442
finally
446443
{
447444
if (deleteAgent ?? true)

‎dotnet/samples/GettingStartedWithAgents/AzureAIAgent/Step09_AzureAIAgent_BingGrounding.cs

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@ public class Step09_AzureAIAgent_BingGrounding(ITestOutputHelper output) : BaseA
1616
public async Task UseBingGroundingToolWithAgent()
1717
{
1818
// Access the BingGrounding connection
19-
BingGroundingSearchConfiguration bingToolConfiguration = new(TestConfiguration.AzureAI.BingConnectionId);
19+
string connectionId = await this.GetConnectionId(TestConfiguration.AzureAI.BingConnectionId);
20+
BingGroundingSearchConfiguration bingToolConfiguration = new(connectionId);
2021
BingGroundingSearchToolParameters bingToolParameters = new([bingToolConfiguration]);
2122
PersistentAgent definition = await this.Client.Administration.CreateAgentAsync(
2223
TestConfiguration.AzureAI.ChatModelId,
@@ -55,7 +56,8 @@ async Task InvokeAgentAsync(string input)
5556
public async Task UseBingGroundingToolWithStreaming()
5657
{
5758
// Access the BingGrounding connection
58-
BingGroundingSearchConfiguration bingToolConfiguration = new(TestConfiguration.AzureAI.BingConnectionId);
59+
string connectionId = await this.GetConnectionId(TestConfiguration.AzureAI.BingConnectionId);
60+
BingGroundingSearchConfiguration bingToolConfiguration = new(connectionId);
5961
BingGroundingSearchToolParameters bingToolParameters = new([bingToolConfiguration]);
6062

6163
// Define the agent

‎dotnet/samples/GettingStartedWithAgents/AzureAIAgent/Step10_JsonResponse.cs

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
using Azure.AI.Agents.Persistent;
3+
using Microsoft.SemanticKernel;
4+
using Microsoft.SemanticKernel.Agents.AzureAI;
5+
using Microsoft.SemanticKernel.ChatCompletion;
6+
7+
namespace GettingStarted.AzureAgents;
8+
9+
/// <summary>
10+
/// Demonstrate parsing JSON response.
11+
/// </summary>
12+
public class Step10_JsonResponse(ITestOutputHelper output) : BaseAzureAgentTest(output)
13+
{
14+
private const string TutorInstructions =
15+
"""
16+
Think step-by-step and rate the user input on creativity and expressiveness from 1-100.
17+
18+
Respond in JSON format with the following JSON schema:
19+
20+
{
21+
"score": "integer (1-100)",
22+
"notes": "the reason for your score"
23+
}
24+
""";
25+
26+
[Fact]
27+
public async Task UseJsonObjectResponse()
28+
{
29+
PersistentAgent definition =
30+
await this.Client.Administration.CreateAgentAsync(
31+
TestConfiguration.AzureAI.ChatModelId,
32+
instructions: TutorInstructions,
33+
responseFormat:
34+
BinaryData.FromString(
35+
"""
36+
{
37+
"type": "json_object"
38+
}
39+
"""));
40+
41+
AzureAIAgent agent = new(definition, this.Client);
42+
43+
await ExecuteAgent(agent);
44+
}
45+
46+
[Fact]
47+
public async Task UseJsonSchemaResponse()
48+
{
49+
PersistentAgent definition =
50+
await this.Client.Administration.CreateAgentAsync(
51+
TestConfiguration.AzureAI.ChatModelId,
52+
instructions: TutorInstructions,
53+
responseFormat: BinaryData.FromString(
54+
"""
55+
{
56+
"type": "json_schema",
57+
"json_schema":
58+
{
59+
"type": "object",
60+
"name": "scoring",
61+
"schema": {
62+
"type": "object",
63+
"properties": {
64+
"score": {
65+
"type": "number"
66+
},
67+
"notes": {
68+
"type": "string"
69+
}
70+
},
71+
"required": [
72+
"score",
73+
"notes"
74+
],
75+
"additionalProperties": false
76+
},
77+
"strict": true
78+
}
79+
}
80+
"""));
81+
82+
AzureAIAgent agent = new(definition, this.Client);
83+
84+
await ExecuteAgent(agent);
85+
}
86+
87+
private async Task ExecuteAgent(AzureAIAgent agent)
88+
{
89+
AzureAIAgentThread thread = new(agent.Client);
90+
91+
await InvokeAgentAsync("The sunset is very colorful.");
92+
await InvokeAgentAsync("The sunset is setting over the mountains.");
93+
await InvokeAgentAsync("The sunset is setting over the mountains and filled the sky with a deep red flame, setting the clouds ablaze.");
94+
95+
// Local function to invoke agent and display the conversation messages.
96+
async Task InvokeAgentAsync(string input)
97+
{
98+
ChatMessageContent message = new(AuthorRole.User, input);
99+
this.WriteAgentChatMessage(message);
100+
101+
await foreach (ChatMessageContent response in agent.InvokeAsync(message, thread))
102+
{
103+
this.WriteAgentChatMessage(response);
104+
}
105+
}
106+
}
107+
}

‎dotnet/samples/GettingStartedWithAgents/GettingStartedWithAgents.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
</PropertyGroup>
1616

1717
<ItemGroup>
18+
<PackageReference Include="Azure.AI.Projects" />
1819
<PackageReference Include="Azure.Identity" />
1920
<PackageReference Include="Azure.Monitor.OpenTelemetry.Exporter" />
2021
<PackageReference Include="Microsoft.Extensions.AI.OpenAI" />

0 commit comments

Comments
 (0)