Test/realtime chat

Directory.Packages.props

@@ -21,6 +21,7 @@
     <PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageVersion Include="SharpHook" Version="5.3.9" />
     <PackageVersion Include="SixLabors.ImageSharp" Version="3.1.7" />
+    <PackageVersion Include="System.ClientModel" Version="1.3.0" />
     <PackageVersion Include="System.ComponentModel.Annotations" Version="5.0.0" />
     <PackageVersion Include="System.IdentityModel.Tokens.Jwt" Version="8.0.0" />
     <PackageVersion Include="System.Memory.Data" Version="8.0.0" />

src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentField.cs

@@ -16,6 +16,7 @@ public enum AgentField
     Instruction,
     Function,
     Template,
+    Link,
     Response,
     Sample,
     LlmConfig,

src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs

@@ -46,6 +46,12 @@ public class Agent
     [JsonIgnore]
     public List<AgentTemplate> Templates { get; set; } = new();
 
+    /// <summary>
+    /// Links that can be filled into parent prompt
+    /// </summary>
+    [JsonIgnore]
+    public List<AgentLink> Links { get; set; } = new();
+
     /// <summary>
     /// Agent tasks
     /// </summary>
@@ -168,6 +174,8 @@ public static Agent Clone(Agent agent)
             Functions = agent.Functions,
             Responses = agent.Responses,
             Samples = agent.Samples,
+            Templates = agent.Templates,
+            Links = agent.Links,
             Utilities = agent.Utilities,
             McpTools = agent.McpTools,
             Knowledges = agent.Knowledges,
@@ -204,6 +212,12 @@ public Agent SetTemplates(List<AgentTemplate> templates)
         return this;
     }
 
+    public Agent SetLinks(List<AgentLink> links)
+    {
+        Links = links ?? [];
+        return this;
+    }
+
     public Agent SetTasks(List<AgentTask> tasks)
     {
         Tasks = tasks ?? [];

src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentLink.cs

@@ -0,0 +1,17 @@
+namespace BotSharp.Abstraction.Agents.Models;
+
+public class AgentLink : AgentPromptBase
+{
+    public AgentLink() : base()
+    {
+    }
+
+    public AgentLink(string name, string content) : base(name, content)
+    {
+    }
+
+    public override string ToString()
+    {
+        return base.ToString();
+    }
+}

src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentPromptBase.cs

@@ -0,0 +1,23 @@
+namespace BotSharp.Abstraction.Agents.Models;
+
+public class AgentPromptBase
+{
+    public string Name { get; set; }
+    public string Content { get; set; }
+
+    public AgentPromptBase()
+    {
+
+    }
+
+    public AgentPromptBase(string name, string content)
+    {
+        Name = name;
+        Content = content;
+    }
+
+    public override string ToString()
+    {
+        return Name;
+    }
+}

src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentTemplate.cs

@@ -1,23 +1,17 @@
 namespace BotSharp.Abstraction.Agents.Models;
 
-public class AgentTemplate
+public class AgentTemplate : AgentPromptBase
 {
-    public string Name { get; set; }
-    public string Content { get; set; }
-
-    public AgentTemplate()
+    public AgentTemplate() : base()
     {
-
     }
 
-    public AgentTemplate(string name, string content)
+    public AgentTemplate(string name, string content) : base(name, content)
     {
-        Name = name;
-        Content = content;
     }
 
     public override string ToString()
     {
-        return Name;
+        return base.ToString();
     }
 }

src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs

@@ -8,7 +8,8 @@ public interface IRealTimeCompletion
     string Model { get; }
     void SetModelName(string model);
 
-    Task Connect(RealtimeHubConnection conn,
+    Task Connect(
+        RealtimeHubConnection conn,
         Action onModelReady,
         Action<string, string> onModelAudioDeltaReceived,
         Action onModelAudioResponseDone,

src/Infrastructure/BotSharp.Abstraction/Templating/ITemplateRender.cs

@@ -3,5 +3,22 @@ namespace BotSharp.Abstraction.Templating;
 public interface ITemplateRender
 {
     string Render(string template, Dictionary<string, object> dict);
-    void Register(Type type);
+
+    /// <summary>
+    /// Register tag
+    /// </summary>
+    /// <param name="tag"></param>
+    /// <param name="content">A dictionary whose key is identifier and value is its content to render</param>
+    /// <param name="data"></param>
+    /// <returns></returns>
+    bool RegisterTag(string tag, Dictionary<string, string> content, Dictionary<string, object>? data = null);
+
+    /// <summary>
+    /// Register tags
+    /// </summary>
+    /// <param name="tags">A dictionary whose key is tag and value is its identifier and content to render</param>
+    /// <param name="data"></param>
+    /// <returns></returns>
+    bool RegisterTags(Dictionary<string, List<AgentPromptBase>> tags, Dictionary<string, object>? data = null);
+    void RegisterType(Type type);
 }

src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj

@@ -12,6 +12,7 @@
 
   <ItemGroup>
     <PackageReference Include="NAudio" />
+    <PackageReference Include="System.ClientModel" />
   </ItemGroup>
 
   <ItemGroup>

src/Infrastructure/BotSharp.Core.Realtime/Models/Chat/ChatSessionUpdate.cs

@@ -0,0 +1,11 @@
+namespace BotSharp.Core.Realtime.Models.Chat;
+
+public class ChatSessionUpdate
+{
+    public string RawResponse { get; set; }
+
+    public ChatSessionUpdate()
+    {
+
+    }
+}

src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs

@@ -42,7 +42,8 @@ public async Task ConnectToModel(Func<string, Task>? responseToUser = null, Func
 
         _completer = _services.GetServices<IRealTimeCompletion>().First(x => x.Provider == settings.Provider);
 
-        await _completer.Connect(_conn, 
+        await _completer.Connect(
+            conn: _conn, 
             onModelReady: async () => 
             {
                 // Not TriggerModelInference, waiting for user utter.

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AiWebsocketPipelineResponse.cs → src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/AiWebsocketPipelineResponse.cs

@@ -1,12 +1,10 @@
 using System.ClientModel.Primitives;
 using System.Net;
-using System.Net.WebSockets;
 
-namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session;
+namespace BotSharp.Core.Realtime.Websocket.Chat;
 
 public class AiWebsocketPipelineResponse : PipelineResponse
 {
-
     public AiWebsocketPipelineResponse()
     {
 

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataCollectionResult.cs → src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/AsyncWebsocketDataCollectionResult.cs

@@ -1,7 +1,6 @@
 using System.ClientModel;
-using System.Net.WebSockets;
 
-namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session;
+namespace BotSharp.Core.Realtime.Websocket.Chat;
 
 public class AsyncWebsocketDataCollectionResult : AsyncCollectionResult<ClientResult>
 {

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataResultEnumerator.cs → src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/AsyncWebsocketDataResultEnumerator.cs

@@ -1,9 +1,7 @@
-using System;
 using System.Buffers;
 using System.ClientModel;
-using System.Net.WebSockets;
 
-namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session;
+namespace BotSharp.Core.Realtime.Websocket.Chat;
 
 public class AsyncWebsocketDataResultEnumerator : IAsyncEnumerator<ClientResult>
 {

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/RealtimeChatSession.cs → src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/RealtimeChatSession.cs

@@ -1,14 +1,13 @@
-using BotSharp.Plugin.OpenAI.Models.Realtime;
 using System.ClientModel;
-using System.Net.WebSockets;
 using System.Runtime.CompilerServices;
+using BotSharp.Core.Realtime.Models.Chat;
 
-namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session;
+namespace BotSharp.Core.Realtime.Websocket.Chat;
 
 public class RealtimeChatSession : IDisposable
 {
     private readonly IServiceProvider _services;
-    private readonly BotSharpOptions _options;
+    private readonly JsonSerializerOptions _jsonOptions;
 
     private ClientWebSocket _webSocket;
     private readonly object _singleReceiveLock = new();
@@ -17,26 +16,26 @@ public class RealtimeChatSession : IDisposable
 
     public RealtimeChatSession(
         IServiceProvider services,
-        BotSharpOptions options)
+        JsonSerializerOptions jsonOptions)
     {
         _services = services;
-        _options = options;
+        _jsonOptions = jsonOptions;
     }
 
-    public async Task ConnectAsync(string provider, string model, CancellationToken cancellationToken = default)
+    public async Task ConnectAsync(Uri uri, Dictionary<string, string> headers, CancellationToken cancellationToken = default)
     {
-        var settingsService = _services.GetRequiredService<ILlmProviderService>();
-        var settings = settingsService.GetSetting(provider, model);
-
         _webSocket?.Dispose();
         _webSocket = new ClientWebSocket();
-        _webSocket.Options.SetRequestHeader("Authorization", $"Bearer {settings.ApiKey}");
-        _webSocket.Options.SetRequestHeader("OpenAI-Beta", "realtime=v1");
 
-        await _webSocket.ConnectAsync(new Uri($"wss://api.openai.com/v1/realtime?model={model}"), cancellationToken);
+        foreach (var header in headers)
+        {
+            _webSocket.Options.SetRequestHeader(header.Key, header.Value);
+        }
+
+        await _webSocket.ConnectAsync(uri, cancellationToken);
     }
 
-    public async IAsyncEnumerable<SessionConversationUpdate> ReceiveUpdatesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default)
+    public async IAsyncEnumerable<ChatSessionUpdate> ReceiveUpdatesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default)
     {
         await foreach (ClientResult result in ReceiveInnerUpdatesAsync(cancellationToken))
         {
@@ -58,12 +57,12 @@ public async IAsyncEnumerable<ClientResult> ReceiveInnerUpdatesAsync([Enumerator
         }
     }
 
-    private SessionConversationUpdate HandleSessionResult(ClientResult result)
+    private ChatSessionUpdate HandleSessionResult(ClientResult result)
     {
         using var response = result.GetRawResponse();
         var bytes = response.Content.ToArray();
         var text = Encoding.UTF8.GetString(bytes, 0, bytes.Length);
-        return new SessionConversationUpdate
+        return new ChatSessionUpdate
         {
             RawResponse = text
         };
@@ -82,7 +81,7 @@ public async Task SendEventToModel(object message)
         {
             if (message is not string data)
             {
-                data = JsonSerializer.Serialize(message, _options.JsonSerializerOptions);
+                data = JsonSerializer.Serialize(message, _jsonOptions);
             }
 
             var buffer = Encoding.UTF8.GetBytes(data);

src/Infrastructure/BotSharp.Core/Agents/AgentPlugin.cs

@@ -45,7 +45,7 @@ public void RegisterDI(IServiceCollection services, IConfiguration config)
         {
             var settingService = provider.GetRequiredService<ISettingService>();
             var render = provider.GetRequiredService<ITemplateRender>();
-            render.Register(typeof(AgentSettings));
+            render.RegisterType(typeof(AgentSettings));
             return settingService.Bind<AgentSettings>("Agent");
         });
     }

src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs

@@ -111,6 +111,26 @@ private List<AgentTemplate> GetTemplatesFromFile(string fileDir)
         return templates;
     }
 
+    private List<AgentLink> GetLinksFromFile(string fileDir)
+    {
+        var links = new List<AgentLink>();
+        var linkDir = Path.Combine(fileDir, "links");
+        if (!Directory.Exists(linkDir)) return links;
+
+        foreach (var file in Directory.GetFiles(linkDir))
+        {
+            var extension = Path.GetExtension(file).Substring(1);
+            if (extension.IsEqualTo(_agentSettings.TemplateFormat))
+            {
+                var name = Path.GetFileNameWithoutExtension(file);
+                var content = File.ReadAllText(file);
+                links.Add(new AgentLink(name, content));
+            }
+        }
+
+        return links;
+    }
+
     private List<FunctionDef> GetFunctionsFromFile(string fileDir)
     {
         var functions = new List<FunctionDef>();

src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs

@@ -52,10 +52,12 @@ public async Task<string> RefreshAgents()
                 var functions = GetFunctionsFromFile(dir);
                 var responses = GetResponsesFromFile(dir);
                 var templates = GetTemplatesFromFile(dir);
+                var links = GetLinksFromFile(dir);
                 var samples = GetSamplesFromFile(dir);
                 agent.SetInstruction(defaultInstruction)
                      .SetChannelInstructions(channelInstructions)
                      .SetTemplates(templates)
+                     .SetLinks(links)
                      .SetFunctions(functions)
                      .SetResponses(responses)
                      .SetSamples(samples);

src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs

@@ -22,6 +22,7 @@ public string RenderedInstruction(Agent agent)
             agent.TemplateDict[t.Key] = t.Value;
         }
 
+        RenderAgentLinks(agent, agent.TemplateDict);
         var res = render.Render(string.Join("\r\n", instructions), agent.TemplateDict);
         return res;
     }
@@ -136,4 +137,26 @@ await hook.OnRenderingTemplate(agent, templateName, content)
 
         return content;
     }
+
+    private void RenderAgentLinks(Agent agent, Dictionary<string, object> dict)
+    {
+        var render = _services.GetRequiredService<ITemplateRender>();
+
+        var links = new Dictionary<string, string>();
+        agent.Links ??= [];
+
+        foreach (var link in agent.Links)
+        {
+            if (string.IsNullOrWhiteSpace(link.Name)
+                || string.IsNullOrWhiteSpace(link.Content))
+            {
+                continue;
+            }
+
+            links[link.Name] = link.Content;
+        }
+
+        render.RegisterTag("link", links, dict);
+        return;
+    }
 }