diff --git a/Directory.Packages.props b/Directory.Packages.props index 7cdfc1a62..f806e0ee4 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -21,6 +21,7 @@ + diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentField.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentField.cs index 0cab0dfb8..ebdffe39a 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentField.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentField.cs @@ -16,6 +16,7 @@ public enum AgentField Instruction, Function, Template, + Link, Response, Sample, LlmConfig, diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs index 81cb7a3be..e43892fa1 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs @@ -46,6 +46,12 @@ public class Agent [JsonIgnore] public List Templates { get; set; } = new(); + /// + /// Links that can be filled into parent prompt + /// + [JsonIgnore] + public List Links { get; set; } = new(); + /// /// Agent tasks /// @@ -168,6 +174,8 @@ public static Agent Clone(Agent agent) Functions = agent.Functions, Responses = agent.Responses, Samples = agent.Samples, + Templates = agent.Templates, + Links = agent.Links, Utilities = agent.Utilities, McpTools = agent.McpTools, Knowledges = agent.Knowledges, @@ -204,6 +212,12 @@ public Agent SetTemplates(List templates) return this; } + public Agent SetLinks(List links) + { + Links = links ?? []; + return this; + } + public Agent SetTasks(List tasks) { Tasks = tasks ?? []; diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentLink.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentLink.cs new file mode 100644 index 000000000..c1bdbfae6 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentLink.cs @@ -0,0 +1,17 @@ +namespace BotSharp.Abstraction.Agents.Models; + +public class AgentLink : AgentPromptBase +{ + public AgentLink() : base() + { + } + + public AgentLink(string name, string content) : base(name, content) + { + } + + public override string ToString() + { + return base.ToString(); + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentPromptBase.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentPromptBase.cs new file mode 100644 index 000000000..c199a773d --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentPromptBase.cs @@ -0,0 +1,23 @@ +namespace BotSharp.Abstraction.Agents.Models; + +public class AgentPromptBase +{ + public string Name { get; set; } + public string Content { get; set; } + + public AgentPromptBase() + { + + } + + public AgentPromptBase(string name, string content) + { + Name = name; + Content = content; + } + + public override string ToString() + { + return Name; + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentTemplate.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentTemplate.cs index 9591934ff..f9225baae 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentTemplate.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentTemplate.cs @@ -1,23 +1,17 @@ namespace BotSharp.Abstraction.Agents.Models; -public class AgentTemplate +public class AgentTemplate : AgentPromptBase { - public string Name { get; set; } - public string Content { get; set; } - - public AgentTemplate() + public AgentTemplate() : base() { - } - public AgentTemplate(string name, string content) + public AgentTemplate(string name, string content) : base(name, content) { - Name = name; - Content = content; } public override string ToString() { - return Name; + return base.ToString(); } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Loggers/IContentGeneratingHook.cs b/src/Infrastructure/BotSharp.Abstraction/Loggers/IContentGeneratingHook.cs index c4f634508..03b69f5f2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Loggers/IContentGeneratingHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Loggers/IContentGeneratingHook.cs @@ -47,5 +47,5 @@ public interface IContentGeneratingHook /// /// /// - Task OnSessionUpdated(Agent agent, string instruction, FunctionDef[] functions) => Task.CompletedTask; + Task OnSessionUpdated(Agent agent, string instruction, FunctionDef[] functions, bool isInit = false) => Task.CompletedTask; } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index 396ccf02b..11174356e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -8,7 +8,8 @@ public interface IRealTimeCompletion string Model { get; } void SetModelName(string model); - Task Connect(RealtimeHubConnection conn, + Task Connect( + RealtimeHubConnection conn, Action onModelReady, Action onModelAudioDeltaReceived, Action onModelAudioResponseDone, @@ -23,7 +24,7 @@ Task Connect(RealtimeHubConnection conn, Task SendEventToModel(object message); Task Disconnect(); - Task UpdateSession(RealtimeHubConnection conn); + Task UpdateSession(RealtimeHubConnection conn, bool isInit = false); Task InsertConversationItem(RoleDialogModel message); Task RemoveConversationItem(string itemId); Task TriggerModelInference(string? instructions = null); diff --git a/src/Infrastructure/BotSharp.Abstraction/Templating/ITemplateRender.cs b/src/Infrastructure/BotSharp.Abstraction/Templating/ITemplateRender.cs index 82eff1c13..77e942ddc 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Templating/ITemplateRender.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Templating/ITemplateRender.cs @@ -3,5 +3,22 @@ namespace BotSharp.Abstraction.Templating; public interface ITemplateRender { string Render(string template, Dictionary dict); - void Register(Type type); + + /// + /// Register tag + /// + /// + /// A dictionary whose key is identifier and value is its content to render + /// + /// + bool RegisterTag(string tag, Dictionary content, Dictionary? data = null); + + /// + /// Register tags + /// + /// A dictionary whose key is tag and value is its identifier and content to render + /// + /// + bool RegisterTags(Dictionary> tags, Dictionary? data = null); + void RegisterType(Type type); } diff --git a/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj b/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj index 6d69b0d7c..a0500769a 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj +++ b/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj @@ -12,6 +12,7 @@ + diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Models/Chat/ChatSessionUpdate.cs b/src/Infrastructure/BotSharp.Core.Realtime/Models/Chat/ChatSessionUpdate.cs new file mode 100644 index 000000000..9fd172560 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core.Realtime/Models/Chat/ChatSessionUpdate.cs @@ -0,0 +1,11 @@ +namespace BotSharp.Core.Realtime.Models.Chat; + +public class ChatSessionUpdate +{ + public string RawResponse { get; set; } + + public ChatSessionUpdate() + { + + } +} diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 38a1a7834..b81fe0f55 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -42,11 +42,12 @@ public async Task ConnectToModel(Func? responseToUser = null, Func _completer = _services.GetServices().First(x => x.Provider == settings.Provider); - await _completer.Connect(_conn, + await _completer.Connect( + conn: _conn, onModelReady: async () => { // Not TriggerModelInference, waiting for user utter. - var instruction = await _completer.UpdateSession(_conn); + var instruction = await _completer.UpdateSession(_conn, isInit: true); var data = _conn.OnModelReady(); await (init?.Invoke(data) ?? Task.CompletedTask); await HookEmitter.Emit(_services, async hook => await hook.OnModelReady(agent, _completer)); diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AiWebsocketPipelineResponse.cs b/src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/AiWebsocketPipelineResponse.cs similarity index 97% rename from src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AiWebsocketPipelineResponse.cs rename to src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/AiWebsocketPipelineResponse.cs index 2a62cdc91..fde66e4fa 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AiWebsocketPipelineResponse.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/AiWebsocketPipelineResponse.cs @@ -1,12 +1,10 @@ using System.ClientModel.Primitives; using System.Net; -using System.Net.WebSockets; -namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session; +namespace BotSharp.Core.Realtime.Websocket.Chat; public class AiWebsocketPipelineResponse : PipelineResponse { - public AiWebsocketPipelineResponse() { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataCollectionResult.cs b/src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/AsyncWebsocketDataCollectionResult.cs similarity index 92% rename from src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataCollectionResult.cs rename to src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/AsyncWebsocketDataCollectionResult.cs index 38b46c905..946f3990c 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataCollectionResult.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/AsyncWebsocketDataCollectionResult.cs @@ -1,7 +1,6 @@ using System.ClientModel; -using System.Net.WebSockets; -namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session; +namespace BotSharp.Core.Realtime.Websocket.Chat; public class AsyncWebsocketDataCollectionResult : AsyncCollectionResult { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataResultEnumerator.cs b/src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/AsyncWebsocketDataResultEnumerator.cs similarity index 93% rename from src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataResultEnumerator.cs rename to src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/AsyncWebsocketDataResultEnumerator.cs index d2950fe17..98492c484 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataResultEnumerator.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/AsyncWebsocketDataResultEnumerator.cs @@ -1,9 +1,7 @@ -using System; using System.Buffers; using System.ClientModel; -using System.Net.WebSockets; -namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session; +namespace BotSharp.Core.Realtime.Websocket.Chat; public class AsyncWebsocketDataResultEnumerator : IAsyncEnumerator { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/RealtimeChatSession.cs b/src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/RealtimeChatSession.cs similarity index 67% rename from src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/RealtimeChatSession.cs rename to src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/RealtimeChatSession.cs index 827b1c982..4b1d79ef3 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/RealtimeChatSession.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/RealtimeChatSession.cs @@ -1,14 +1,13 @@ -using BotSharp.Plugin.OpenAI.Models.Realtime; using System.ClientModel; -using System.Net.WebSockets; using System.Runtime.CompilerServices; +using BotSharp.Core.Realtime.Models.Chat; -namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session; +namespace BotSharp.Core.Realtime.Websocket.Chat; public class RealtimeChatSession : IDisposable { private readonly IServiceProvider _services; - private readonly BotSharpOptions _options; + private readonly JsonSerializerOptions _jsonOptions; private ClientWebSocket _webSocket; private readonly object _singleReceiveLock = new(); @@ -17,26 +16,26 @@ public class RealtimeChatSession : IDisposable public RealtimeChatSession( IServiceProvider services, - BotSharpOptions options) + JsonSerializerOptions jsonOptions) { _services = services; - _options = options; + _jsonOptions = jsonOptions; } - public async Task ConnectAsync(string provider, string model, CancellationToken cancellationToken = default) + public async Task ConnectAsync(Uri uri, Dictionary headers, CancellationToken cancellationToken = default) { - var settingsService = _services.GetRequiredService(); - var settings = settingsService.GetSetting(provider, model); - _webSocket?.Dispose(); _webSocket = new ClientWebSocket(); - _webSocket.Options.SetRequestHeader("Authorization", $"Bearer {settings.ApiKey}"); - _webSocket.Options.SetRequestHeader("OpenAI-Beta", "realtime=v1"); - await _webSocket.ConnectAsync(new Uri($"wss://api.openai.com/v1/realtime?model={model}"), cancellationToken); + foreach (var header in headers) + { + _webSocket.Options.SetRequestHeader(header.Key, header.Value); + } + + await _webSocket.ConnectAsync(uri, cancellationToken); } - public async IAsyncEnumerable ReceiveUpdatesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable ReceiveUpdatesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) { await foreach (ClientResult result in ReceiveInnerUpdatesAsync(cancellationToken)) { @@ -58,12 +57,12 @@ public async IAsyncEnumerable ReceiveInnerUpdatesAsync([Enumerator } } - private SessionConversationUpdate HandleSessionResult(ClientResult result) + private ChatSessionUpdate HandleSessionResult(ClientResult result) { using var response = result.GetRawResponse(); var bytes = response.Content.ToArray(); var text = Encoding.UTF8.GetString(bytes, 0, bytes.Length); - return new SessionConversationUpdate + return new ChatSessionUpdate { RawResponse = text }; @@ -82,7 +81,7 @@ public async Task SendEventToModel(object message) { if (message is not string data) { - data = JsonSerializer.Serialize(message, _options.JsonSerializerOptions); + data = JsonSerializer.Serialize(message, _jsonOptions); } var buffer = Encoding.UTF8.GetBytes(data); diff --git a/src/Infrastructure/BotSharp.Core/Agents/AgentPlugin.cs b/src/Infrastructure/BotSharp.Core/Agents/AgentPlugin.cs index e6998fe4f..41e13da4c 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/AgentPlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/AgentPlugin.cs @@ -45,7 +45,7 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) { var settingService = provider.GetRequiredService(); var render = provider.GetRequiredService(); - render.Register(typeof(AgentSettings)); + render.RegisterType(typeof(AgentSettings)); return settingService.Bind("Agent"); }); } diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs index 5becbec63..9984c2228 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs @@ -111,6 +111,26 @@ private List GetTemplatesFromFile(string fileDir) return templates; } + private List GetLinksFromFile(string fileDir) + { + var links = new List(); + var linkDir = Path.Combine(fileDir, "links"); + if (!Directory.Exists(linkDir)) return links; + + foreach (var file in Directory.GetFiles(linkDir)) + { + var extension = Path.GetExtension(file).Substring(1); + if (extension.IsEqualTo(_agentSettings.TemplateFormat)) + { + var name = Path.GetFileNameWithoutExtension(file); + var content = File.ReadAllText(file); + links.Add(new AgentLink(name, content)); + } + } + + return links; + } + private List GetFunctionsFromFile(string fileDir) { var functions = new List(); diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs index c9b3c90cb..b659083f9 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs @@ -52,10 +52,12 @@ public async Task RefreshAgents() var functions = GetFunctionsFromFile(dir); var responses = GetResponsesFromFile(dir); var templates = GetTemplatesFromFile(dir); + var links = GetLinksFromFile(dir); var samples = GetSamplesFromFile(dir); agent.SetInstruction(defaultInstruction) .SetChannelInstructions(channelInstructions) .SetTemplates(templates) + .SetLinks(links) .SetFunctions(functions) .SetResponses(responses) .SetSamples(samples); diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs index 288a7ca17..6e63c86b4 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs @@ -22,6 +22,7 @@ public string RenderedInstruction(Agent agent) agent.TemplateDict[t.Key] = t.Value; } + RenderAgentLinks(agent, agent.TemplateDict); var res = render.Render(string.Join("\r\n", instructions), agent.TemplateDict); return res; } @@ -136,4 +137,26 @@ await hook.OnRenderingTemplate(agent, templateName, content) return content; } + + private void RenderAgentLinks(Agent agent, Dictionary dict) + { + var render = _services.GetRequiredService(); + + var links = new Dictionary(); + agent.Links ??= []; + + foreach (var link in agent.Links) + { + if (string.IsNullOrWhiteSpace(link.Name) + || string.IsNullOrWhiteSpace(link.Content)) + { + continue; + } + + links[link.Name] = link.Content; + } + + render.RegisterTag("link", links, dict); + return; + } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs index 29caa9728..9b15e7772 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs @@ -38,6 +38,7 @@ public async Task UpdateAgent(Agent agent, AgentField updateField) record.ChannelInstructions = agent.ChannelInstructions ?? []; record.Functions = agent.Functions ?? []; record.Templates = agent.Templates ?? []; + record.Links = agent.Links ?? []; record.Responses = agent.Responses ?? []; record.Samples = agent.Samples ?? []; record.Utilities = agent.Utilities ?? []; @@ -105,6 +106,7 @@ public async Task UpdateAgentFromFile(string id) .SetInstruction(foundAgent.Instruction) .SetChannelInstructions(foundAgent.ChannelInstructions) .SetTemplates(foundAgent.Templates) + .SetLinks(foundAgent.Links) .SetFunctions(foundAgent.Functions) .SetResponses(foundAgent.Responses) .SetSamples(foundAgent.Samples) @@ -196,10 +198,12 @@ public async Task PatchAgentTemplate(Agent agent) var functions = GetFunctionsFromFile(dir); var responses = GetResponsesFromFile(dir); var templates = GetTemplatesFromFile(dir); + var links = GetLinksFromFile(dir); var samples = GetSamplesFromFile(dir); return agent.SetInstruction(defaultInstruction) .SetChannelInstructions(channelInstructions) - .SetTemplates(templates) + .SetTemplates(templates) + .SetLinks(links) .SetFunctions(functions) .SetResponses(responses) .SetSamples(samples); diff --git a/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs b/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs index 09936dec8..13ee1de62 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs @@ -31,7 +31,7 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) { var settingService = provider.GetRequiredService(); var render = provider.GetRequiredService(); - render.Register(typeof(ConversationSetting)); + render.RegisterType(typeof(ConversationSetting)); return settingService.Bind("Conversation"); }); diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs index 930eb13dd..5219a0b05 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs @@ -51,6 +51,9 @@ public void UpdateAgent(Agent agent, AgentField field) case AgentField.Template: UpdateAgentTemplates(agent.Id, agent.Templates); break; + case AgentField.Link: + UpdateAgentLinks(agent.Id, agent.Links); + break; case AgentField.Response: UpdateAgentResponses(agent.Id, agent.Responses); break; @@ -325,6 +328,23 @@ private void UpdateAgentTemplates(string agentId, List templates) } } + private void UpdateAgentLinks(string agentId, List links) + { + if (links == null) return; + + var (agent, agentFile) = GetAgentFromFile(agentId); + if (agent == null) return; + + var linkDir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId, AGENT_LINKS_FOLDER); + DeleteBeforeCreateDirectory(linkDir); + + foreach (var link in links) + { + var file = Path.Combine(linkDir, $"{link.Name}.{_agentSettings.TemplateFormat}"); + File.WriteAllText(file, link.Content); + } + } + private void UpdateAgentResponses(string agentId, List responses) { if (responses == null) return; @@ -404,6 +424,7 @@ private void UpdateAgentAllFields(Agent inputAgent) UpdateAgentInstructions(inputAgent.Id, inputAgent.Instruction, inputAgent.ChannelInstructions); UpdateAgentResponses(inputAgent.Id, inputAgent.Responses); UpdateAgentTemplates(inputAgent.Id, inputAgent.Templates); + UpdateAgentLinks(inputAgent.Id, inputAgent.Links); UpdateAgentFunctions(inputAgent.Id, inputAgent.Functions); UpdateAgentSamples(inputAgent.Id, inputAgent.Samples); } @@ -447,11 +468,13 @@ public List GetAgentResponses(string agentId, string prefix, string inte var functions = FetchFunctions(dir); var samples = FetchSamples(dir); var templates = FetchTemplates(dir); + var links = FetchLinks(dir); var responses = FetchResponses(dir); return record.SetInstruction(defaultInstruction) .SetChannelInstructions(channelInstructions) .SetFunctions(functions) .SetTemplates(templates) + .SetLinks(links) .SetSamples(samples) .SetResponses(responses); } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs index f26e8bc04..eecf9b453 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs @@ -24,6 +24,7 @@ public partial class FileRepository : IBotSharpRepository private const string AGENT_INSTRUCTIONS_FOLDER = "instructions"; private const string AGENT_FUNCTIONS_FOLDER = "functions"; private const string AGENT_TEMPLATES_FOLDER = "templates"; + private const string AGENT_LINKS_FOLDER = "links"; private const string AGENT_RESPONSES_FOLDER = "responses"; private const string AGENT_TASKS_FOLDER = "tasks"; private const string AGENT_TASK_PREFIX = "#metadata"; @@ -228,6 +229,7 @@ private IQueryable Agents .SetChannelInstructions(channelInstructions) .SetFunctions(FetchFunctions(d)) .SetTemplates(FetchTemplates(d)) + .SetLinks(FetchLinks(d)) .SetResponses(FetchResponses(d)) .SetSamples(FetchSamples(d)); _agents.Add(agent); @@ -386,7 +388,7 @@ private List FetchTemplates(string fileDir) foreach (var file in Directory.GetFiles(templateDir)) { - var fileName = file.Split(Path.DirectorySeparatorChar).Last(); + var fileName = Path.GetFileName(file); var splitIdx = fileName.LastIndexOf("."); var name = fileName.Substring(0, splitIdx); var extension = fileName.Substring(splitIdx + 1); @@ -400,6 +402,29 @@ private List FetchTemplates(string fileDir) return templates; } + private List FetchLinks(string fileDir) + { + var links = new List(); + var linkDir = Path.Combine(fileDir, AGENT_LINKS_FOLDER); + + if (!Directory.Exists(linkDir)) return links; + + foreach (var file in Directory.GetFiles(linkDir)) + { + var fileName = Path.GetFileName(file); + var splitIdx = fileName.LastIndexOf("."); + var name = fileName.Substring(0, splitIdx); + var extension = fileName.Substring(splitIdx + 1); + if (extension.Equals(_agentSettings.TemplateFormat, StringComparison.OrdinalIgnoreCase)) + { + var content = File.ReadAllText(file); + links.Add(new AgentLink(name, content)); + } + + } + return links; + } + private List FetchTasks(string fileDir) { var tasks = new List(); diff --git a/src/Infrastructure/BotSharp.Core/Templating/TemplateRender.cs b/src/Infrastructure/BotSharp.Core/Templating/TemplateRender.cs index c16a8c4ba..ace11c411 100644 --- a/src/Infrastructure/BotSharp.Core/Templating/TemplateRender.cs +++ b/src/Infrastructure/BotSharp.Core/Templating/TemplateRender.cs @@ -3,6 +3,7 @@ using BotSharp.Abstraction.Templating; using BotSharp.Abstraction.Translation.Models; using Fluid; +using Fluid.Ast; using System.Collections; using System.Reflection; @@ -40,17 +41,71 @@ public string Render(string template, Dictionary dict) { var context = new TemplateContext(dict, _options); template = t.Render(context); - return template; } else { _logger.LogWarning(error); - return template; } + + return template; } + public bool RegisterTag(string tag, Dictionary content, Dictionary? data = null) + { + _parser.RegisterIdentifierTag(tag, (identifier, writer, encoder, context) => + { + if (content?.TryGetValue(identifier, out var value) == true) + { + var str = Render(value, data ?? []); + writer.Write(str); + } + else + { + writer.Write(string.Empty); + } + return Statement.Normal(); + }); + + return true; + } + + public bool RegisterTags(Dictionary> tags, Dictionary? data = null) + { + if (tags.IsNullOrEmpty()) return false; + + foreach (var item in tags) + { + var tag = item.Key; + if (string.IsNullOrWhiteSpace(tag) + || item.Value.IsNullOrEmpty()) + { + continue; + } + + foreach (var prompt in item.Value) + { + _parser.RegisterIdentifierTag(tag, (identifier, writer, encoder, context) => + { + var found = item.Value.FirstOrDefault(x => x.Name.IsEqualTo(identifier)); + if (found != null) + { + var str = Render(found.Content, data ?? []); + writer.Write(str); + } + else + { + writer.Write(string.Empty); + } + + return Statement.Normal(); + }); + } + } + + return true; + } - public void Register(Type type) + public void RegisterType(Type type) { if (type == null || IsStringType(type)) return; @@ -59,7 +114,7 @@ public void Register(Type type) if (type.IsGenericType) { var genericType = type.GetGenericArguments()[0]; - Register(genericType); + RegisterType(genericType); } } else if (IsTrackToNextLevel(type)) @@ -68,7 +123,7 @@ public void Register(Type type) var props = type.GetProperties(); foreach (var prop in props) { - Register(prop.PropertyType); + RegisterType(prop.PropertyType); } } } diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/instructions/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/instructions/instruction.liquid index d944594b0..7401fe352 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/instructions/instruction.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/instructions/instruction.liquid @@ -1 +1 @@ -You are a AI Assistant. You can answer user's question. +You are a AI Assistant. You can answer user's question. \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index a76d197c6..ebbda722e 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -654,4 +654,4 @@ private JsonSerializerOptions InitJsonOptions(BotSharpOptions options) return jsonOption; } #endregion -} +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/AgentCreationModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/AgentCreationModel.cs index 8265ebe44..61aa89fd7 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/AgentCreationModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/AgentCreationModel.cs @@ -24,6 +24,8 @@ public class AgentCreationModel /// public List Templates { get; set; } = new(); + public List Links { get; set; } = new(); + /// /// LLM callable function definition /// @@ -70,6 +72,7 @@ public Agent ToAgent() Instruction = Instruction, ChannelInstructions = ChannelInstructions, Templates = Templates, + Links = Links, Functions = Functions, Responses = Responses, Samples = Samples, diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/AgentUpdateModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/AgentUpdateModel.cs index b11a9db0e..0e9b8ebb8 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/AgentUpdateModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/AgentUpdateModel.cs @@ -25,6 +25,11 @@ public class AgentUpdateModel /// public List? Templates { get; set; } + /// + /// Links + /// + public List? Links { get; set; } + /// /// Samples /// @@ -105,6 +110,7 @@ public Agent ToAgent() Instruction = Instruction ?? string.Empty, ChannelInstructions = ChannelInstructions ?? [], Templates = Templates ?? [], + Links = Links ?? [], Functions = Functions ?? [], Responses = Responses ?? [], Utilities = Utilities ?? [], diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/View/AgentViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/View/AgentViewModel.cs index 4fede545b..6fe468d05 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/View/AgentViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/View/AgentViewModel.cs @@ -18,6 +18,7 @@ public class AgentViewModel [JsonPropertyName("channel_instructions")] public List ChannelInstructions { get; set; } public List Templates { get; set; } + public List Links { get; set; } public List Functions { get; set; } public List Responses { get; set; } public List Samples { get; set; } @@ -87,6 +88,7 @@ public static AgentViewModel FromAgent(Agent agent) Instruction = agent.Instruction, ChannelInstructions = agent.ChannelInstructions ?? [], Templates = agent.Templates ?? [], + Links = agent.Links ?? [], Functions = agent.Functions ?? [], Responses = agent.Responses ?? [], Samples = agent.Samples ?? [], diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/WebSocketsMiddleware.cs b/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubMiddleware.cs similarity index 94% rename from src/Plugins/BotSharp.Plugin.ChatHub/WebSocketsMiddleware.cs rename to src/Plugins/BotSharp.Plugin.ChatHub/ChatHubMiddleware.cs index 47c646b7a..0c63a0927 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/WebSocketsMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubMiddleware.cs @@ -3,11 +3,11 @@ namespace BotSharp.Plugin.ChatHub; -public class WebSocketsMiddleware +public class ChatHubMiddleware { private readonly RequestDelegate _next; - public WebSocketsMiddleware(RequestDelegate next) + public ChatHubMiddleware(RequestDelegate next) { _next = next; } diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/ChatStreamMiddleware.cs b/src/Plugins/BotSharp.Plugin.ChatHub/ChatStreamMiddleware.cs new file mode 100644 index 000000000..90318ff29 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ChatHub/ChatStreamMiddleware.cs @@ -0,0 +1,159 @@ +using BotSharp.Abstraction.Realtime; +using BotSharp.Abstraction.Realtime.Models; +using Microsoft.AspNetCore.Http; +using System.Net.WebSockets; + +namespace BotSharp.Plugin.ChatHub; + +public class ChatStreamMiddleware +{ + private readonly RequestDelegate _next; + private readonly ILogger _logger; + + public ChatStreamMiddleware( + RequestDelegate next, + ILogger logger) + { + _next = next; + _logger = logger; + } + + public async Task Invoke(HttpContext httpContext) + { + var request = httpContext.Request; + + if (request.Path.StartsWithSegments("/chat/stream")) + { + if (httpContext.WebSockets.IsWebSocketRequest) + { + try + { + var services = httpContext.RequestServices; + var segments = request.Path.Value.Split("/"); + var agentId = segments[segments.Length - 2]; + var conversationId = segments[segments.Length - 1]; + + using var webSocket = await httpContext.WebSockets.AcceptWebSocketAsync(); + await HandleWebSocket(services, agentId, conversationId, webSocket); + } + catch (Exception ex) + { + _logger.LogError(ex, $"Error when connecting Chat stream. ({ex.Message})"); + } + return; + } + } + + await _next(httpContext); + } + + private async Task HandleWebSocket(IServiceProvider services, string agentId, string conversationId, WebSocket webSocket) + { + var hub = services.GetRequiredService(); + var conn = hub.SetHubConnection(conversationId); + conn.CurrentAgentId = agentId; + + // load conversation and state + var convService = services.GetRequiredService(); + convService.SetConversationId(conversationId, []); + await convService.GetConversationRecordOrCreateNew(agentId); + + var buffer = new byte[1024 * 32]; + WebSocketReceiveResult result; + + do + { + result = await webSocket.ReceiveAsync(new(buffer), CancellationToken.None); + + if (result.MessageType != WebSocketMessageType.Text) + { + continue; + } + + var receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count); + if (string.IsNullOrEmpty(receivedText)) + { + continue; + } + + var (eventType, data) = MapEvents(conn, receivedText); + + if (eventType == "start") + { + await ConnectToModel(hub, webSocket); + } + else if (eventType == "media") + { + if (!string.IsNullOrEmpty(data)) + { + await hub.Completer.AppenAudioBuffer(data); + } + } + else if (eventType == "disconnect") + { + await hub.Completer.Disconnect(); + } + } + while (!webSocket.CloseStatus.HasValue); + + await webSocket.CloseAsync(result.CloseStatus.Value, result.CloseStatusDescription, CancellationToken.None); + } + + private async Task ConnectToModel(IRealtimeHub hub, WebSocket webSocket) + { + await hub.ConnectToModel(async data => + { + await SendEventToUser(webSocket, data); + }); + } + + private async Task SendEventToUser(WebSocket webSocket, string message) + { + if (webSocket.State == WebSocketState.Open) + { + var buffer = Encoding.UTF8.GetBytes(message); + await webSocket.SendAsync(new ArraySegment(buffer), WebSocketMessageType.Text, true, CancellationToken.None); + } + } + + private (string, string) MapEvents(RealtimeHubConnection conn, string receivedText) + { + var response = JsonSerializer.Deserialize(receivedText); + string data = string.Empty; + + switch (response.Event) + { + case "start": + conn.ResetStreamState(); + break; + case "media": + var mediaResponse = JsonSerializer.Deserialize(receivedText); + data = mediaResponse?.Body?.Payload ?? string.Empty; + break; + case "disconnect": + break; + } + + conn.OnModelMessageReceived = message => + JsonSerializer.Serialize(new + { + @event = "media", + media = new { payload = message } + }); + + conn.OnModelAudioResponseDone = () => + JsonSerializer.Serialize(new + { + @event = "mark", + mark = new { name = "responsePart" } + }); + + conn.OnModelUserInterrupted = () => + JsonSerializer.Serialize(new + { + @event = "clear" + }); + + return (response.Event, data); + } +} diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs index 069b6fc21..c5d6b15c2 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs @@ -85,7 +85,7 @@ public override async Task OnPostbackMessageReceived(RoleDialogModel message, Po await SendContentLog(conversationId, input); } - public async Task OnSessionUpdated(Agent agent, string instruction, FunctionDef[] functions) + public async Task OnSessionUpdated(Agent agent, string instruction, FunctionDef[] functions, bool isInit = false) { var conversationId = _state.GetConversationId(); if (string.IsNullOrEmpty(conversationId)) return; @@ -98,6 +98,8 @@ public async Task OnSessionUpdated(Agent agent, string instruction, FunctionDef[ } _logger.LogInformation(log); + if (isInit) return; + var message = new RoleDialogModel(AgentRole.Assistant, log) { MessageId = _routingCtx.MessageId diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Models/Stream/ChatStreamEventResponse.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Models/Stream/ChatStreamEventResponse.cs new file mode 100644 index 000000000..aee742df7 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Models/Stream/ChatStreamEventResponse.cs @@ -0,0 +1,21 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.ChatHub.Models.Stream; + +internal class ChatStreamEventResponse +{ + [JsonPropertyName("event")] + public string Event { get; set; } +} + +internal class ChatStreamMediaEventResponse : ChatStreamEventResponse +{ + [JsonPropertyName("body")] + public MediaEventResponseBody Body { get; set; } +} + +internal class MediaEventResponseBody +{ + [JsonPropertyName("payload")] + public string Payload { get; set; } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Using.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Using.cs index bc50d257c..73240fbb8 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Using.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Using.cs @@ -33,4 +33,5 @@ global using BotSharp.Abstraction.Messaging.Models.RichContent; global using BotSharp.Abstraction.Templating; global using BotSharp.Plugin.ChatHub.Settings; -global using BotSharp.Plugin.ChatHub.Enums; \ No newline at end of file +global using BotSharp.Plugin.ChatHub.Enums; +global using BotSharp.Plugin.ChatHub.Models.Stream; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs index cdebc5f6d..98896ac3c 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -9,8 +9,9 @@ namespace BotSharp.Plugin.GoogleAi.Providers.Realtime; public class GoogleRealTimeProvider : IRealTimeCompletion { public string Provider => "google-ai"; - private string _model = GoogleAIModels.Gemini2FlashExp; public string Model => _model; + + private string _model = GoogleAIModels.Gemini2FlashExp; private MultiModalLiveClient _client; private GenerativeModel _chatClient; private readonly IServiceProvider _services; @@ -34,15 +35,16 @@ public void SetModelName(string model) _model = model; } - private Action onModelReady; - Action onModelAudioDeltaReceived; - private Action onModelAudioResponseDone; - Action onModelAudioTranscriptDone; - private Action> onModelResponseDone; - Action onConversationItemCreated; - private Action onInputAudioTranscriptionCompleted; - Action onUserInterrupted; - RealtimeHubConnection conn; + private RealtimeHubConnection _conn; + private Action _onModelReady; + private Action _onModelAudioDeltaReceived; + private Action _onModelAudioResponseDone; + private Action _onModelAudioTranscriptDone; + private Action> _onModelResponseDone; + private Action _onConversationItemCreated; + private Action _onInputAudioTranscriptionCompleted; + private Action _onUserInterrupted; + public async Task Connect(RealtimeHubConnection conn, Action onModelReady, @@ -54,15 +56,15 @@ public async Task Connect(RealtimeHubConnection conn, Action onInputAudioTranscriptionCompleted, Action onUserInterrupted) { - this.conn = conn; - this.onModelReady = onModelReady; - this.onModelAudioDeltaReceived = onModelAudioDeltaReceived; - this.onModelAudioResponseDone = onModelAudioResponseDone; - this.onModelAudioTranscriptDone = onModelAudioTranscriptDone; - this.onModelResponseDone = onModelResponseDone; - this.onConversationItemCreated = onConversationItemCreated; - this.onInputAudioTranscriptionCompleted = onInputAudioTranscriptionCompleted; - this.onUserInterrupted = onUserInterrupted; + _conn = conn; + _onModelReady = onModelReady; + _onModelAudioDeltaReceived = onModelAudioDeltaReceived; + _onModelAudioResponseDone = onModelAudioResponseDone; + _onModelAudioTranscriptDone = onModelAudioTranscriptDone; + _onModelResponseDone = onModelResponseDone; + _onConversationItemCreated = onConversationItemCreated; + _onInputAudioTranscriptionCompleted = onInputAudioTranscriptionCompleted; + _onUserInterrupted = onUserInterrupted; var realtimeModelSettings = _services.GetRequiredService(); _model = realtimeModelSettings.Model; @@ -120,7 +122,7 @@ private Task AttachEvents(MultiModalLiveClient client) client.Connected += (sender, e) => { _logger.LogInformation("Google Realtime Client connected."); - onModelReady(); + _onModelReady(); }; client.Disconnected += (sender, e) => @@ -133,39 +135,39 @@ private Task AttachEvents(MultiModalLiveClient client) _logger.LogInformation("User message received."); if (e.Payload.SetupComplete != null) { - onConversationItemCreated(_client.ConnectionId.ToString()); + _onConversationItemCreated(_client.ConnectionId.ToString()); } if (e.Payload.ServerContent != null) { if (e.Payload.ServerContent.TurnComplete == true) { - var responseDone = await ResponseDone(conn, e.Payload.ServerContent); - onModelResponseDone(responseDone); + var responseDone = await ResponseDone(_conn, e.Payload.ServerContent); + _onModelResponseDone(responseDone); } } }; client.AudioChunkReceived += (sender, e) => { - onModelAudioDeltaReceived(Convert.ToBase64String(e.Buffer), Guid.NewGuid().ToString()); + _onModelAudioDeltaReceived(Convert.ToBase64String(e.Buffer), Guid.NewGuid().ToString()); }; client.TextChunkReceived += (sender, e) => { - onInputAudioTranscriptionCompleted(new RoleDialogModel(AgentRole.Assistant, e.Text)); + _onInputAudioTranscriptionCompleted(new RoleDialogModel(AgentRole.Assistant, e.Text)); }; client.GenerationInterrupted += (sender, e) => { _logger.LogInformation("Audio generation interrupted."); - onUserInterrupted(); + _onUserInterrupted(); }; client.AudioReceiveCompleted += (sender, e) => { _logger.LogInformation("Audio receive completed."); - onModelAudioResponseDone(); + _onModelAudioResponseDone(); }; client.ErrorOccurred += (sender, e) => @@ -236,7 +238,7 @@ public async Task SendEventToModel(object message) //todo Send Audio Chunks to Model, Botsharp RealTime Implementation seems to be incomplete } - public async Task UpdateSession(RealtimeHubConnection conn) + public async Task UpdateSession(RealtimeHubConnection conn, bool isInit = false) { var convService = _services.GetRequiredService(); var conv = await convService.GetConversation(conn.ConversationId); @@ -276,7 +278,7 @@ public async Task UpdateSession(RealtimeHubConnection conn) }).ToArray(); await HookEmitter.Emit(_services, - async hook => { await hook.OnSessionUpdated(agent, prompt, functions); }); + async hook => { await hook.OnSessionUpdated(agent, prompt, functions, isInit); }); if (_settings.Gemini.UseGoogleSearch) { diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs index 7ebded25e..47e7d061a 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs @@ -15,6 +15,7 @@ public class AgentDocument : MongoBase public int? MaxMessageCount { get; set; } public List ChannelInstructions { get; set; } public List Templates { get; set; } + public List Links { get; set; } public List Functions { get; set; } public List Responses { get; set; } public List Samples { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLinkMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLinkMongoElement.cs new file mode 100644 index 000000000..f618e03a4 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLinkMongoElement.cs @@ -0,0 +1,28 @@ +using BotSharp.Abstraction.Agents.Models; + +namespace BotSharp.Plugin.MongoStorage.Models; + +[BsonIgnoreExtraElements(Inherited = true)] +public class AgentLinkMongoElement +{ + public string Name { get; set; } = default!; + public string Content { get; set; } = default!; + + public static AgentLinkMongoElement ToMongoElement(AgentLink link) + { + return new AgentLinkMongoElement + { + Name = link.Name, + Content = link.Content + }; + } + + public static AgentLink ToDomainElement(AgentLinkMongoElement mongoLink) + { + return new AgentLink + { + Name = mongoLink.Name, + Content = mongoLink.Content + }; + } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs index 6eda94eda..35cde7ee1 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs @@ -52,6 +52,9 @@ public void UpdateAgent(Agent agent, AgentField field) case AgentField.Template: UpdateAgentTemplates(agent.Id, agent.Templates); break; + case AgentField.Link: + UpdateAgentLinks(agent.Id, agent.Links); + break; case AgentField.Response: UpdateAgentResponses(agent.Id, agent.Responses); break; @@ -237,6 +240,19 @@ private void UpdateAgentTemplates(string agentId, List templates) _dc.Agents.UpdateOne(filter, update); } + private void UpdateAgentLinks(string agentId, List links) + { + if (links == null) return; + + var linksToUpdate = links.Select(t => AgentLinkMongoElement.ToMongoElement(t)).ToList(); + var filter = Builders.Filter.Eq(x => x.Id, agentId); + var update = Builders.Update + .Set(x => x.Links, linksToUpdate) + .Set(x => x.UpdatedTime, DateTime.UtcNow); + + _dc.Agents.UpdateOne(filter, update); + } + private void UpdateAgentResponses(string agentId, List responses) { if (responses == null || string.IsNullOrWhiteSpace(agentId)) return; @@ -356,6 +372,7 @@ private void UpdateAgentAllFields(Agent agent) .Set(x => x.Instruction, agent.Instruction) .Set(x => x.ChannelInstructions, agent.ChannelInstructions.Select(i => ChannelInstructionMongoElement.ToMongoElement(i)).ToList()) .Set(x => x.Templates, agent.Templates.Select(t => AgentTemplateMongoElement.ToMongoElement(t)).ToList()) + .Set(x => x.Links, agent.Links.Select(t => AgentLinkMongoElement.ToMongoElement(t)).ToList()) .Set(x => x.Functions, agent.Functions.Select(f => FunctionDefMongoElement.ToMongoElement(f)).ToList()) .Set(x => x.Responses, agent.Responses.Select(r => AgentResponseMongoElement.ToMongoElement(r)).ToList()) .Set(x => x.Samples, agent.Samples) @@ -538,6 +555,7 @@ public void BulkInsertAgents(List agents) LlmConfig = AgentLlmConfigMongoElement.ToMongoElement(x.LlmConfig), ChannelInstructions = x.ChannelInstructions?.Select(i => ChannelInstructionMongoElement.ToMongoElement(i))?.ToList() ?? [], Templates = x.Templates?.Select(t => AgentTemplateMongoElement.ToMongoElement(t))?.ToList() ?? [], + Links = x.Links?.Select(l => AgentLinkMongoElement.ToMongoElement(l))?.ToList() ?? [], Functions = x.Functions?.Select(f => FunctionDefMongoElement.ToMongoElement(f))?.ToList() ?? [], Responses = x.Responses?.Select(r => AgentResponseMongoElement.ToMongoElement(r))?.ToList() ?? [], RoutingRules = x.RoutingRules?.Select(r => RoutingRuleMongoElement.ToMongoElement(r))?.ToList() ?? [], @@ -634,6 +652,7 @@ private Agent TransformAgentDocument(AgentDocument? agentDoc) LlmConfig = AgentLlmConfigMongoElement.ToDomainElement(agentDoc.LlmConfig), ChannelInstructions = agentDoc.ChannelInstructions?.Select(i => ChannelInstructionMongoElement.ToDomainElement(i))?.ToList() ?? [], Templates = agentDoc.Templates?.Select(t => AgentTemplateMongoElement.ToDomainElement(t))?.ToList() ?? [], + Links = agentDoc.Links?.Select(l => AgentLinkMongoElement.ToDomainElement(l))?.ToList() ?? [], Functions = agentDoc.Functions?.Select(f => FunctionDefMongoElement.ToDomainElement(f)).ToList() ?? [], Responses = agentDoc.Responses?.Select(r => AgentResponseMongoElement.ToDomainElement(r))?.ToList() ?? [], RoutingRules = agentDoc.RoutingRules?.Select(r => RoutingRuleMongoElement.ToDomainElement(agentDoc.Id, agentDoc.Name, r))?.ToList() ?? [], diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj b/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj index 8b817fbcf..c1c3dfb57 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj +++ b/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj @@ -15,8 +15,7 @@ - - + \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ConversationItemCreated.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ConversationItemCreated.cs index b46d8cc7b..921de6269 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ConversationItemCreated.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ConversationItemCreated.cs @@ -10,6 +10,7 @@ public class ConversationItemBody { [JsonPropertyName("id")] public string Id { get; set; } = null!; + [JsonPropertyName("type")] public string Type { get; set; } = null!; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/SessionConversationUpdate.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/SessionConversationUpdate.cs deleted file mode 100644 index e2b12f57a..000000000 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/SessionConversationUpdate.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace BotSharp.Plugin.OpenAI.Models.Realtime; - -public class SessionConversationUpdate -{ - public string RawResponse { get; set; } - - public SessionConversationUpdate() - { - - } -} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index eda8d75bc..064718b49 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -1,5 +1,6 @@ +using BotSharp.Core.Realtime.Models.Chat; +using BotSharp.Core.Realtime.Websocket.Chat; using BotSharp.Plugin.OpenAI.Models.Realtime; -using BotSharp.Plugin.OpenAI.Providers.Realtime.Session; using OpenAI.Chat; namespace BotSharp.Plugin.OpenAI.Providers.Realtime; @@ -12,27 +13,28 @@ public class RealTimeCompletionProvider : IRealTimeCompletion public string Provider => "openai"; public string Model => _model; - protected readonly OpenAiSettings _settings; - protected readonly IServiceProvider _services; - protected readonly ILogger _logger; - private readonly BotSharpOptions _options; + private readonly RealtimeModelSettings _settings; + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private readonly BotSharpOptions _botsharpOptions; protected string _model = "gpt-4o-mini-realtime-preview"; private RealtimeChatSession _session; public RealTimeCompletionProvider( - OpenAiSettings settings, + RealtimeModelSettings settings, ILogger logger, IServiceProvider services, - BotSharpOptions options) + BotSharpOptions botsharpOptions) { _settings = settings; _logger = logger; _services = services; - _options = options; + _botsharpOptions = botsharpOptions; } - public async Task Connect(RealtimeHubConnection conn, + public async Task Connect( + RealtimeHubConnection conn, Action onModelReady, Action onModelAudioDeltaReceived, Action onModelAudioResponseDone, @@ -42,14 +44,28 @@ public async Task Connect(RealtimeHubConnection conn, Action onInputAudioTranscriptionCompleted, Action onInterruptionDetected) { + var settingsService = _services.GetRequiredService(); var realtimeModelSettings = _services.GetRequiredService(); + _model = realtimeModelSettings.Model; + var settings = settingsService.GetSetting(Provider, _model); - _session?.Dispose(); - _session = new RealtimeChatSession(_services, _options); - await _session.ConnectAsync(Provider, _model, CancellationToken.None); + if (_session != null) + { + _session.Dispose(); + } + _session = new RealtimeChatSession(_services, _botsharpOptions.JsonSerializerOptions); + await _session.ConnectAsync( + new Uri($"wss://api.openai.com/v1/realtime?model={_model}"), + new Dictionary + { + {"Authorization", $"Bearer {settings.ApiKey}"}, + {"OpenAI-Beta", "realtime=v1"} + }, + CancellationToken.None); - _ = ReceiveMessage(conn, + _ = ReceiveMessage( + conn, onModelReady, onModelAudioDeltaReceived, onModelAudioResponseDone, @@ -62,6 +78,7 @@ public async Task Connect(RealtimeHubConnection conn, public async Task Disconnect() { + _session?.Disconnect(); } @@ -132,7 +149,7 @@ private async Task ReceiveMessage(RealtimeHubConnection conn, Action onUserAudioTranscriptionCompleted, Action onInterruptionDetected) { - await foreach (SessionConversationUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None)) + await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None)) { var receivedText = update?.RawResponse; if (string.IsNullOrEmpty(receivedText)) @@ -205,11 +222,15 @@ private async Task ReceiveMessage(RealtimeHubConnection conn, else if (response.Type == "conversation.item.created") { _logger.LogInformation($"{response.Type}: {receivedText}"); + + var data = JsonSerializer.Deserialize(receivedText); + await Task.Delay(500); onConversationItemCreated(receivedText); } else if (response.Type == "conversation.item.input_audio_transcription.completed") { _logger.LogInformation($"{response.Type}: {receivedText}"); + var message = await OnUserAudioTranscriptionCompleted(conn, receivedText); if (!string.IsNullOrEmpty(message.Content)) { @@ -223,10 +244,17 @@ private async Task ReceiveMessage(RealtimeHubConnection conn, onInterruptionDetected(); } else if (response.Type == "input_audio_buffer.speech_stopped") + { + _logger.LogInformation($"{response.Type}: {receivedText}"); + await Task.Delay(500); + } + else if (response.Type == "input_audio_buffer.committed") { _logger.LogInformation($"{response.Type}: {receivedText}"); } } + + _session.Dispose(); } public async Task SendEventToModel(object message) @@ -236,7 +264,7 @@ public async Task SendEventToModel(object message) await _session.SendEventToModel(message); } - public async Task UpdateSession(RealtimeHubConnection conn) + public async Task UpdateSession(RealtimeHubConnection conn, bool isInit = false) { var convService = _services.GetRequiredService(); var conv = await convService.GetConversation(conn.ConversationId); @@ -257,28 +285,26 @@ public async Task UpdateSession(RealtimeHubConnection conn) return fn; }).ToArray(); - var realtimeModelSettings = _services.GetRequiredService(); - var sessionUpdate = new { type = "session.update", session = new RealtimeSessionUpdateRequest { - InputAudioFormat = realtimeModelSettings.InputAudioFormat, - OutputAudioFormat = realtimeModelSettings.OutputAudioFormat, - Voice = realtimeModelSettings.Voice, + InputAudioFormat = _settings.InputAudioFormat, + OutputAudioFormat = _settings.OutputAudioFormat, + Voice = _settings.Voice, Instructions = instruction, ToolChoice = "auto", Tools = functions, - Modalities = realtimeModelSettings.Modalities, - Temperature = Math.Max(options.Temperature ?? realtimeModelSettings.Temperature, 0.6f), - MaxResponseOutputTokens = realtimeModelSettings.MaxResponseOutputTokens, + Modalities = _settings.Modalities, + Temperature = Math.Max(options.Temperature ?? _settings.Temperature, 0.6f), + MaxResponseOutputTokens = _settings.MaxResponseOutputTokens, TurnDetection = new RealtimeSessionTurnDetection { - InterruptResponse = realtimeModelSettings.InterruptResponse/*, - Threshold = realtimeModelSettings.TurnDetection.Threshold, - PrefixPadding = realtimeModelSettings.TurnDetection.PrefixPadding, - SilenceDuration = realtimeModelSettings.TurnDetection.SilenceDuration*/ + InterruptResponse = _settings.InterruptResponse/*, + Threshold = _settings.TurnDetection.Threshold, + PrefixPadding = _settings.TurnDetection.PrefixPadding, + SilenceDuration = _settings.TurnDetection.SilenceDuration*/ }, InputAudioNoiseReduction = new InputAudioNoiseReduction { @@ -287,28 +313,26 @@ public async Task UpdateSession(RealtimeHubConnection conn) } }; - if (realtimeModelSettings.InputAudioTranscribe) + if (_settings.InputAudioTranscribe) { var words = new List(); HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent))); sessionUpdate.session.InputAudioTranscription = new InputAudioTranscription { - Model = realtimeModelSettings.InputAudioTranscription.Model, - Language = realtimeModelSettings.InputAudioTranscription.Language, + Model = _settings.InputAudioTranscription.Model, + Language = _settings.InputAudioTranscription.Language, Prompt = string.Join(", ", words.Select(x => x.ToLower().Trim()).Distinct()).SubstringMax(1024) }; } await HookEmitter.Emit(_services, async hook => { - await hook.OnSessionUpdated(agent, instruction, functions); + await hook.OnSessionUpdated(agent, instruction, functions, isInit); }); await SendEventToModel(sessionUpdate); - await Task.Delay(300); - return instruction; } @@ -584,6 +608,7 @@ public async Task> OnResponsedDone(RealtimeHubConnection c var contentHooks = _services.GetServices().ToList(); + var prompts = new List(); var inputTokenDetails = data.Usage?.InputTokenDetails; var outputTokenDetails = data.Usage?.OutputTokenDetails; @@ -601,26 +626,7 @@ public async Task> OnResponsedDone(RealtimeHubConnection c MessageType = MessageTypeName.FunctionCall }); - // After chat completion hook - foreach (var hook in contentHooks) - { - await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, $"{output.Name}\r\n{output.Arguments}") - { - CurrentAgentId = conn.CurrentAgentId - }, - new TokenStatsModel - { - Provider = Provider, - Model = _model, - Prompt = $"{output.Name}\r\n{output.Arguments}", - TextInputTokens = inputTokenDetails?.TextTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0, - CachedTextInputTokens = data.Usage?.InputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0, - AudioInputTokens = inputTokenDetails?.AudioTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0, - CachedAudioInputTokens = inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0, - TextOutputTokens = outputTokenDetails?.TextTokens ?? 0, - AudioOutputTokens = outputTokenDetails?.AudioTokens ?? 0 - }); - } + prompts.Add($"{output.Name}({output.Arguments})"); } else if (output.Type == "message") { @@ -633,29 +639,32 @@ await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, $"{output.Nam MessageType = MessageTypeName.Plain }); - // After chat completion hook - foreach (var hook in contentHooks) - { - await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, content.Transcript) - { - CurrentAgentId = conn.CurrentAgentId - }, - new TokenStatsModel - { - Provider = Provider, - Model = _model, - Prompt = content.Transcript, - TextInputTokens = inputTokenDetails?.TextTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0, - CachedTextInputTokens = data.Usage?.InputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0, - AudioInputTokens = inputTokenDetails?.AudioTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0, - CachedAudioInputTokens = inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0, - TextOutputTokens = outputTokenDetails?.TextTokens ?? 0, - AudioOutputTokens = outputTokenDetails?.AudioTokens ?? 0 - }); - } + prompts.Add(content.Transcript); } } + var text = string.Join("\r\n", prompts); + // After chat completion hook + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, text) + { + CurrentAgentId = conn.CurrentAgentId + }, + new TokenStatsModel + { + Provider = Provider, + Model = _model, + Prompt = text, + TextInputTokens = inputTokenDetails?.TextTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0, + CachedTextInputTokens = data.Usage?.InputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0, + AudioInputTokens = inputTokenDetails?.AudioTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0, + CachedAudioInputTokens = inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0, + TextOutputTokens = outputTokenDetails?.TextTokens ?? 0, + AudioOutputTokens = outputTokenDetails?.AudioTokens ?? 0 + }); + } + return outputs; } @@ -676,3 +685,11 @@ public async Task OnConversationItemCreated(RealtimeHubConnecti return message; } } + +class AudioMessage +{ + public string ItemId { get; set; } + public string Event { get; set; } + public string ReceivedText { get; set; } + public string? Transcript { get; set; } +} \ No newline at end of file diff --git a/src/WebStarter/Program.cs b/src/WebStarter/Program.cs index 21599a68b..814aa7e18 100644 --- a/src/WebStarter/Program.cs +++ b/src/WebStarter/Program.cs @@ -42,9 +42,12 @@ var app = builder.Build(); +app.UseWebSockets(); + // Enable SignalR app.MapHub("/chatHub"); -app.UseMiddleware(); +app.UseMiddleware(); +app.UseMiddleware(); // Use BotSharp app.UseBotSharp() diff --git a/tests/BotSharp.Test.RealtimeVoice/Program.cs b/tests/BotSharp.Test.RealtimeVoice/Program.cs index 1e6c94f5d..8a3ad9979 100644 --- a/tests/BotSharp.Test.RealtimeVoice/Program.cs +++ b/tests/BotSharp.Test.RealtimeVoice/Program.cs @@ -25,6 +25,7 @@ var hub = services.GetRequiredService(); var conn = hub.SetHubConnection(conv.Id); +conn.CurrentAgentId = conv.AgentId; conn.OnModelReady = () => JsonSerializer.Serialize(new @@ -74,7 +75,7 @@ await hub.ConnectToModel(async data => }); StreamReceiveResult result; -var buffer = new byte[1024 * 8]; +var buffer = new byte[1024 * 32]; do {