-
Notifications
You must be signed in to change notification settings - Fork 336
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
13 changed files
with
665 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
18 changes: 18 additions & 0 deletions
18
examples/301-discord-test-application/301-discord-test-application.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk.Web"> | ||
|
|
||
| <PropertyGroup> | ||
| <OutputType>Exe</OutputType> | ||
| <TargetFramework>net8.0</TargetFramework> | ||
| <NoWarn>$(NoWarn);CA1303;CA1031;</NoWarn> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <ProjectReference Include="..\..\extensions\Discord\Discord\Discord.csproj" /> | ||
| <ProjectReference Include="..\..\service\Service.AspNetCore\Service.AspNetCore.csproj" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="Aspire.Npgsql.EntityFrameworkCore.PostgreSQL" /> | ||
| </ItemGroup> | ||
|
|
||
| </Project> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using Microsoft.EntityFrameworkCore; | ||
|
|
||
| namespace Microsoft.Discord.TestApplication; | ||
|
|
||
| public class DiscordDbContext : DbContext | ||
| { | ||
| public DbContextOptions<DiscordDbContext> Options { get; } | ||
|
|
||
| public DbSet<DiscordDbMessage> Messages { get; set; } | ||
|
|
||
| public DiscordDbContext(DbContextOptions<DiscordDbContext> options) : base(options) | ||
| { | ||
| this.Options = options; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using System.ComponentModel.DataAnnotations; | ||
| using Microsoft.KernelMemory.Sources.DiscordBot; | ||
|
|
||
| namespace Microsoft.Discord.TestApplication; | ||
|
|
||
| public class DiscordDbMessage : DiscordMessage | ||
| { | ||
| [Key] | ||
| public string Id | ||
| { | ||
| get | ||
| { | ||
| return this.MessageId; | ||
| } | ||
| set | ||
| { | ||
| this.MessageId = value; | ||
| } | ||
| } | ||
| } |
119 changes: 119 additions & 0 deletions
119
examples/301-discord-test-application/DiscordMessageHandler.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,119 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using System.Text.Json; | ||
| using Microsoft.KernelMemory; | ||
| using Microsoft.KernelMemory.Diagnostics; | ||
| using Microsoft.KernelMemory.Pipeline; | ||
| using Microsoft.KernelMemory.Sources.DiscordBot; | ||
|
|
||
| namespace Microsoft.Discord.TestApplication; | ||
|
|
||
| /// <summary> | ||
| /// KM pipeline handler fetching discord data files from content storage | ||
| /// and storing messages in Postgres. | ||
| /// </summary> | ||
| public sealed class DiscordMessageHandler : IPipelineStepHandler, IDisposable, IAsyncDisposable | ||
| { | ||
| // Name of the file where to store Discord data | ||
| private readonly string _filename; | ||
|
|
||
| // KM pipelines orchestrator | ||
| private readonly IPipelineOrchestrator _orchestrator; | ||
|
|
||
| // .NET service provider, used to get thread safe instances of EF DbContext | ||
| private readonly IServiceProvider _serviceProvider; | ||
|
|
||
| // EF DbContext used to create the database | ||
| private DiscordDbContext? _firstInvokeDb; | ||
|
|
||
| // .NET logger | ||
| private readonly ILogger<DiscordMessageHandler> _log; | ||
|
|
||
| public string StepName { get; } = string.Empty; | ||
|
|
||
| public DiscordMessageHandler( | ||
| string stepName, | ||
| IPipelineOrchestrator orchestrator, | ||
| DiscordConnectorConfig config, | ||
| IServiceProvider serviceProvider, | ||
| ILoggerFactory? loggerFactory = null) | ||
| { | ||
| this.StepName = stepName; | ||
| this._log = loggerFactory?.CreateLogger<DiscordMessageHandler>() ?? DefaultLogger<DiscordMessageHandler>.Instance; | ||
|
|
||
| this._orchestrator = orchestrator; | ||
| this._serviceProvider = serviceProvider; | ||
| this._filename = config.FileName; | ||
|
|
||
| // This DbContext instance is used only to create the database | ||
| this._firstInvokeDb = serviceProvider.GetService<DiscordDbContext>() ?? throw new ConfigurationException("Discord DB Content is not defined"); | ||
| } | ||
|
|
||
| public async Task<(bool success, DataPipeline updatedPipeline)> InvokeAsync(DataPipeline pipeline, CancellationToken cancellationToken = default) | ||
| { | ||
| this.OnFirstInvoke(); | ||
|
|
||
| // Note: use a new DbContext instance each time, because DbContext is not thread safe and would throw the following | ||
| // exception: System.InvalidOperationException: a second operation was started on this context instance before a previous | ||
| // operation completed. This is usually caused by different threads concurrently using the same instance of DbContext. | ||
| // For more information on how to avoid threading issues with DbContext, see https://go.microsoft.com/fwlink/?linkid=2097913. | ||
| await using (var db = (this._serviceProvider.GetService<DiscordDbContext>())!) | ||
| { | ||
| foreach (DataPipeline.FileDetails uploadedFile in pipeline.Files) | ||
| { | ||
| // Process only the file containing the discord data | ||
| if (uploadedFile.Name != this._filename) { continue; } | ||
|
|
||
| string fileContent = await this._orchestrator.ReadTextFileAsync(pipeline, uploadedFile.Name, cancellationToken).ConfigureAwait(false); | ||
|
|
||
| DiscordDbMessage? data; | ||
| try | ||
| { | ||
| data = JsonSerializer.Deserialize<DiscordDbMessage>(fileContent); | ||
| if (data == null) | ||
| { | ||
| this._log.LogError("Failed to deserialize Discord data file, result is NULL"); | ||
| return (true, pipeline); | ||
| } | ||
| } | ||
| catch (Exception e) | ||
| { | ||
| this._log.LogError(e, "Failed to deserialize Discord data file"); | ||
| return (true, pipeline); | ||
| } | ||
|
|
||
| await db.Messages.AddAsync(data, cancellationToken).ConfigureAwait(false); | ||
| } | ||
|
|
||
| await db.SaveChangesAsync(cancellationToken).ConfigureAwait(false); | ||
| } | ||
|
|
||
| return (true, pipeline); | ||
| } | ||
|
|
||
| public void Dispose() | ||
| { | ||
| this._firstInvokeDb?.Dispose(); | ||
| this._firstInvokeDb = null; | ||
| } | ||
|
|
||
| public async ValueTask DisposeAsync() | ||
| { | ||
| if (this._firstInvokeDb != null) { await this._firstInvokeDb.DisposeAsync(); } | ||
|
|
||
| this._firstInvokeDb = null; | ||
| } | ||
|
|
||
| private void OnFirstInvoke() | ||
| { | ||
| if (this._firstInvokeDb == null) { return; } | ||
|
|
||
| lock (this._firstInvokeDb) | ||
| { | ||
| // Create DB / Tables if needed | ||
| this._firstInvokeDb.Database.EnsureCreated(); | ||
| this._firstInvokeDb.Dispose(); | ||
| this._firstInvokeDb = null; | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,103 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using Microsoft.KernelMemory; | ||
| using Microsoft.KernelMemory.ContentStorage.DevTools; | ||
| using Microsoft.KernelMemory.Sources.DiscordBot; | ||
|
|
||
| namespace Microsoft.Discord.TestApplication; | ||
|
|
||
| /* Example: Listen for new messages in Discord, and save them in a table in Postgres. | ||
| * | ||
| * Use ASP.NET hosted services to host a Discord Bot. The discord bot logic is based | ||
| * on DiscordConnector class. | ||
| * | ||
| * While the Discord bot is running, every time there is a new message, DiscordConnector | ||
| * invokes KM.ImportDocument API, uploading a JSON file that contains details about the | ||
| * Discord message, including server ID, channel ID, author ID, message content, etc. | ||
| * | ||
| * The call to KM.ImportDocument API asks to process the JSON file uploaded using | ||
| * DiscordMessageHandler, included in this project. No other handlers are used. | ||
| * | ||
| * DiscordMessageHandler, loads the uploaded file, deserializes its content, and | ||
| * save each Discord message into a table in Postgres, using Entity Framework. | ||
| */ | ||
|
|
||
| internal static class Program | ||
| { | ||
| public static void Main(string[] args) | ||
| { | ||
| WebApplicationBuilder appBuilder = WebApplication.CreateBuilder(); | ||
|
|
||
| appBuilder.Configuration | ||
| .AddJsonFile("appsettings.json") | ||
| .AddJsonFile("appsettings.Development.json", optional: true) | ||
| .AddEnvironmentVariables() | ||
| .AddCommandLine(args); | ||
|
|
||
| // Discord setup | ||
| // Use DiscordConnector to connect to Discord and listen for messages. | ||
| // The Discord connection can listen from multiple servers and channels. | ||
| // For each message, DiscordConnector will send a file to Kernel Memory to process. | ||
| // Files sent to Kernel Memory are processed by DiscordMessageHandler (in this project) | ||
| var discordCfg = appBuilder.Configuration.GetSection("Discord").Get<DiscordConnectorConfig>(); | ||
| ArgumentNullExceptionEx.ThrowIfNull(discordCfg, nameof(discordCfg), "Discord config is NULL"); | ||
| appBuilder.Services.AddSingleton<DiscordConnectorConfig>(discordCfg); | ||
| appBuilder.Services.AddHostedService<DiscordConnector>(); | ||
|
|
||
| // Postgres with Entity Framework | ||
| // DiscordMessageHandler reads files received by Kernel Memory and store each message in a table in Postgres. | ||
| // See DiscordDbMessage for the table schema. | ||
| appBuilder.AddNpgsqlDbContext<DiscordDbContext>("postgresDb"); | ||
|
|
||
| // Run Kernel Memory and DiscordMessageHandler | ||
| // var kmApp = BuildAsynchronousKernelMemoryApp(appBuilder, discordConfig); | ||
| var kmApp = BuildSynchronousKernelMemoryApp(appBuilder, discordCfg); | ||
|
|
||
| Console.WriteLine("Starting KM application...\n"); | ||
| kmApp.Run(); | ||
| Console.WriteLine("\n... KM application stopped."); | ||
| } | ||
|
|
||
| private static WebApplication BuildSynchronousKernelMemoryApp(WebApplicationBuilder appBuilder, DiscordConnectorConfig discordConfig) | ||
| { | ||
| appBuilder.AddKernelMemory(kmb => | ||
| { | ||
| // Note: there's no queue system, so the memory instance will be synchronous (ie MemoryServerless) | ||
|
|
||
| // Store files on disk | ||
| kmb.WithSimpleFileStorage(SimpleFileStorageConfig.Persistent); | ||
|
|
||
| // Disable AI, not needed for this example | ||
| kmb.WithoutEmbeddingGenerator(); | ||
| kmb.WithoutTextGenerator(); | ||
| }); | ||
|
|
||
| WebApplication app = appBuilder.Build(); | ||
|
|
||
| // In synchronous apps, handlers are added to the serverless memory orchestrator | ||
| (app.Services.GetService<IKernelMemory>() as MemoryServerless)! | ||
| .Orchestrator | ||
| .AddHandler<DiscordMessageHandler>(discordConfig.Steps[0]); | ||
|
|
||
| return app; | ||
| } | ||
|
|
||
| private static WebApplication BuildAsynchronousKernelMemoryApp(WebApplicationBuilder appBuilder, DiscordConnectorConfig discordConfig) | ||
| { | ||
| appBuilder.Services.AddHandlerAsHostedService<DiscordMessageHandler>(discordConfig.Steps[0]); | ||
| appBuilder.AddKernelMemory(kmb => | ||
| { | ||
| // Note: because of this the memory instance will be asynchronous (ie MemoryService) | ||
| kmb.WithSimpleQueuesPipeline(); | ||
|
|
||
| // Store files on disk | ||
| kmb.WithSimpleFileStorage(SimpleFileStorageConfig.Persistent); | ||
|
|
||
| // Disable AI, not needed for this example | ||
| kmb.WithoutEmbeddingGenerator(); | ||
| kmb.WithoutTextGenerator(); | ||
| }); | ||
|
|
||
| return appBuilder.Build(); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| { | ||
| "Discord": { | ||
| // Discord bot authentication token | ||
| // See https://discord.com/developers | ||
| "DiscordToken": "", | ||
| // Index where to store files, e.g. disk folder, Azure blobs folder, etc. | ||
| "Index": "discord", | ||
| // File name used when uploading a message to content storage. | ||
| "FileName": "discord-msg.json", | ||
| // Handlers processing the incoming Discord events | ||
| "Steps": [ | ||
| "store_discord_message" | ||
| ] | ||
| }, | ||
| "ConnectionStrings": { | ||
| // Db where Discord messages are stored, e.g. | ||
| // "Host=contoso.postgres.database.azure.com;Port=5432;Username=adminuser;Password=mypassword;Database=discorddata;SSL Mode=VerifyFull" | ||
| "postgresDb": "Host=localhost;Port=5432;Username=;Password=" | ||
| }, | ||
| "Logging": { | ||
| "LogLevel": { | ||
| "Default": "Warning" | ||
| }, | ||
| "Console": { | ||
| "LogToStandardErrorThreshold": "Critical", | ||
| "FormatterName": "simple", | ||
| "FormatterOptions": { | ||
| "TimestampFormat": "[HH:mm:ss.fff] ", | ||
| "SingleLine": true, | ||
| "UseUtcTimestamp": false, | ||
| "IncludeScopes": false, | ||
| "JsonWriterOptions": { | ||
| "Indented": true | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.