diff --git a/Directory.Packages.props b/Directory.Packages.props index 1885e9c77..d34f41597 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -3,12 +3,14 @@ true + + @@ -21,6 +23,7 @@ + diff --git a/KernelMemory.sln b/KernelMemory.sln index d792f8139..ccb93ddf1 100644 --- a/KernelMemory.sln +++ b/KernelMemory.sln @@ -274,6 +274,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elasticsearch", "extensions EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elasticsearch.UnitTests", "extensions\Elasticsearch\Elasticsearch.FunctionalTests\Elasticsearch.FunctionalTests.csproj", "{C5E6B28C-F54D-423D-954D-A9EAEFB89732}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Discord", "extensions\Discord\Discord\Discord.csproj", "{43877864-6AE8-4B03-BEDA-6B6FA8BB1D8B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "301-discord-test-application", "examples\301-discord-test-application\301-discord-test-application.csproj", "{FAE4C6B8-38B2-43E7-8881-99693C9CEDC6}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -500,6 +504,13 @@ Global {C5E6B28C-F54D-423D-954D-A9EAEFB89732}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {C5E6B28C-F54D-423D-954D-A9EAEFB89732}.Debug|Any CPU.Build.0 = Debug|Any CPU {C5E6B28C-F54D-423D-954D-A9EAEFB89732}.Release|Any CPU.ActiveCfg = Release|Any CPU + {43877864-6AE8-4B03-BEDA-6B6FA8BB1D8B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {43877864-6AE8-4B03-BEDA-6B6FA8BB1D8B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {43877864-6AE8-4B03-BEDA-6B6FA8BB1D8B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {43877864-6AE8-4B03-BEDA-6B6FA8BB1D8B}.Release|Any CPU.Build.0 = Release|Any CPU + {FAE4C6B8-38B2-43E7-8881-99693C9CEDC6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FAE4C6B8-38B2-43E7-8881-99693C9CEDC6}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FAE4C6B8-38B2-43E7-8881-99693C9CEDC6}.Release|Any CPU.ActiveCfg = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -580,6 +591,8 @@ Global {B9BE1099-F78F-4A5F-A897-BF2C75E19C57} = {155DA079-E267-49AF-973A-D1D44681970F} {2E10420F-BF96-411C-8FE0-F6268F2EEB67} = {155DA079-E267-49AF-973A-D1D44681970F} {C5E6B28C-F54D-423D-954D-A9EAEFB89732} = {3C17F42B-CFC8-4900-8CFB-88936311E919} + {43877864-6AE8-4B03-BEDA-6B6FA8BB1D8B} = {155DA079-E267-49AF-973A-D1D44681970F} + {FAE4C6B8-38B2-43E7-8881-99693C9CEDC6} = {0A43C65C-6007-4BB4-B3FE-8D439FC91841} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {CC136C62-115C-41D1-B414-F9473EFF6EA8} diff --git a/examples/301-discord-test-application/301-discord-test-application.csproj b/examples/301-discord-test-application/301-discord-test-application.csproj new file mode 100644 index 000000000..ccc49c130 --- /dev/null +++ b/examples/301-discord-test-application/301-discord-test-application.csproj @@ -0,0 +1,18 @@ + + + + Exe + net8.0 + $(NoWarn);CA1303;CA1031; + + + + + + + + + + + + diff --git a/examples/301-discord-test-application/DiscordDbContext.cs b/examples/301-discord-test-application/DiscordDbContext.cs new file mode 100644 index 000000000..ec0c007b4 --- /dev/null +++ b/examples/301-discord-test-application/DiscordDbContext.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.EntityFrameworkCore; + +namespace Microsoft.Discord.TestApplication; + +public class DiscordDbContext : DbContext +{ + public DbContextOptions Options { get; } + + public DbSet Messages { get; set; } + + public DiscordDbContext(DbContextOptions options) : base(options) + { + this.Options = options; + } +} diff --git a/examples/301-discord-test-application/DiscordDbMessage.cs b/examples/301-discord-test-application/DiscordDbMessage.cs new file mode 100644 index 000000000..e175cee54 --- /dev/null +++ b/examples/301-discord-test-application/DiscordDbMessage.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.ComponentModel.DataAnnotations; +using Microsoft.KernelMemory.Sources.DiscordBot; + +namespace Microsoft.Discord.TestApplication; + +public class DiscordDbMessage : DiscordMessage +{ + [Key] + public string Id + { + get + { + return this.MessageId; + } + set + { + this.MessageId = value; + } + } +} diff --git a/examples/301-discord-test-application/DiscordMessageHandler.cs b/examples/301-discord-test-application/DiscordMessageHandler.cs new file mode 100644 index 000000000..6923f5549 --- /dev/null +++ b/examples/301-discord-test-application/DiscordMessageHandler.cs @@ -0,0 +1,119 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Json; +using Microsoft.KernelMemory; +using Microsoft.KernelMemory.Diagnostics; +using Microsoft.KernelMemory.Pipeline; +using Microsoft.KernelMemory.Sources.DiscordBot; + +namespace Microsoft.Discord.TestApplication; + +/// +/// KM pipeline handler fetching discord data files from content storage +/// and storing messages in Postgres. +/// +public sealed class DiscordMessageHandler : IPipelineStepHandler, IDisposable, IAsyncDisposable +{ + // Name of the file where to store Discord data + private readonly string _filename; + + // KM pipelines orchestrator + private readonly IPipelineOrchestrator _orchestrator; + + // .NET service provider, used to get thread safe instances of EF DbContext + private readonly IServiceProvider _serviceProvider; + + // EF DbContext used to create the database + private DiscordDbContext? _firstInvokeDb; + + // .NET logger + private readonly ILogger _log; + + public string StepName { get; } = string.Empty; + + public DiscordMessageHandler( + string stepName, + IPipelineOrchestrator orchestrator, + DiscordConnectorConfig config, + IServiceProvider serviceProvider, + ILoggerFactory? loggerFactory = null) + { + this.StepName = stepName; + this._log = loggerFactory?.CreateLogger() ?? DefaultLogger.Instance; + + this._orchestrator = orchestrator; + this._serviceProvider = serviceProvider; + this._filename = config.FileName; + + // This DbContext instance is used only to create the database + this._firstInvokeDb = serviceProvider.GetService() ?? throw new ConfigurationException("Discord DB Content is not defined"); + } + + public async Task<(bool success, DataPipeline updatedPipeline)> InvokeAsync(DataPipeline pipeline, CancellationToken cancellationToken = default) + { + this.OnFirstInvoke(); + + // Note: use a new DbContext instance each time, because DbContext is not thread safe and would throw the following + // exception: System.InvalidOperationException: a second operation was started on this context instance before a previous + // operation completed. This is usually caused by different threads concurrently using the same instance of DbContext. + // For more information on how to avoid threading issues with DbContext, see https://go.microsoft.com/fwlink/?linkid=2097913. + await using (var db = (this._serviceProvider.GetService())!) + { + foreach (DataPipeline.FileDetails uploadedFile in pipeline.Files) + { + // Process only the file containing the discord data + if (uploadedFile.Name != this._filename) { continue; } + + string fileContent = await this._orchestrator.ReadTextFileAsync(pipeline, uploadedFile.Name, cancellationToken).ConfigureAwait(false); + + DiscordDbMessage? data; + try + { + data = JsonSerializer.Deserialize(fileContent); + if (data == null) + { + this._log.LogError("Failed to deserialize Discord data file, result is NULL"); + return (true, pipeline); + } + } + catch (Exception e) + { + this._log.LogError(e, "Failed to deserialize Discord data file"); + return (true, pipeline); + } + + await db.Messages.AddAsync(data, cancellationToken).ConfigureAwait(false); + } + + await db.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + } + + return (true, pipeline); + } + + public void Dispose() + { + this._firstInvokeDb?.Dispose(); + this._firstInvokeDb = null; + } + + public async ValueTask DisposeAsync() + { + if (this._firstInvokeDb != null) { await this._firstInvokeDb.DisposeAsync(); } + + this._firstInvokeDb = null; + } + + private void OnFirstInvoke() + { + if (this._firstInvokeDb == null) { return; } + + lock (this._firstInvokeDb) + { + // Create DB / Tables if needed + this._firstInvokeDb.Database.EnsureCreated(); + this._firstInvokeDb.Dispose(); + this._firstInvokeDb = null; + } + } +} diff --git a/examples/301-discord-test-application/Program.cs b/examples/301-discord-test-application/Program.cs new file mode 100644 index 000000000..5701fc6a5 --- /dev/null +++ b/examples/301-discord-test-application/Program.cs @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.KernelMemory; +using Microsoft.KernelMemory.ContentStorage.DevTools; +using Microsoft.KernelMemory.Sources.DiscordBot; + +namespace Microsoft.Discord.TestApplication; + +/* Example: Listen for new messages in Discord, and save them in a table in Postgres. + * + * Use ASP.NET hosted services to host a Discord Bot. The discord bot logic is based + * on DiscordConnector class. + * + * While the Discord bot is running, every time there is a new message, DiscordConnector + * invokes KM.ImportDocument API, uploading a JSON file that contains details about the + * Discord message, including server ID, channel ID, author ID, message content, etc. + * + * The call to KM.ImportDocument API asks to process the JSON file uploaded using + * DiscordMessageHandler, included in this project. No other handlers are used. + * + * DiscordMessageHandler, loads the uploaded file, deserializes its content, and + * save each Discord message into a table in Postgres, using Entity Framework. + */ + +internal static class Program +{ + public static void Main(string[] args) + { + WebApplicationBuilder appBuilder = WebApplication.CreateBuilder(); + + appBuilder.Configuration + .AddJsonFile("appsettings.json") + .AddJsonFile("appsettings.Development.json", optional: true) + .AddEnvironmentVariables() + .AddCommandLine(args); + + // Discord setup + // Use DiscordConnector to connect to Discord and listen for messages. + // The Discord connection can listen from multiple servers and channels. + // For each message, DiscordConnector will send a file to Kernel Memory to process. + // Files sent to Kernel Memory are processed by DiscordMessageHandler (in this project) + var discordCfg = appBuilder.Configuration.GetSection("Discord").Get(); + ArgumentNullExceptionEx.ThrowIfNull(discordCfg, nameof(discordCfg), "Discord config is NULL"); + appBuilder.Services.AddSingleton(discordCfg); + appBuilder.Services.AddHostedService(); + + // Postgres with Entity Framework + // DiscordMessageHandler reads files received by Kernel Memory and store each message in a table in Postgres. + // See DiscordDbMessage for the table schema. + appBuilder.AddNpgsqlDbContext("postgresDb"); + + // Run Kernel Memory and DiscordMessageHandler + // var kmApp = BuildAsynchronousKernelMemoryApp(appBuilder, discordConfig); + var kmApp = BuildSynchronousKernelMemoryApp(appBuilder, discordCfg); + + Console.WriteLine("Starting KM application...\n"); + kmApp.Run(); + Console.WriteLine("\n... KM application stopped."); + } + + private static WebApplication BuildSynchronousKernelMemoryApp(WebApplicationBuilder appBuilder, DiscordConnectorConfig discordConfig) + { + appBuilder.AddKernelMemory(kmb => + { + // Note: there's no queue system, so the memory instance will be synchronous (ie MemoryServerless) + + // Store files on disk + kmb.WithSimpleFileStorage(SimpleFileStorageConfig.Persistent); + + // Disable AI, not needed for this example + kmb.WithoutEmbeddingGenerator(); + kmb.WithoutTextGenerator(); + }); + + WebApplication app = appBuilder.Build(); + + // In synchronous apps, handlers are added to the serverless memory orchestrator + (app.Services.GetService() as MemoryServerless)! + .Orchestrator + .AddHandler(discordConfig.Steps[0]); + + return app; + } + + private static WebApplication BuildAsynchronousKernelMemoryApp(WebApplicationBuilder appBuilder, DiscordConnectorConfig discordConfig) + { + appBuilder.Services.AddHandlerAsHostedService(discordConfig.Steps[0]); + appBuilder.AddKernelMemory(kmb => + { + // Note: because of this the memory instance will be asynchronous (ie MemoryService) + kmb.WithSimpleQueuesPipeline(); + + // Store files on disk + kmb.WithSimpleFileStorage(SimpleFileStorageConfig.Persistent); + + // Disable AI, not needed for this example + kmb.WithoutEmbeddingGenerator(); + kmb.WithoutTextGenerator(); + }); + + return appBuilder.Build(); + } +} diff --git a/examples/301-discord-test-application/appsettings.json b/examples/301-discord-test-application/appsettings.json new file mode 100644 index 000000000..f4bd01e51 --- /dev/null +++ b/examples/301-discord-test-application/appsettings.json @@ -0,0 +1,38 @@ +{ + "Discord": { + // Discord bot authentication token + // See https://discord.com/developers + "DiscordToken": "", + // Index where to store files, e.g. disk folder, Azure blobs folder, etc. + "Index": "discord", + // File name used when uploading a message to content storage. + "FileName": "discord-msg.json", + // Handlers processing the incoming Discord events + "Steps": [ + "store_discord_message" + ] + }, + "ConnectionStrings": { + // Db where Discord messages are stored, e.g. + // "Host=contoso.postgres.database.azure.com;Port=5432;Username=adminuser;Password=mypassword;Database=discorddata;SSL Mode=VerifyFull" + "postgresDb": "Host=localhost;Port=5432;Username=;Password=" + }, + "Logging": { + "LogLevel": { + "Default": "Warning" + }, + "Console": { + "LogToStandardErrorThreshold": "Critical", + "FormatterName": "simple", + "FormatterOptions": { + "TimestampFormat": "[HH:mm:ss.fff] ", + "SingleLine": true, + "UseUtcTimestamp": false, + "IncludeScopes": false, + "JsonWriterOptions": { + "Indented": true + } + } + } + } +} \ No newline at end of file diff --git a/extensions/Discord/Discord/Discord.csproj b/extensions/Discord/Discord/Discord.csproj new file mode 100644 index 000000000..8c76bf4b0 --- /dev/null +++ b/extensions/Discord/Discord/Discord.csproj @@ -0,0 +1,32 @@ + + + + net8.0 + LatestMajor + Microsoft.KernelMemory.Sources.DiscordBot + Microsoft.KernelMemory.Sources.DiscordBot + $(NoWarn);CS1591;CA1303; + + + + + + + + false + Microsoft.KernelMemory.Sources.Discord + Discord connector for Kernel Memory + Discord connector for Kernel Memory + Discord, Kernel Memory, AI, Artificial Intelligence, ETL + + + + + + + + + + + + diff --git a/extensions/Discord/Discord/DiscordConnector.cs b/extensions/Discord/Discord/DiscordConnector.cs new file mode 100644 index 000000000..36cf458d3 --- /dev/null +++ b/extensions/Discord/Discord/DiscordConnector.cs @@ -0,0 +1,180 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Discord; +using Discord.WebSocket; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace Microsoft.KernelMemory.Sources.DiscordBot; + +/// +/// Service responsible for connecting to Discord, listening for messages +/// and generating events for Kernel Memory. +/// +public sealed class DiscordConnector : IHostedService, IDisposable, IAsyncDisposable +{ + private readonly DiscordSocketClient _client; + private readonly IKernelMemory _memory; + private readonly ILogger _log; + private readonly string _authToken; + private readonly string _contentStorageIndex; + private readonly string _contentStorageFilename; + private readonly List _pipelineSteps; + + /// + /// New instance of Discord bot + /// + /// Discord settings + /// Memory instance used to upload files when messages arrives + /// App log factory + public DiscordConnector( + DiscordConnectorConfig config, + IKernelMemory memory, + ILoggerFactory logFactory) + { + this._log = logFactory.CreateLogger(); + this._authToken = config.DiscordToken; + + var dc = new DiscordSocketConfig + { + LogLevel = LogSeverity.Debug, + GatewayIntents = GatewayIntents.AllUnprivileged | GatewayIntents.MessageContent, + LogGatewayIntentWarnings = true, + SuppressUnknownDispatchWarnings = false + }; + + this._client = new DiscordSocketClient(dc); + this._client.Log += this.OnLog; + this._client.MessageReceived += this.OnMessage; + this._memory = memory; + this._contentStorageIndex = config.Index; + this._pipelineSteps = config.Steps; + this._contentStorageFilename = config.FileName; + } + + /// + public async Task StartAsync(CancellationToken cancellationToken) + { + await this._client.LoginAsync(TokenType.Bot, this._authToken).ConfigureAwait(false); + await this._client.StartAsync().ConfigureAwait(false); + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + await this._client.LogoutAsync().ConfigureAwait(false); + await this._client.StopAsync().ConfigureAwait(false); + } + + /// + public void Dispose() + { + this._client.Dispose(); + } + + /// + public async ValueTask DisposeAsync() + { + await this._client.DisposeAsync().ConfigureAwait(false); + } + + #region private + + private static readonly Dictionary s_logLevels = new() + { + [LogSeverity.Critical] = LogLevel.Critical, + [LogSeverity.Error] = LogLevel.Error, + [LogSeverity.Warning] = LogLevel.Warning, + [LogSeverity.Info] = LogLevel.Information, + [LogSeverity.Verbose] = LogLevel.Debug, // note the inconsistency + [LogSeverity.Debug] = LogLevel.Trace // note the inconsistency + }; + + private Task OnMessage(SocketMessage message) + { + var msg = new DiscordMessage + { + MessageId = message.Id.ToString(CultureInfo.InvariantCulture), + AuthorId = message.Author.Id.ToString(CultureInfo.InvariantCulture), + ChannelId = message.Channel.Id.ToString(CultureInfo.InvariantCulture), + ReferenceMessageId = message.Reference?.MessageId.ToString() ?? string.Empty, + AuthorUsername = message.Author.Username, + ChannelName = message.Channel.Name, + Timestamp = message.Timestamp, + Content = message.Content, + CleanContent = message.CleanContent, + EmbedsCount = message.Embeds.Count, + }; + + if (message.Channel is SocketTextChannel textChannel) + { + msg.ChannelMention = textChannel.Mention; + msg.ChannelTopic = textChannel.Topic; + msg.ServerId = textChannel.Guild.Id.ToString(CultureInfo.InvariantCulture); + msg.ServerName = textChannel.Guild.Name; + msg.ServerDescription = textChannel.Guild.Description; + msg.ServerMemberCount = textChannel.Guild.MemberCount; + } + + this._log.LogTrace("[{0}] New message from '{1}' [{2}]", msg.MessageId, msg.AuthorUsername, msg.AuthorId); + this._log.LogTrace("[{0}] Channel: {1}", msg.MessageId, msg.ChannelId); + this._log.LogTrace("[{0}] Channel: {1}", msg.MessageId, msg.ChannelName); + this._log.LogTrace("[{0}] Timestamp: {1}", msg.MessageId, msg.Timestamp); + this._log.LogTrace("[{0}] Content: {1}", msg.MessageId, msg.Content); + this._log.LogTrace("[{0}] CleanContent: {1}", msg.MessageId, msg.CleanContent); + this._log.LogTrace("[{0}] Reference: {1}", msg.MessageId, msg.ReferenceMessageId); + this._log.LogTrace("[{0}] EmbedsCount: {1}", msg.MessageId, msg.EmbedsCount); + if (message.Embeds.Count > 0) + { + foreach (Embed? x in message.Embeds) + { + if (x == null) { continue; } + + this._log.LogTrace("[{0}] Embed Title: {1}", message.Id, x.Title); + this._log.LogTrace("[{0}] Embed Url: {1}", message.Id, x.Url); + this._log.LogTrace("[{0}] Embed Description: {1}", message.Id, x.Description); + } + } + + Task.Run(async () => + { + string documentId = $"{msg.ServerId}_{msg.ChannelId}_{msg.MessageId}"; + string content = JsonSerializer.Serialize(msg); + Stream fileContent = new MemoryStream(Encoding.UTF8.GetBytes(content), false); + await using (fileContent) + { + await this._memory.ImportDocumentAsync( + fileContent, + fileName: this._contentStorageFilename, + documentId: documentId, + index: this._contentStorageIndex, + steps: this._pipelineSteps).ConfigureAwait(false); + } + }); + + return Task.CompletedTask; + } + + private Task OnLog(LogMessage msg) + { + var logLevel = LogLevel.Information; + if (s_logLevels.TryGetValue(msg.Severity, out LogLevel value)) + { + logLevel = value; + } + + this._log.Log(logLevel, "{0}: {1}", msg.Source, msg.Message); + + return Task.CompletedTask; + } + + #endregion +} diff --git a/extensions/Discord/Discord/DiscordConnectorConfig.cs b/extensions/Discord/Discord/DiscordConnectorConfig.cs new file mode 100644 index 000000000..65200f83b --- /dev/null +++ b/extensions/Discord/Discord/DiscordConnectorConfig.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; + +namespace Microsoft.KernelMemory.Sources.DiscordBot; + +/// +/// Discord bot settings +/// +public class DiscordConnectorConfig +{ + /// + /// Discord bot authentication token + /// + public string DiscordToken { get; set; } = string.Empty; + + /// + /// Index where to store files (not memories) + /// + public string Index { get; set; } = "discord"; + + /// + /// File name used when uploading a message. + /// + public string FileName { get; set; } = "discord.json"; + + /// + /// Handlers processing the incoming Discord events + /// + public List Steps { get; set; } = []; +} diff --git a/extensions/Discord/Discord/DiscordMessage.cs b/extensions/Discord/Discord/DiscordMessage.cs new file mode 100644 index 000000000..0120caa17 --- /dev/null +++ b/extensions/Discord/Discord/DiscordMessage.cs @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text.Json.Serialization; + +namespace Microsoft.KernelMemory.Sources.DiscordBot; + +public class DiscordMessage +{ + [JsonPropertyOrder(0)] + [JsonPropertyName("message_id")] + public string MessageId { get; set; } = string.Empty; + + [JsonPropertyOrder(1)] + [JsonPropertyName("reference_message_id")] + public string? ReferenceMessageId { get; set; } = string.Empty; + + [JsonPropertyOrder(2)] + [JsonPropertyName("author_username")] + public string? AuthorUsername { get; set; } = string.Empty; + + [JsonPropertyOrder(3)] + [JsonPropertyName("author_id")] + public string? AuthorId { get; set; } = string.Empty; + + [JsonPropertyOrder(4)] + [JsonPropertyName("channel_name")] + public string? ChannelName { get; set; } = string.Empty; + + [JsonPropertyOrder(5)] + [JsonPropertyName("channel_id")] + public string? ChannelId { get; set; } = string.Empty; + + [JsonPropertyOrder(6)] + [JsonPropertyName("channel_mention")] + public string? ChannelMention { get; set; } = string.Empty; + + [JsonPropertyOrder(7)] + [JsonPropertyName("channel_topic")] + public string? ChannelTopic { get; set; } = string.Empty; + + [JsonPropertyOrder(8)] + [JsonPropertyName("server_id")] + public string? ServerId { get; set; } = string.Empty; + + [JsonPropertyOrder(9)] + [JsonPropertyName("server_name")] + public string? ServerName { get; set; } = string.Empty; + + [JsonPropertyOrder(10)] + [JsonPropertyName("server_description")] + public string? ServerDescription { get; set; } = string.Empty; + + [JsonPropertyOrder(11)] + [JsonPropertyName("server_member_count")] + public int ServerMemberCount { get; set; } = 0; + + [JsonPropertyOrder(12)] + [JsonPropertyName("embeds_count")] + public int EmbedsCount { get; set; } = 0; + + [JsonPropertyOrder(13)] + [JsonPropertyName("timestamp")] + public DateTimeOffset Timestamp { get; set; } = DateTimeOffset.MinValue; + + [JsonPropertyOrder(14)] + [JsonPropertyName("content")] + public string? Content { get; set; } = string.Empty; + + [JsonPropertyOrder(15)] + [JsonPropertyName("clean_content")] + public string? CleanContent { get; set; } = string.Empty; +} diff --git a/service/Service/appsettings.json b/service/Service/appsettings.json index 4a9c07526..89fb4cc11 100644 --- a/service/Service/appsettings.json +++ b/service/Service/appsettings.json @@ -415,5 +415,21 @@ "TagsTableName": "KMMemoriesTags" } } + }, + // This is an experimental Discord connector used to process Discord messages. + // The connector uses a Discord bot to upload messages to content storage, which + // are processed with custom handlers. + "Discord": { + // Discord bot authentication token + // See https://discord.com/developers + "DiscordToken": "", + // Index where to store files, e.g. disk folder, Azure blobs folder, etc. + "Index": "discord", + // File name used when uploading a message to content storage. + "FileName": "discord-msg.json", + // Handlers processing the incoming Discord events + "Steps": [ + "store_discord_message" + ] } } \ No newline at end of file