diff --git a/examples/001-dotnet-WebClient/Program.cs b/examples/001-dotnet-WebClient/Program.cs index 04991961f..f54f218b8 100644 --- a/examples/001-dotnet-WebClient/Program.cs +++ b/examples/001-dotnet-WebClient/Program.cs @@ -86,6 +86,7 @@ private static async Task StoreText() "in a system's rest frame, where the two quantities differ only by a multiplicative " + "constant and the units of measurement. The principle is described by the physicist " + "Albert Einstein's formula: E = m*c^2"); + Console.WriteLine($"- Document Id: {docId}"); s_toDelete.Add(docId); } @@ -93,8 +94,9 @@ private static async Task StoreText() private static async Task StoreFile() { Console.WriteLine("Uploading article file about Carbon"); - await s_memory.ImportDocumentAsync("file1-Wikipedia-Carbon.txt", documentId: "doc001"); - s_toDelete.Add("doc001"); + var docId = await s_memory.ImportDocumentAsync("file1-Wikipedia-Carbon.txt", documentId: "doc001"); + s_toDelete.Add(docId); + Console.WriteLine($"- Document Id: {docId}"); } // Extract memory from images (OCR required) @@ -103,8 +105,9 @@ private static async Task StoreImage() if (!ImageSupportDemoEnabled) { return; } Console.WriteLine("Uploading Image file with a news about a conference sponsored by Microsoft"); - await s_memory.ImportDocumentAsync(new Document("img001").AddFiles(new[] { "file6-ANWC-image.jpg" })); - s_toDelete.Add("img001"); + var docId = await s_memory.ImportDocumentAsync(new Document("img001").AddFiles(new[] { "file6-ANWC-image.jpg" })); + s_toDelete.Add(docId); + Console.WriteLine($"- Document Id: {docId}"); } // Uploading multiple files and adding a user tag, checking if the document already exists @@ -113,9 +116,11 @@ private static async Task StoreMultipleFiles() if (!await s_memory.IsDocumentReadyAsync(documentId: "doc002")) { Console.WriteLine("Uploading a text file, a Word doc, and a PDF about Kernel Memory"); - await s_memory.ImportDocumentAsync(new Document("doc002") + var docId = await s_memory.ImportDocumentAsync(new Document("doc002") .AddFiles(new[] { "file2-Wikipedia-Moon.txt", "file3-lorem-ipsum.docx", "file4-KM-Readme.pdf" }) .AddTag("user", "Blake")); + s_toDelete.Add(docId); + Console.WriteLine($"- Document Id: {docId}"); } else { @@ -131,13 +136,15 @@ private static async Task StoreFileWithMultipleTags() if (!await s_memory.IsDocumentReadyAsync(documentId: "doc003")) { Console.WriteLine("Uploading a PDF with a news about NASA and Orion"); - await s_memory.ImportDocumentAsync(new Document("doc003") + var docId = await s_memory.ImportDocumentAsync(new Document("doc003") .AddFile("file5-NASA-news.pdf") .AddTag("user", "Taylor") .AddTag("collection", "meetings") .AddTag("collection", "NASA") .AddTag("collection", "space") .AddTag("type", "news")); + s_toDelete.Add(docId); + Console.WriteLine($"- Document Id: {docId}"); } else { @@ -153,7 +160,9 @@ private static async Task StoreWebPage() if (!await s_memory.IsDocumentReadyAsync("webPage1")) { Console.WriteLine("Uploading https://raw.githubusercontent.com/microsoft/kernel-memory/main/README.md"); - await s_memory.ImportWebPageAsync("https://raw.githubusercontent.com/microsoft/kernel-memory/main/README.md", documentId: "webPage1"); + var docId = await s_memory.ImportWebPageAsync("https://raw.githubusercontent.com/microsoft/kernel-memory/main/README.md", documentId: "webPage1"); + s_toDelete.Add(docId); + Console.WriteLine($"- Document Id: {docId}"); } else { @@ -169,7 +178,9 @@ private static async Task StoreHTMLFile() if (!await s_memory.IsDocumentReadyAsync(documentId: "htmlDoc001")) { Console.WriteLine("Uploading a HTML file about Apache Submarine project"); - await s_memory.ImportDocumentAsync(new Document("htmlDoc001").AddFile("file7-submarine.html").AddTag("user", "Ela")); + var docId = await s_memory.ImportDocumentAsync(new Document("htmlDoc001").AddFile("file7-submarine.html").AddTag("user", "Ela")); + s_toDelete.Add(docId); + Console.WriteLine($"- Document Id: {docId}"); } else { @@ -185,9 +196,11 @@ private static async Task StoreWithCustomPipeline() if (!await s_memory.IsDocumentReadyAsync("webPage2")) { Console.WriteLine("Uploading https://raw.githubusercontent.com/microsoft/kernel-memory/main/docs/security/security-filters.md"); - await s_memory.ImportWebPageAsync("https://raw.githubusercontent.com/microsoft/kernel-memory/main/docs/security/security-filters.md", + var docId = await s_memory.ImportWebPageAsync("https://raw.githubusercontent.com/microsoft/kernel-memory/main/docs/security/security-filters.md", documentId: "webPage2", steps: Constants.PipelineWithoutSummary); + s_toDelete.Add(docId); + Console.WriteLine($"- Document Id: {docId}"); } else { @@ -203,7 +216,9 @@ private static async Task StoreExcel() if (!await s_memory.IsDocumentReadyAsync(documentId: "xls01")) { Console.WriteLine("Uploading Excel file with some empty cells"); - await s_memory.ImportDocumentAsync(new Document("xls01").AddFiles(new[] { "file8-data.xlsx" })); + var docId = await s_memory.ImportDocumentAsync(new Document("xls01").AddFiles(new[] { "file8-data.xlsx" })); + s_toDelete.Add(docId); + Console.WriteLine($"- Document Id: {docId}"); } else { @@ -219,7 +234,9 @@ private static async Task StoreJson() if (!await s_memory.IsDocumentReadyAsync(documentId: "json01")) { Console.WriteLine("Uploading JSON file"); - await s_memory.ImportDocumentAsync(new Document("json01").AddFiles(new[] { "file9-settings.json" })); + var docId = await s_memory.ImportDocumentAsync(new Document("json01").AddFiles(new[] { "file9-settings.json" })); + s_toDelete.Add(docId); + Console.WriteLine($"- Document Id: {docId}"); } else { diff --git a/examples/002-dotnet-Serverless/Program.cs b/examples/002-dotnet-Serverless/Program.cs index 9f45a4cd4..af0e8b01d 100644 --- a/examples/002-dotnet-Serverless/Program.cs +++ b/examples/002-dotnet-Serverless/Program.cs @@ -22,10 +22,11 @@ public static class Program private static readonly List s_toDelete = new(); // Change this to True and configure Azure Document Intelligence to test OCR and support for images - private const bool imageSupportDemoEnabled = false; + private const bool ImageSupportDemoEnabled = true; public static async Task Main() { + var memoryConfiguration = new KernelMemoryConfig(); var openAIConfig = new OpenAIConfig(); var azureOpenAITextConfig = new AzureOpenAIConfig(); var azureOpenAIEmbeddingConfig = new AzureOpenAIConfig(); @@ -39,6 +40,7 @@ public static async Task Main() .AddJsonFile("appsettings.json") .AddJsonFile("appsettings.Development.json", optional: true) .Build() + .BindSection("KernelMemory", memoryConfiguration) .BindSection("KernelMemory:Services:OpenAI", openAIConfig) .BindSection("KernelMemory:Services:AzureOpenAIText", azureOpenAITextConfig) .BindSection("KernelMemory:Services:AzureOpenAIEmbedding", azureOpenAIEmbeddingConfig) @@ -49,6 +51,7 @@ public static async Task Main() .BindSection("KernelMemory:Retrieval:SearchClient", searchClientConfig); s_memory = new KernelMemoryBuilder() + .AddSingleton(memoryConfiguration) // .WithOpenAIDefaults(Environment.GetEnvironmentVariable("OPENAI_API_KEY")) // .WithOpenAI(openAIConfig) .WithAzureOpenAITextGeneration(azureOpenAITextConfig, new DefaultGPTTokenizer()) @@ -97,6 +100,8 @@ public static async Task Main() // ======================= await DeleteMemories(); + + Console.WriteLine("\n# DONE"); } // ======================= @@ -111,6 +116,7 @@ private static async Task StoreText() "in a system's rest frame, where the two quantities differ only by a multiplicative " + "constant and the units of measurement. The principle is described by the physicist " + "Albert Einstein's formula: E = m*c^2"); + Console.WriteLine($"- Document Id: {docId}"); s_toDelete.Add(docId); } @@ -118,18 +124,20 @@ private static async Task StoreText() private static async Task StoreFile() { Console.WriteLine("Uploading article file about Carbon"); - await s_memory.ImportDocumentAsync("file1-Wikipedia-Carbon.txt", documentId: "doc001"); - s_toDelete.Add("doc001"); + var docId = await s_memory.ImportDocumentAsync("file1-Wikipedia-Carbon.txt", documentId: "doc001"); + s_toDelete.Add(docId); + Console.WriteLine($"- Document Id: {docId}"); } // Extract memory from images (OCR required) private static async Task StoreImage() { - if (!imageSupportDemoEnabled) { return; } + if (!ImageSupportDemoEnabled) { return; } Console.WriteLine("Uploading Image file with a news about a conference sponsored by Microsoft"); - await s_memory.ImportDocumentAsync(new Document("img001").AddFiles(new[] { "file6-ANWC-image.jpg" })); - s_toDelete.Add("img001"); + var docId = await s_memory.ImportDocumentAsync(new Document("img001").AddFiles(new[] { "file6-ANWC-image.jpg" })); + s_toDelete.Add(docId); + Console.WriteLine($"- Document Id: {docId}"); } // Uploading multiple files and adding a user tag, checking if the document already exists @@ -138,9 +146,11 @@ private static async Task StoreMultipleFiles() if (!await s_memory.IsDocumentReadyAsync(documentId: "doc002")) { Console.WriteLine("Uploading a text file, a Word doc, and a PDF about Kernel Memory"); - await s_memory.ImportDocumentAsync(new Document("doc002") + var docId = await s_memory.ImportDocumentAsync(new Document("doc002") .AddFiles(new[] { "file2-Wikipedia-Moon.txt", "file3-lorem-ipsum.docx", "file4-KM-Readme.pdf" }) .AddTag("user", "Blake")); + s_toDelete.Add(docId); + Console.WriteLine($"- Document Id: {docId}"); } else { @@ -156,13 +166,15 @@ private static async Task StoreFileWithMultipleTags() if (!await s_memory.IsDocumentReadyAsync(documentId: "doc003")) { Console.WriteLine("Uploading a PDF with a news about NASA and Orion"); - await s_memory.ImportDocumentAsync(new Document("doc003") + var docId = await s_memory.ImportDocumentAsync(new Document("doc003") .AddFile("file5-NASA-news.pdf") .AddTag("user", "Taylor") .AddTag("collection", "meetings") .AddTag("collection", "NASA") .AddTag("collection", "space") .AddTag("type", "news")); + s_toDelete.Add(docId); + Console.WriteLine($"- Document Id: {docId}"); } else { @@ -178,7 +190,9 @@ private static async Task StoreWebPage() if (!await s_memory.IsDocumentReadyAsync("webPage1")) { Console.WriteLine("Uploading https://raw.githubusercontent.com/microsoft/kernel-memory/main/README.md"); - await s_memory.ImportWebPageAsync("https://raw.githubusercontent.com/microsoft/kernel-memory/main/README.md", documentId: "webPage1"); + var docId = await s_memory.ImportWebPageAsync("https://raw.githubusercontent.com/microsoft/kernel-memory/main/README.md", documentId: "webPage1"); + s_toDelete.Add(docId); + Console.WriteLine($"- Document Id: {docId}"); } else { @@ -194,7 +208,9 @@ private static async Task StoreHTMLFile() if (!await s_memory.IsDocumentReadyAsync(documentId: "htmlDoc001")) { Console.WriteLine("Uploading a HTML file about Apache Submarine project"); - await s_memory.ImportDocumentAsync(new Document("htmlDoc001").AddFile("file7-submarine.html").AddTag("user", "Ela")); + var docId = await s_memory.ImportDocumentAsync(new Document("htmlDoc001").AddFile("file7-submarine.html").AddTag("user", "Ela")); + s_toDelete.Add(docId); + Console.WriteLine($"- Document Id: {docId}"); } else { @@ -210,9 +226,11 @@ private static async Task StoreWithCustomPipeline() if (!await s_memory.IsDocumentReadyAsync("webPage2")) { Console.WriteLine("Uploading https://raw.githubusercontent.com/microsoft/kernel-memory/main/docs/security/security-filters.md"); - await s_memory.ImportWebPageAsync("https://raw.githubusercontent.com/microsoft/kernel-memory/main/docs/security/security-filters.md", + var docId = await s_memory.ImportWebPageAsync("https://raw.githubusercontent.com/microsoft/kernel-memory/main/docs/security/security-filters.md", documentId: "webPage2", steps: Constants.PipelineWithoutSummary); + s_toDelete.Add(docId); + Console.WriteLine($"- Document Id: {docId}"); } else { @@ -228,7 +246,9 @@ private static async Task StoreExcel() if (!await s_memory.IsDocumentReadyAsync(documentId: "xls01")) { Console.WriteLine("Uploading Excel file with some empty cells"); - await s_memory.ImportDocumentAsync(new Document("xls01").AddFiles(new[] { "file8-data.xlsx" })); + var docId = await s_memory.ImportDocumentAsync(new Document("xls01").AddFiles(new[] { "file8-data.xlsx" })); + s_toDelete.Add(docId); + Console.WriteLine($"- Document Id: {docId}"); } else { @@ -244,7 +264,9 @@ private static async Task StoreJson() if (!await s_memory.IsDocumentReadyAsync(documentId: "json01")) { Console.WriteLine("Uploading JSON file"); - await s_memory.ImportDocumentAsync(new Document("json01").AddFiles(new[] { "file9-settings.json" })); + var docId = await s_memory.ImportDocumentAsync(new Document("json01").AddFiles(new[] { "file9-settings.json" })); + s_toDelete.Add(docId); + Console.WriteLine($"- Document Id: {docId}"); } else { @@ -339,7 +361,7 @@ private static async Task AskQuestionAboutImageContent() var answer = await s_memory.AskAsync(question, minRelevance: 0.76); - Console.WriteLine(imageSupportDemoEnabled + Console.WriteLine(ImageSupportDemoEnabled ? $"\nAnswer: {answer.Result}\n\n Sources:\n" : $"\nAnswer (none expected): {answer.Result}\n\n Sources:\n"); diff --git a/examples/002-dotnet-Serverless/appsettings.json b/examples/002-dotnet-Serverless/appsettings.json index be3382377..72a4eeec2 100644 --- a/examples/002-dotnet-Serverless/appsettings.json +++ b/examples/002-dotnet-Serverless/appsettings.json @@ -56,7 +56,10 @@ // using the env vars AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET. "Auth": "AzureIdentity", "Endpoint": "https://<...>", - "APIKey": "" + "APIKey": "", + // Hybrid search is not enabled by default. Note that when using hybrid search + // relevance scores are different, usually lower, than when using just vector search + "UseHybridSearch": false }, "OpenAI": { // Name of the model used to generate text (text completion or chat completion) @@ -91,6 +94,11 @@ // "Seed": 1337, } }, + "DataIngestion": { + // How many memory DB records to insert at once when extracting memories from + // uploaded documents (used only if the Memory Db supports batching). + "MemoryDbUpsertBatchSize": 100 + }, "Retrieval": { "SearchClient": { // Maximum number of tokens accepted by the LLM used to generate answers.