Skip to content

Commit

Permalink
Update examples
Browse files Browse the repository at this point in the history
  • Loading branch information
@dluc
dluc committed May 11, 2024
1 parent bde65d7 commit 79c5a76
Show file tree
Hide file tree
Showing 3 changed files with 73 additions and 26 deletions.
39 changes: 28 additions & 11 deletions examples/001-dotnet-WebClient/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,15 +86,17 @@ private static async Task StoreText()
"in a system's rest frame, where the two quantities differ only by a multiplicative " +
"constant and the units of measurement. The principle is described by the physicist " +
"Albert Einstein's formula: E = m*c^2");
Console.WriteLine($"- Document Id: {docId}");
s_toDelete.Add(docId);
}

// Simple file upload, with document ID
private static async Task StoreFile()
{
Console.WriteLine("Uploading article file about Carbon");
await s_memory.ImportDocumentAsync("file1-Wikipedia-Carbon.txt", documentId: "doc001");
s_toDelete.Add("doc001");
var docId = await s_memory.ImportDocumentAsync("file1-Wikipedia-Carbon.txt", documentId: "doc001");
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}

// Extract memory from images (OCR required)
Expand All @@ -103,8 +105,9 @@ private static async Task StoreImage()
if (!ImageSupportDemoEnabled) { return; }

Console.WriteLine("Uploading Image file with a news about a conference sponsored by Microsoft");
await s_memory.ImportDocumentAsync(new Document("img001").AddFiles(new[] { "file6-ANWC-image.jpg" }));
s_toDelete.Add("img001");
var docId = await s_memory.ImportDocumentAsync(new Document("img001").AddFiles(new[] { "file6-ANWC-image.jpg" }));
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}

// Uploading multiple files and adding a user tag, checking if the document already exists
Expand All @@ -113,9 +116,11 @@ private static async Task StoreMultipleFiles()
if (!await s_memory.IsDocumentReadyAsync(documentId: "doc002"))
{
Console.WriteLine("Uploading a text file, a Word doc, and a PDF about Kernel Memory");
await s_memory.ImportDocumentAsync(new Document("doc002")
var docId = await s_memory.ImportDocumentAsync(new Document("doc002")
.AddFiles(new[] { "file2-Wikipedia-Moon.txt", "file3-lorem-ipsum.docx", "file4-KM-Readme.pdf" })
.AddTag("user", "Blake"));
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}
else
{
Expand All @@ -131,13 +136,15 @@ private static async Task StoreFileWithMultipleTags()
if (!await s_memory.IsDocumentReadyAsync(documentId: "doc003"))
{
Console.WriteLine("Uploading a PDF with a news about NASA and Orion");
await s_memory.ImportDocumentAsync(new Document("doc003")
var docId = await s_memory.ImportDocumentAsync(new Document("doc003")
.AddFile("file5-NASA-news.pdf")
.AddTag("user", "Taylor")
.AddTag("collection", "meetings")
.AddTag("collection", "NASA")
.AddTag("collection", "space")
.AddTag("type", "news"));
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}
else
{
Expand All @@ -153,7 +160,9 @@ private static async Task StoreWebPage()
if (!await s_memory.IsDocumentReadyAsync("webPage1"))
{
Console.WriteLine("Uploading https://raw.githubusercontent.com/microsoft/kernel-memory/main/README.md");
await s_memory.ImportWebPageAsync("https://raw.githubusercontent.com/microsoft/kernel-memory/main/README.md", documentId: "webPage1");
var docId = await s_memory.ImportWebPageAsync("https://raw.githubusercontent.com/microsoft/kernel-memory/main/README.md", documentId: "webPage1");
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}
else
{
Expand All @@ -169,7 +178,9 @@ private static async Task StoreHTMLFile()
if (!await s_memory.IsDocumentReadyAsync(documentId: "htmlDoc001"))
{
Console.WriteLine("Uploading a HTML file about Apache Submarine project");
await s_memory.ImportDocumentAsync(new Document("htmlDoc001").AddFile("file7-submarine.html").AddTag("user", "Ela"));
var docId = await s_memory.ImportDocumentAsync(new Document("htmlDoc001").AddFile("file7-submarine.html").AddTag("user", "Ela"));
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}
else
{
Expand All @@ -185,9 +196,11 @@ private static async Task StoreWithCustomPipeline()
if (!await s_memory.IsDocumentReadyAsync("webPage2"))
{
Console.WriteLine("Uploading https://raw.githubusercontent.com/microsoft/kernel-memory/main/docs/security/security-filters.md");
await s_memory.ImportWebPageAsync("https://raw.githubusercontent.com/microsoft/kernel-memory/main/docs/security/security-filters.md",
var docId = await s_memory.ImportWebPageAsync("https://raw.githubusercontent.com/microsoft/kernel-memory/main/docs/security/security-filters.md",
documentId: "webPage2",
steps: Constants.PipelineWithoutSummary);
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}
else
{
Expand All @@ -203,7 +216,9 @@ private static async Task StoreExcel()
if (!await s_memory.IsDocumentReadyAsync(documentId: "xls01"))
{
Console.WriteLine("Uploading Excel file with some empty cells");
await s_memory.ImportDocumentAsync(new Document("xls01").AddFiles(new[] { "file8-data.xlsx" }));
var docId = await s_memory.ImportDocumentAsync(new Document("xls01").AddFiles(new[] { "file8-data.xlsx" }));
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}
else
{
Expand All @@ -219,7 +234,9 @@ private static async Task StoreJson()
if (!await s_memory.IsDocumentReadyAsync(documentId: "json01"))
{
Console.WriteLine("Uploading JSON file");
await s_memory.ImportDocumentAsync(new Document("json01").AddFiles(new[] { "file9-settings.json" }));
var docId = await s_memory.ImportDocumentAsync(new Document("json01").AddFiles(new[] { "file9-settings.json" }));
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}
else
{
Expand Down
50 changes: 36 additions & 14 deletions examples/002-dotnet-Serverless/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,11 @@ public static class Program
private static readonly List<string> s_toDelete = new();

// Change this to True and configure Azure Document Intelligence to test OCR and support for images
private const bool imageSupportDemoEnabled = false;
private const bool ImageSupportDemoEnabled = true;

public static async Task Main()
{
var memoryConfiguration = new KernelMemoryConfig();
var openAIConfig = new OpenAIConfig();
var azureOpenAITextConfig = new AzureOpenAIConfig();
var azureOpenAIEmbeddingConfig = new AzureOpenAIConfig();
Expand All @@ -39,6 +40,7 @@ public static async Task Main()
.AddJsonFile("appsettings.json")
.AddJsonFile("appsettings.Development.json", optional: true)
.Build()
.BindSection("KernelMemory", memoryConfiguration)
.BindSection("KernelMemory:Services:OpenAI", openAIConfig)
.BindSection("KernelMemory:Services:AzureOpenAIText", azureOpenAITextConfig)
.BindSection("KernelMemory:Services:AzureOpenAIEmbedding", azureOpenAIEmbeddingConfig)
Expand All @@ -49,6 +51,7 @@ public static async Task Main()
.BindSection("KernelMemory:Retrieval:SearchClient", searchClientConfig);

s_memory = new KernelMemoryBuilder()
.AddSingleton(memoryConfiguration)
// .WithOpenAIDefaults(Environment.GetEnvironmentVariable("OPENAI_API_KEY"))
// .WithOpenAI(openAIConfig)
.WithAzureOpenAITextGeneration(azureOpenAITextConfig, new DefaultGPTTokenizer())
Expand Down Expand Up @@ -97,6 +100,8 @@ public static async Task Main()
// =======================

await DeleteMemories();

Console.WriteLine("\n# DONE");
}

// =======================
Expand All @@ -111,25 +116,28 @@ private static async Task StoreText()
"in a system's rest frame, where the two quantities differ only by a multiplicative " +
"constant and the units of measurement. The principle is described by the physicist " +
"Albert Einstein's formula: E = m*c^2");
Console.WriteLine($"- Document Id: {docId}");
s_toDelete.Add(docId);
}

// Simple file upload, with document ID
private static async Task StoreFile()
{
Console.WriteLine("Uploading article file about Carbon");
await s_memory.ImportDocumentAsync("file1-Wikipedia-Carbon.txt", documentId: "doc001");
s_toDelete.Add("doc001");
var docId = await s_memory.ImportDocumentAsync("file1-Wikipedia-Carbon.txt", documentId: "doc001");
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}

// Extract memory from images (OCR required)
private static async Task StoreImage()
{
if (!imageSupportDemoEnabled) { return; }
if (!ImageSupportDemoEnabled) { return; }

Console.WriteLine("Uploading Image file with a news about a conference sponsored by Microsoft");
await s_memory.ImportDocumentAsync(new Document("img001").AddFiles(new[] { "file6-ANWC-image.jpg" }));
s_toDelete.Add("img001");
var docId = await s_memory.ImportDocumentAsync(new Document("img001").AddFiles(new[] { "file6-ANWC-image.jpg" }));
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}

// Uploading multiple files and adding a user tag, checking if the document already exists
Expand All @@ -138,9 +146,11 @@ private static async Task StoreMultipleFiles()
if (!await s_memory.IsDocumentReadyAsync(documentId: "doc002"))
{
Console.WriteLine("Uploading a text file, a Word doc, and a PDF about Kernel Memory");
await s_memory.ImportDocumentAsync(new Document("doc002")
var docId = await s_memory.ImportDocumentAsync(new Document("doc002")
.AddFiles(new[] { "file2-Wikipedia-Moon.txt", "file3-lorem-ipsum.docx", "file4-KM-Readme.pdf" })
.AddTag("user", "Blake"));
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}
else
{
Expand All @@ -156,13 +166,15 @@ private static async Task StoreFileWithMultipleTags()
if (!await s_memory.IsDocumentReadyAsync(documentId: "doc003"))
{
Console.WriteLine("Uploading a PDF with a news about NASA and Orion");
await s_memory.ImportDocumentAsync(new Document("doc003")
var docId = await s_memory.ImportDocumentAsync(new Document("doc003")
.AddFile("file5-NASA-news.pdf")
.AddTag("user", "Taylor")
.AddTag("collection", "meetings")
.AddTag("collection", "NASA")
.AddTag("collection", "space")
.AddTag("type", "news"));
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}
else
{
Expand All @@ -178,7 +190,9 @@ private static async Task StoreWebPage()
if (!await s_memory.IsDocumentReadyAsync("webPage1"))
{
Console.WriteLine("Uploading https://raw.githubusercontent.com/microsoft/kernel-memory/main/README.md");
await s_memory.ImportWebPageAsync("https://raw.githubusercontent.com/microsoft/kernel-memory/main/README.md", documentId: "webPage1");
var docId = await s_memory.ImportWebPageAsync("https://raw.githubusercontent.com/microsoft/kernel-memory/main/README.md", documentId: "webPage1");
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}
else
{
Expand All @@ -194,7 +208,9 @@ private static async Task StoreHTMLFile()
if (!await s_memory.IsDocumentReadyAsync(documentId: "htmlDoc001"))
{
Console.WriteLine("Uploading a HTML file about Apache Submarine project");
await s_memory.ImportDocumentAsync(new Document("htmlDoc001").AddFile("file7-submarine.html").AddTag("user", "Ela"));
var docId = await s_memory.ImportDocumentAsync(new Document("htmlDoc001").AddFile("file7-submarine.html").AddTag("user", "Ela"));
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}
else
{
Expand All @@ -210,9 +226,11 @@ private static async Task StoreWithCustomPipeline()
if (!await s_memory.IsDocumentReadyAsync("webPage2"))
{
Console.WriteLine("Uploading https://raw.githubusercontent.com/microsoft/kernel-memory/main/docs/security/security-filters.md");
await s_memory.ImportWebPageAsync("https://raw.githubusercontent.com/microsoft/kernel-memory/main/docs/security/security-filters.md",
var docId = await s_memory.ImportWebPageAsync("https://raw.githubusercontent.com/microsoft/kernel-memory/main/docs/security/security-filters.md",
documentId: "webPage2",
steps: Constants.PipelineWithoutSummary);
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}
else
{
Expand All @@ -228,7 +246,9 @@ private static async Task StoreExcel()
if (!await s_memory.IsDocumentReadyAsync(documentId: "xls01"))
{
Console.WriteLine("Uploading Excel file with some empty cells");
await s_memory.ImportDocumentAsync(new Document("xls01").AddFiles(new[] { "file8-data.xlsx" }));
var docId = await s_memory.ImportDocumentAsync(new Document("xls01").AddFiles(new[] { "file8-data.xlsx" }));
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}
else
{
Expand All @@ -244,7 +264,9 @@ private static async Task StoreJson()
if (!await s_memory.IsDocumentReadyAsync(documentId: "json01"))
{
Console.WriteLine("Uploading JSON file");
await s_memory.ImportDocumentAsync(new Document("json01").AddFiles(new[] { "file9-settings.json" }));
var docId = await s_memory.ImportDocumentAsync(new Document("json01").AddFiles(new[] { "file9-settings.json" }));
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}
else
{
Expand Down Expand Up @@ -339,7 +361,7 @@ private static async Task AskQuestionAboutImageContent()

var answer = await s_memory.AskAsync(question, minRelevance: 0.76);

Console.WriteLine(imageSupportDemoEnabled
Console.WriteLine(ImageSupportDemoEnabled
? $"\nAnswer: {answer.Result}\n\n Sources:\n"
: $"\nAnswer (none expected): {answer.Result}\n\n Sources:\n");

Expand Down
10 changes: 9 additions & 1 deletion examples/002-dotnet-Serverless/appsettings.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,10 @@
// using the env vars AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET.
"Auth": "AzureIdentity",
"Endpoint": "https://<...>",
"APIKey": ""
"APIKey": "",
// Hybrid search is not enabled by default. Note that when using hybrid search
// relevance scores are different, usually lower, than when using just vector search
"UseHybridSearch": false
},
"OpenAI": {
// Name of the model used to generate text (text completion or chat completion)
Expand Down Expand Up @@ -91,6 +94,11 @@
// "Seed": 1337,
}
},
"DataIngestion": {
// How many memory DB records to insert at once when extracting memories from
// uploaded documents (used only if the Memory Db supports batching).
"MemoryDbUpsertBatchSize": 100
},
"Retrieval": {
"SearchClient": {
// Maximum number of tokens accepted by the LLM used to generate answers.
Expand Down

0 comments on commit 79c5a76

Please sign in to comment.