Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update examples #474

Merged
merged 1 commit into from
May 11, 2024
Merged

Update examples #474

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 28 additions & 11 deletions examples/001-dotnet-WebClient/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,15 +86,17 @@ private static async Task StoreText()
"in a system's rest frame, where the two quantities differ only by a multiplicative " +
"constant and the units of measurement. The principle is described by the physicist " +
"Albert Einstein's formula: E = m*c^2");
Console.WriteLine($"- Document Id: {docId}");
s_toDelete.Add(docId);
}

// Simple file upload, with document ID
private static async Task StoreFile()
{
Console.WriteLine("Uploading article file about Carbon");
await s_memory.ImportDocumentAsync("file1-Wikipedia-Carbon.txt", documentId: "doc001");
s_toDelete.Add("doc001");
var docId = await s_memory.ImportDocumentAsync("file1-Wikipedia-Carbon.txt", documentId: "doc001");
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}

// Extract memory from images (OCR required)
Expand All @@ -103,8 +105,9 @@ private static async Task StoreImage()
if (!ImageSupportDemoEnabled) { return; }

Console.WriteLine("Uploading Image file with a news about a conference sponsored by Microsoft");
await s_memory.ImportDocumentAsync(new Document("img001").AddFiles(new[] { "file6-ANWC-image.jpg" }));
s_toDelete.Add("img001");
var docId = await s_memory.ImportDocumentAsync(new Document("img001").AddFiles(new[] { "file6-ANWC-image.jpg" }));
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}

// Uploading multiple files and adding a user tag, checking if the document already exists
Expand All @@ -113,9 +116,11 @@ private static async Task StoreMultipleFiles()
if (!await s_memory.IsDocumentReadyAsync(documentId: "doc002"))
{
Console.WriteLine("Uploading a text file, a Word doc, and a PDF about Kernel Memory");
await s_memory.ImportDocumentAsync(new Document("doc002")
var docId = await s_memory.ImportDocumentAsync(new Document("doc002")
.AddFiles(new[] { "file2-Wikipedia-Moon.txt", "file3-lorem-ipsum.docx", "file4-KM-Readme.pdf" })
.AddTag("user", "Blake"));
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}
else
{
Expand All @@ -131,13 +136,15 @@ private static async Task StoreFileWithMultipleTags()
if (!await s_memory.IsDocumentReadyAsync(documentId: "doc003"))
{
Console.WriteLine("Uploading a PDF with a news about NASA and Orion");
await s_memory.ImportDocumentAsync(new Document("doc003")
var docId = await s_memory.ImportDocumentAsync(new Document("doc003")
.AddFile("file5-NASA-news.pdf")
.AddTag("user", "Taylor")
.AddTag("collection", "meetings")
.AddTag("collection", "NASA")
.AddTag("collection", "space")
.AddTag("type", "news"));
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}
else
{
Expand All @@ -153,7 +160,9 @@ private static async Task StoreWebPage()
if (!await s_memory.IsDocumentReadyAsync("webPage1"))
{
Console.WriteLine("Uploading https://raw.githubusercontent.com/microsoft/kernel-memory/main/README.md");
await s_memory.ImportWebPageAsync("https://raw.githubusercontent.com/microsoft/kernel-memory/main/README.md", documentId: "webPage1");
var docId = await s_memory.ImportWebPageAsync("https://raw.githubusercontent.com/microsoft/kernel-memory/main/README.md", documentId: "webPage1");
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}
else
{
Expand All @@ -169,7 +178,9 @@ private static async Task StoreHTMLFile()
if (!await s_memory.IsDocumentReadyAsync(documentId: "htmlDoc001"))
{
Console.WriteLine("Uploading a HTML file about Apache Submarine project");
await s_memory.ImportDocumentAsync(new Document("htmlDoc001").AddFile("file7-submarine.html").AddTag("user", "Ela"));
var docId = await s_memory.ImportDocumentAsync(new Document("htmlDoc001").AddFile("file7-submarine.html").AddTag("user", "Ela"));
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}
else
{
Expand All @@ -185,9 +196,11 @@ private static async Task StoreWithCustomPipeline()
if (!await s_memory.IsDocumentReadyAsync("webPage2"))
{
Console.WriteLine("Uploading https://raw.githubusercontent.com/microsoft/kernel-memory/main/docs/security/security-filters.md");
await s_memory.ImportWebPageAsync("https://raw.githubusercontent.com/microsoft/kernel-memory/main/docs/security/security-filters.md",
var docId = await s_memory.ImportWebPageAsync("https://raw.githubusercontent.com/microsoft/kernel-memory/main/docs/security/security-filters.md",
documentId: "webPage2",
steps: Constants.PipelineWithoutSummary);
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}
else
{
Expand All @@ -203,7 +216,9 @@ private static async Task StoreExcel()
if (!await s_memory.IsDocumentReadyAsync(documentId: "xls01"))
{
Console.WriteLine("Uploading Excel file with some empty cells");
await s_memory.ImportDocumentAsync(new Document("xls01").AddFiles(new[] { "file8-data.xlsx" }));
var docId = await s_memory.ImportDocumentAsync(new Document("xls01").AddFiles(new[] { "file8-data.xlsx" }));
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}
else
{
Expand All @@ -219,7 +234,9 @@ private static async Task StoreJson()
if (!await s_memory.IsDocumentReadyAsync(documentId: "json01"))
{
Console.WriteLine("Uploading JSON file");
await s_memory.ImportDocumentAsync(new Document("json01").AddFiles(new[] { "file9-settings.json" }));
var docId = await s_memory.ImportDocumentAsync(new Document("json01").AddFiles(new[] { "file9-settings.json" }));
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}
else
{
Expand Down
50 changes: 36 additions & 14 deletions examples/002-dotnet-Serverless/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,11 @@ public static class Program
private static readonly List<string> s_toDelete = new();

// Change this to True and configure Azure Document Intelligence to test OCR and support for images
private const bool imageSupportDemoEnabled = false;
private const bool ImageSupportDemoEnabled = true;

public static async Task Main()
{
var memoryConfiguration = new KernelMemoryConfig();
var openAIConfig = new OpenAIConfig();
var azureOpenAITextConfig = new AzureOpenAIConfig();
var azureOpenAIEmbeddingConfig = new AzureOpenAIConfig();
Expand All @@ -39,6 +40,7 @@ public static async Task Main()
.AddJsonFile("appsettings.json")
.AddJsonFile("appsettings.Development.json", optional: true)
.Build()
.BindSection("KernelMemory", memoryConfiguration)
.BindSection("KernelMemory:Services:OpenAI", openAIConfig)
.BindSection("KernelMemory:Services:AzureOpenAIText", azureOpenAITextConfig)
.BindSection("KernelMemory:Services:AzureOpenAIEmbedding", azureOpenAIEmbeddingConfig)
Expand All @@ -49,6 +51,7 @@ public static async Task Main()
.BindSection("KernelMemory:Retrieval:SearchClient", searchClientConfig);

s_memory = new KernelMemoryBuilder()
.AddSingleton(memoryConfiguration)
// .WithOpenAIDefaults(Environment.GetEnvironmentVariable("OPENAI_API_KEY"))
// .WithOpenAI(openAIConfig)
.WithAzureOpenAITextGeneration(azureOpenAITextConfig, new DefaultGPTTokenizer())
Expand Down Expand Up @@ -97,6 +100,8 @@ public static async Task Main()
// =======================

await DeleteMemories();

Console.WriteLine("\n# DONE");
}

// =======================
Expand All @@ -111,25 +116,28 @@ private static async Task StoreText()
"in a system's rest frame, where the two quantities differ only by a multiplicative " +
"constant and the units of measurement. The principle is described by the physicist " +
"Albert Einstein's formula: E = m*c^2");
Console.WriteLine($"- Document Id: {docId}");
s_toDelete.Add(docId);
}

// Simple file upload, with document ID
private static async Task StoreFile()
{
Console.WriteLine("Uploading article file about Carbon");
await s_memory.ImportDocumentAsync("file1-Wikipedia-Carbon.txt", documentId: "doc001");
s_toDelete.Add("doc001");
var docId = await s_memory.ImportDocumentAsync("file1-Wikipedia-Carbon.txt", documentId: "doc001");
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}

// Extract memory from images (OCR required)
private static async Task StoreImage()
{
if (!imageSupportDemoEnabled) { return; }
if (!ImageSupportDemoEnabled) { return; }

Console.WriteLine("Uploading Image file with a news about a conference sponsored by Microsoft");
await s_memory.ImportDocumentAsync(new Document("img001").AddFiles(new[] { "file6-ANWC-image.jpg" }));
s_toDelete.Add("img001");
var docId = await s_memory.ImportDocumentAsync(new Document("img001").AddFiles(new[] { "file6-ANWC-image.jpg" }));
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}

// Uploading multiple files and adding a user tag, checking if the document already exists
Expand All @@ -138,9 +146,11 @@ private static async Task StoreMultipleFiles()
if (!await s_memory.IsDocumentReadyAsync(documentId: "doc002"))
{
Console.WriteLine("Uploading a text file, a Word doc, and a PDF about Kernel Memory");
await s_memory.ImportDocumentAsync(new Document("doc002")
var docId = await s_memory.ImportDocumentAsync(new Document("doc002")
.AddFiles(new[] { "file2-Wikipedia-Moon.txt", "file3-lorem-ipsum.docx", "file4-KM-Readme.pdf" })
.AddTag("user", "Blake"));
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}
else
{
Expand All @@ -156,13 +166,15 @@ private static async Task StoreFileWithMultipleTags()
if (!await s_memory.IsDocumentReadyAsync(documentId: "doc003"))
{
Console.WriteLine("Uploading a PDF with a news about NASA and Orion");
await s_memory.ImportDocumentAsync(new Document("doc003")
var docId = await s_memory.ImportDocumentAsync(new Document("doc003")
.AddFile("file5-NASA-news.pdf")
.AddTag("user", "Taylor")
.AddTag("collection", "meetings")
.AddTag("collection", "NASA")
.AddTag("collection", "space")
.AddTag("type", "news"));
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}
else
{
Expand All @@ -178,7 +190,9 @@ private static async Task StoreWebPage()
if (!await s_memory.IsDocumentReadyAsync("webPage1"))
{
Console.WriteLine("Uploading https://raw.githubusercontent.com/microsoft/kernel-memory/main/README.md");
await s_memory.ImportWebPageAsync("https://raw.githubusercontent.com/microsoft/kernel-memory/main/README.md", documentId: "webPage1");
var docId = await s_memory.ImportWebPageAsync("https://raw.githubusercontent.com/microsoft/kernel-memory/main/README.md", documentId: "webPage1");
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}
else
{
Expand All @@ -194,7 +208,9 @@ private static async Task StoreHTMLFile()
if (!await s_memory.IsDocumentReadyAsync(documentId: "htmlDoc001"))
{
Console.WriteLine("Uploading a HTML file about Apache Submarine project");
await s_memory.ImportDocumentAsync(new Document("htmlDoc001").AddFile("file7-submarine.html").AddTag("user", "Ela"));
var docId = await s_memory.ImportDocumentAsync(new Document("htmlDoc001").AddFile("file7-submarine.html").AddTag("user", "Ela"));
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}
else
{
Expand All @@ -210,9 +226,11 @@ private static async Task StoreWithCustomPipeline()
if (!await s_memory.IsDocumentReadyAsync("webPage2"))
{
Console.WriteLine("Uploading https://raw.githubusercontent.com/microsoft/kernel-memory/main/docs/security/security-filters.md");
await s_memory.ImportWebPageAsync("https://raw.githubusercontent.com/microsoft/kernel-memory/main/docs/security/security-filters.md",
var docId = await s_memory.ImportWebPageAsync("https://raw.githubusercontent.com/microsoft/kernel-memory/main/docs/security/security-filters.md",
documentId: "webPage2",
steps: Constants.PipelineWithoutSummary);
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}
else
{
Expand All @@ -228,7 +246,9 @@ private static async Task StoreExcel()
if (!await s_memory.IsDocumentReadyAsync(documentId: "xls01"))
{
Console.WriteLine("Uploading Excel file with some empty cells");
await s_memory.ImportDocumentAsync(new Document("xls01").AddFiles(new[] { "file8-data.xlsx" }));
var docId = await s_memory.ImportDocumentAsync(new Document("xls01").AddFiles(new[] { "file8-data.xlsx" }));
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}
else
{
Expand All @@ -244,7 +264,9 @@ private static async Task StoreJson()
if (!await s_memory.IsDocumentReadyAsync(documentId: "json01"))
{
Console.WriteLine("Uploading JSON file");
await s_memory.ImportDocumentAsync(new Document("json01").AddFiles(new[] { "file9-settings.json" }));
var docId = await s_memory.ImportDocumentAsync(new Document("json01").AddFiles(new[] { "file9-settings.json" }));
s_toDelete.Add(docId);
Console.WriteLine($"- Document Id: {docId}");
}
else
{
Expand Down Expand Up @@ -339,7 +361,7 @@ private static async Task AskQuestionAboutImageContent()

var answer = await s_memory.AskAsync(question, minRelevance: 0.76);

Console.WriteLine(imageSupportDemoEnabled
Console.WriteLine(ImageSupportDemoEnabled
? $"\nAnswer: {answer.Result}\n\n Sources:\n"
: $"\nAnswer (none expected): {answer.Result}\n\n Sources:\n");

Expand Down
10 changes: 9 additions & 1 deletion examples/002-dotnet-Serverless/appsettings.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,10 @@
// using the env vars AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET.
"Auth": "AzureIdentity",
"Endpoint": "https://<...>",
"APIKey": ""
"APIKey": "",
// Hybrid search is not enabled by default. Note that when using hybrid search
// relevance scores are different, usually lower, than when using just vector search
"UseHybridSearch": false
},
"OpenAI": {
// Name of the model used to generate text (text completion or chat completion)
Expand Down Expand Up @@ -91,6 +94,11 @@
// "Seed": 1337,
}
},
"DataIngestion": {
// How many memory DB records to insert at once when extracting memories from
// uploaded documents (used only if the Memory Db supports batching).
"MemoryDbUpsertBatchSize": 100
},
"Retrieval": {
"SearchClient": {
// Maximum number of tokens accepted by the LLM used to generate answers.
Expand Down
Loading