Workaround for HuggingFace streaming API

microsoft · May 14, 2024 · 48c0933 · 48c0933
1 parent d3d6fe4
commit 48c0933
Show file tree

Hide file tree

Showing 5 changed files with 89 additions and 0 deletions.
diff --git a/KernelMemory.sln b/KernelMemory.sln
@@ -270,6 +270,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Discord", "extensions\Disco
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "301-discord-test-application", "examples\301-discord-test-application\301-discord-test-application.csproj", "{FAE4C6B8-38B2-43E7-8881-99693C9CEDC6}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "209-dotnet-HuggingFace", "examples\209-dotnet-HuggingFace\209-dotnet-HuggingFace.csproj", "{C13B27F4-1415-45CF-945C-CC49EA421EFC}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -503,6 +505,9 @@ Global
 		{FAE4C6B8-38B2-43E7-8881-99693C9CEDC6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{FAE4C6B8-38B2-43E7-8881-99693C9CEDC6}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{FAE4C6B8-38B2-43E7-8881-99693C9CEDC6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C13B27F4-1415-45CF-945C-CC49EA421EFC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C13B27F4-1415-45CF-945C-CC49EA421EFC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C13B27F4-1415-45CF-945C-CC49EA421EFC}.Release|Any CPU.ActiveCfg = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -584,6 +589,7 @@ Global
 		{C5E6B28C-F54D-423D-954D-A9EAEFB89732} = {3C17F42B-CFC8-4900-8CFB-88936311E919}
 		{43877864-6AE8-4B03-BEDA-6B6FA8BB1D8B} = {155DA079-E267-49AF-973A-D1D44681970F}
 		{FAE4C6B8-38B2-43E7-8881-99693C9CEDC6} = {0A43C65C-6007-4BB4-B3FE-8D439FC91841}
+		{C13B27F4-1415-45CF-945C-CC49EA421EFC} = {0A43C65C-6007-4BB4-B3FE-8D439FC91841}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {CC136C62-115C-41D1-B414-F9473EFF6EA8}

diff --git a/examples/209-dotnet-HuggingFace/209-dotnet-HuggingFace.csproj b/examples/209-dotnet-HuggingFace/209-dotnet-HuggingFace.csproj
@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+    <PropertyGroup>
+        <TargetFramework>net8.0</TargetFramework>
+        <OutputType>Exe</OutputType>
+        <ImplicitUsings>enable</ImplicitUsings>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <ProjectReference Include="..\..\service\Core\Core.csproj" />
+    </ItemGroup>
+
+</Project>
diff --git a/examples/209-dotnet-HuggingFace/Program.cs b/examples/209-dotnet-HuggingFace/Program.cs
@@ -0,0 +1,44 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+// ReSharper disable InconsistentNaming
+
+using Microsoft.KernelMemory;
+
+internal static class Program
+{
+    internal static async Task Main()
+    {
+        // Using HuggingFace text generation
+        var huggingFaceConfig = new OpenAIConfig
+        {
+            Endpoint = "https://api-inference.huggingface.co/models/NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO/v1",
+            TextModel = "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO",
+            TextModelMaxTokenTotal = 4096,
+            TextGenerationType = OpenAIConfig.TextGenerationTypes.Chat,
+            APIKey = Environment.GetEnvironmentVariable("HF_API_KEY")!
+        };
+
+        // Using OpenAI for embeddings
+        var openAIEmbeddingConfig = new OpenAIConfig
+        {
+            EmbeddingModel = "text-embedding-ada-002",
+            EmbeddingModelMaxTokenTotal = 8191,
+            APIKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY")!
+        };
+
+        var memory = new KernelMemoryBuilder()
+            // IMPORTANT: TopP = 0.01 is required by HuggingFace API
+            .WithSearchClientConfig(new SearchClientConfig { TopP = 0.01, AnswerTokens = 100 })
+            .WithOpenAITextGeneration(huggingFaceConfig) // Hugging Face
+            .WithOpenAITextEmbeddingGeneration(openAIEmbeddingConfig) // OpenAI
+            .Build();
+
+        // Import some text - This will use OpenAI embeddings
+        await memory.ImportTextAsync("Today is December 12th");
+
+        // Generate an answer - This uses OpenAI for embeddings and finding relevant data, and Hugging Face to generate an answer
+        var answer = await memory.AskAsync("How many days to Christmas?");
+        Console.WriteLine(answer.Question);
+        Console.WriteLine(answer.Result);
+    }
+}
diff --git a/examples/209-dotnet-HuggingFace/appsettings.json b/examples/209-dotnet-HuggingFace/appsettings.json
@@ -0,0 +1,21 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Trace",
+      "Microsoft.AspNetCore": "Trace"
+    },
+    "Console": {
+      "LogToStandardErrorThreshold": "Critical",
+      "FormatterName": "simple",
+      "FormatterOptions": {
+        "TimestampFormat": "[HH:mm:ss.fff] ",
+        "SingleLine": true,
+        "UseUtcTimestamp": false,
+        "IncludeScopes": false,
+        "JsonWriterOptions": {
+          "Indented": true
+        }
+      }
+    }
+  }
+}
diff --git a/extensions/OpenAI/OpenAITextGenerator.cs b/extensions/OpenAI/OpenAITextGenerator.cs
@@ -177,9 +177,14 @@ public async IAsyncEnumerable<string> GenerateTextAsync(
             openaiOptions.Messages.Add(new ChatRequestSystemMessage(prompt));
 
             StreamingResponse<StreamingChatCompletionsUpdate>? response = await this._client.GetChatCompletionsStreamingAsync(openaiOptions, cancellationToken).ConfigureAwait(false);
+            var totalTokens = 0;
             await foreach (StreamingChatCompletionsUpdate? update in response.EnumerateValues().WithCancellation(cancellationToken).ConfigureAwait(false))
             {
                 yield return update.ContentUpdate;
+
+                // Workaround for HuggingFace, to force a stop after max_tokens
+                totalTokens += this._textTokenizer!.CountTokens(update.ContentUpdate);
+                if (totalTokens >= options.MaxTokens) { break; }
             }
         }
     }