Merge pull request #23 from cnblogs/upgrade-kernel-memory

feat: upgrade kernel memory

Author: ikesnowy

Diff for: src/KernelMemory.DashScope/DashScopeTextEmbeddingGenerator.cs

@@ -24,6 +24,12 @@ public int CountTokens(string text)
         return tokenizer?.CountTokens(text) ?? text.Length;
     }
 
+    /// <inheritdoc />
+    public IReadOnlyList<string> GetTokens(string text)
+    {
+        return tokenizer?.GetTokens(text) ?? [text];
+    }
+
     /// <inheritdoc />
     public async Task<Embedding> GenerateEmbeddingAsync(
         string text,

Diff for: src/KernelMemory.DashScope/DashScopeTextGenerator.cs

@@ -30,6 +30,12 @@ public int CountTokens(string text)
         return tokenizer?.CountTokens(text) ?? QWenTokenizer.CountTokensStatic(text);
     }
 
+    /// <inheritdoc />
+    public IReadOnlyList<string> GetTokens(string text)
+    {
+        return tokenizer?.GetTokens(text) ?? QWenTokenizer.GetTokensStatic(text);
+    }
+
     /// <inheritdoc />
     public async IAsyncEnumerable<string> GenerateTextAsync(
         string prompt,

Diff for: src/KernelMemory.DashScope/KernelMemory.DashScope.csproj

@@ -19,8 +19,8 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.DeepDev.TokenizerLib" Version="1.3.3" />
-    <PackageReference Include="Microsoft.KernelMemory.Abstractions" Version="0.65.240620.1" />
-    <PackageReference Include="Cnblogs.DashScope.Core" Version="0.2.2" />
+    <PackageReference Include="Microsoft.KernelMemory.Abstractions" Version="0.70.240803.1" />
+    <PackageReference Include="Cnblogs.DashScope.Core" Version="0.3.0" />
   </ItemGroup>
 
   <ItemGroup>

Diff for: src/KernelMemory.DashScope/LengthTokenizer.cs

@@ -12,4 +12,10 @@ public int CountTokens(string text)
     {
         return text.Length;
     }
+
+    /// <inheritdoc />
+    public IReadOnlyList<string> GetTokens(string text)
+    {
+        return text.Select(x => $"{x}").ToList();
+    }
 }

Diff for: src/KernelMemory.DashScope/QWenTokenizer.cs

@@ -13,7 +13,6 @@ public class QWenTokenizer : ITextTokenizer
         .Concat(Enumerable.Range(0, 205).Select(x => $"<|extra_{x}|>"))
         .Select((x, i) => new KeyValuePair<string, int>(x, 151643 + i))
         .ToDictionary();
-
     private static readonly ITokenizer Tokenizer = TokenizerBuilder.CreateTokenizer(
         DashScopeEmbeddedResource.ReadBpeFile(),
         SpecialTokens,
@@ -45,6 +44,12 @@ public int CountTokens(string text)
         return Tokenizer.Encode(text).Count;
     }
 
+    /// <inheritdoc />
+    public IReadOnlyList<string> GetTokens(string text)
+    {
+        return Tokenizer.Encode(text).Select(x => Tokenizer.Decode([x])).ToList();
+    }
+
     /// <summary>
     /// Count tokens.
     /// </summary>
@@ -54,4 +59,14 @@ public static int CountTokensStatic(string text)
     {
         return Tokenizer.Encode(text).Count;
     }
+
+    /// <summary>
+    /// Get tokens
+    /// </summary>
+    /// <param name="text">The text to tokenizers.</param>
+    /// <returns></returns>
+    public static IReadOnlyList<string> GetTokensStatic(string text)
+    {
+        return Tokenizer.Encode(text).Select(x => Tokenizer.Decode([x])).ToList();
+    }
 }

Diff for: src/SemanticKernel.DashScope/SemanticKernel.DashScope.csproj

@@ -19,9 +19,9 @@
 
   <ItemGroup>
     <FrameworkReference Include="Microsoft.AspNetCore.App"/>
-    <PackageReference Include="Cnblogs.DashScope.Core" Version="0.2.2" />
-    <PackageReference Include="JsonSchema.Net.Generation" Version="4.3.0.2" />
-    <PackageReference Include="Microsoft.SemanticKernel.Core" Version="1.15.1" />
+    <PackageReference Include="Cnblogs.DashScope.Core" Version="0.3.0" />
+    <PackageReference Include="JsonSchema.Net.Generation" Version="4.5.0" />
+    <PackageReference Include="Microsoft.SemanticKernel.Core" Version="1.16.2" />
   </ItemGroup>
 
   <ItemGroup>

Diff for: test/KernelMemory.DashScope.UnitTests/KernelMemory.DashScope.UnitTests.csproj

@@ -14,10 +14,10 @@
           <PrivateAssets>all</PrivateAssets>
           <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
         </PackageReference>
-        <PackageReference Include="Microsoft.KernelMemory.Core" Version="0.65.240620.1" />
+        <PackageReference Include="Microsoft.KernelMemory.Core" Version="0.70.240803.1" />
         <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.10.0" />
-        <PackageReference Include="xunit" Version="2.8.1" />
-        <PackageReference Include="xunit.runner.visualstudio" Version="2.8.1">
+        <PackageReference Include="xunit" Version="2.9.0" />
+        <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
           <PrivateAssets>all</PrivateAssets>
           <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
         </PackageReference>

Diff for: test/SemanticKernel.DashScope.UnitTest/SemanticKernel.DashScope.UnitTest.csproj

@@ -5,8 +5,8 @@
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.10.0" />
-    <PackageReference Include="xunit" Version="2.8.1" />
-    <PackageReference Include="xunit.runner.visualstudio" Version="2.8.1">
+    <PackageReference Include="xunit" Version="2.9.0" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>