diff --git a/Directory.Packages.props b/Directory.Packages.props index 443aae022..d664cb79c 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -27,6 +27,7 @@ + diff --git a/KernelMemory.sln b/KernelMemory.sln index 47798b311..f54bab428 100644 --- a/KernelMemory.sln +++ b/KernelMemory.sln @@ -74,6 +74,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tools", "tools", "{CA49F1A1 tools\search.sh = tools\search.sh tools\run-qdrant.sh = tools\run-qdrant.sh tools\create-azure-webapp-publish-artifacts.sh = tools\create-azure-webapp-publish-artifacts.sh + tools\run-redis.sh = tools\run-redis.sh EndProjectSection EndProject @@ -240,6 +241,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "packages", "packages", "{16 EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redis", "extensions\Redis\Redis.csproj", "{EC434C8D-4811-4B16-8F04-5E8FAD5BE233}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "redis-tests", "service\tests\redis-tests\redis-tests.csproj", "{A8EB745F-5767-42CC-9E3A-692060720F35}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -299,6 +302,7 @@ Global {DD7ED79F-95D3-48AE-85AB-D7119F8530C2} = {5E7DD43D-B5E7-4827-B57D-447E5B428589} {0675D9B5-B3BB-4C9A-A1A5-11540E2ED715} = {5E7DD43D-B5E7-4827-B57D-447E5B428589} {EC434C8D-4811-4B16-8F04-5E8FAD5BE233} = {155DA079-E267-49AF-973A-D1D44681970F} + {A8EB745F-5767-42CC-9E3A-692060720F35} = {5E7DD43D-B5E7-4827-B57D-447E5B428589} EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {8A9FA587-7EBA-4D43-BE47-38D798B1C74C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU @@ -445,5 +449,8 @@ Global {EC434C8D-4811-4B16-8F04-5E8FAD5BE233}.Debug|Any CPU.Build.0 = Debug|Any CPU {EC434C8D-4811-4B16-8F04-5E8FAD5BE233}.Release|Any CPU.ActiveCfg = Release|Any CPU {EC434C8D-4811-4B16-8F04-5E8FAD5BE233}.Release|Any CPU.Build.0 = Release|Any CPU + {A8EB745F-5767-42CC-9E3A-692060720F35}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A8EB745F-5767-42CC-9E3A-692060720F35}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A8EB745F-5767-42CC-9E3A-692060720F35}.Release|Any CPU.ActiveCfg = Release|Any CPU EndGlobalSection EndGlobal diff --git a/extensions/Redis/DependencyInjection.cs b/extensions/Redis/DependencyInjection.cs new file mode 100644 index 000000000..cc7171b2e --- /dev/null +++ b/extensions/Redis/DependencyInjection.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.KernelMemory.MemoryStorage; +using Microsoft.KernelMemory.MemoryDb.Redis; +using StackExchange.Redis; + +#pragma warning disable IDE0130 // reduce number of "using" statements +// ReSharper disable once CheckNamespace - reduce number of "using" statements +namespace Microsoft.KernelMemory; + +/// +/// DI pipelines for Redis Memory. +/// +public static partial class KernelMemoryBuilderExtensions +{ + /// + /// Adds RedisMemory as a service. + /// + /// The kernel builder + /// The Redis connection string based on StackExchange.Redis' connection string + public static IKernelMemoryBuilder WithRedisMemoryDb( + this IKernelMemoryBuilder builder, + string connString) + { + builder.Services.AddRedisAsMemoryDb(new RedisConfig { ConnectionString = connString }); + return builder; + } + + /// + /// Adds RedisMemory as a service. + /// + /// The kernel builder + /// Redis configuration. + public static IKernelMemoryBuilder WithRedisMemoryDb( + this IKernelMemoryBuilder builder, + RedisConfig redisConfig) + { + builder.Services.AddRedisAsMemoryDb(redisConfig); + return builder; + } +} + +/// +/// setup Redis memory within the semantic kernel +/// +public static partial class DependencyInjection +{ + /// + /// Adds RedisMemory as a service. + /// + /// The services collection + /// Redis configuration. + public static IServiceCollection AddRedisAsMemoryDb( + this IServiceCollection services, + RedisConfig redisConfig) + { + return services + .AddSingleton(redisConfig) + .AddSingleton(_ => ConnectionMultiplexer.Connect(redisConfig.ConnectionString)) + .AddSingleton(); + } +} diff --git a/extensions/Redis/README.md b/extensions/Redis/README.md index 275447a20..30f35f04b 100644 --- a/extensions/Redis/README.md +++ b/extensions/Redis/README.md @@ -2,4 +2,28 @@ [![Discord](https://img.shields.io/discord/1063152441819942922?label=Discord&logo=discord&logoColor=white&color=d82679)](https://aka.ms/KMdiscord) -This project will contain the [Redis](https://redis.io) adapter allowing to use Kernel Memory with Redis. +## Notes about Redis Vector Search: + +Redis Vector search requires the use of +Redis' [Search and Query capabilities](https://redis.io/docs/interact/search-and-query/). + +This is available in: + +* [Redis Stack](https://redis.io/docs/about/about-stack/) +* [Azure Cache for Redis](https://azure.microsoft.com/en-us/products/cache) - Enterprise Tier only +* [Redis Cloud](https://app.redislabs.com/) +* [Redis Enterprise](https://redis.io/docs/about/redis-enterprise/) + +You can run Redis Stack locally in docker with the following command: + +```sh +docker run -p 8001:8001 -p 6379:6379 redis/redis-stack +``` + +## Configuring Tag Filters + +Using tag filters with Redis requires you to to pre-define which tag fields you want. You can +do so using the `RedisMemoryConfiguration.Tags` property (with the characters being the tag separators) +while creating the dependency-injection pipeline. It's important that you pick a separator that will +not appear in your data (otherwise your tags might over-match) + diff --git a/extensions/Redis/Redis.csproj b/extensions/Redis/Redis.csproj index 51058902c..285b0e852 100644 --- a/extensions/Redis/Redis.csproj +++ b/extensions/Redis/Redis.csproj @@ -5,7 +5,9 @@ LatestMajor Microsoft.KernelMemory.MemoryDb.Redis Microsoft.KernelMemory.MemoryDb.Redis - $(NoWarn);CA1724;CS1591; + $(NoWarn);CA1724;CS1591;CA1308;CA1859; + enable + enable @@ -14,10 +16,7 @@ - - - - + @@ -29,11 +28,11 @@ Microsoft.KernelMemory.MemoryDb.Redis Redis connector for Kernel Memory Redis connector for Microsoft Kernel Memory, to store and search memory using Redis vector search and other Redis features. - Memory, RAG, Kernel Memory, Redis, HNSW, AI, Artificial Intelligence, Embeddings, Vector DB, Vector Search, ETL + Redis Memory, RAG, Kernel Memory, Redis, HNSW, AI, Artificial Intelligence, Embeddings, Vector DB, Vector Search, ETL - + \ No newline at end of file diff --git a/extensions/Redis/RedisConfig.cs b/extensions/Redis/RedisConfig.cs index 6ae4e7c2f..21f55e03a 100644 --- a/extensions/Redis/RedisConfig.cs +++ b/extensions/Redis/RedisConfig.cs @@ -1,10 +1,54 @@ -// Copyright (c) Microsoft. All rights reserved. +// Copyright (c) Microsoft. All rights reserved. #pragma warning disable IDE0130 // reduce number of "using" statements // ReSharper disable once CheckNamespace - reduce number of "using" statements namespace Microsoft.KernelMemory; +/// +/// Lays out the tag fields that you want redis to index. +/// public class RedisConfig { - // TODO + /// + /// Connection string required to connect to Redis + /// + public string ConnectionString { get; set; } = string.Empty; + + /// + /// Gets the Prefix to use for prefix index names and all documents + /// inserted into Redis as part of Kernel Memory's operations. + /// + public string AppPrefix { get; } + + /// + /// The Collection of tags that you want to be able to search on. + /// The Key is the tag name, and the char is the separator that you + /// want Redis to use to separate your tag fields. The default separator + /// is ','. + /// + public Dictionary Tags { get; } = new() + { + { Constants.ReservedDocumentIdTag, '|' }, + { Constants.ReservedFileIdTag, '|' }, + { Constants.ReservedFilePartitionTag, '|' }, + { Constants.ReservedFileTypeTag, '|' }, + }; + + /// + /// Initializes an instance of RedisMemoryConfiguration. + /// + /// The prefix to use for the index name and all documents inserted into Redis. + /// The collection of tags you want to be able to search on. The key + public RedisConfig(string appPrefix = "km", Dictionary? tags = null) + { + this.AppPrefix = appPrefix; + + if (tags is not null) + { + foreach (var tag in tags) + { + this.Tags[tag.Key] = tag.Value; + } + } + } } diff --git a/extensions/Redis/RedisEmbeddingExtensions.cs b/extensions/Redis/RedisEmbeddingExtensions.cs new file mode 100644 index 000000000..800b2b139 --- /dev/null +++ b/extensions/Redis/RedisEmbeddingExtensions.cs @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.KernelMemory.MemoryDb.Redis; + +/// +/// Helper method for Embeddings. +/// +internal static class RedisEmbeddingExtensions +{ + public static byte[] VectorBlob(this Embedding embedding) => embedding.Data.ToArray().SelectMany(BitConverter.GetBytes).ToArray(); +} diff --git a/extensions/Redis/RedisException.cs b/extensions/Redis/RedisException.cs index 54b6582b4..a60dd4482 100644 --- a/extensions/Redis/RedisException.cs +++ b/extensions/Redis/RedisException.cs @@ -1,7 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System; - namespace Microsoft.KernelMemory.MemoryDb.Redis; public class RedisException : KernelMemoryException diff --git a/extensions/Redis/RedisMemory.cs b/extensions/Redis/RedisMemory.cs index b8d9bd1f9..cb2e8c9cc 100644 --- a/extensions/Redis/RedisMemory.cs +++ b/extensions/Redis/RedisMemory.cs @@ -1,82 +1,352 @@ -// Copyright (c) Microsoft. All rights reserved. +// Copyright (c) Microsoft. All rights reserved. -using System; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; +using System.Globalization; +using System.Runtime.CompilerServices; +using System.Text; +using System.Text.Json; +using System.Text.RegularExpressions; using Microsoft.Extensions.Logging; using Microsoft.KernelMemory.AI; using Microsoft.KernelMemory.Diagnostics; using Microsoft.KernelMemory.MemoryStorage; +using NRedisStack; +using NRedisStack.RedisStackCommands; +using NRedisStack.Search; +using NRedisStack.Search.Literals.Enums; +using StackExchange.Redis; namespace Microsoft.KernelMemory.MemoryDb.Redis; -/// -public class RedisMemory : IMemoryDb +/// +/// Implementation of an IMemoryDb using Redis. +/// +public sealed class RedisMemory : IMemoryDb { + private readonly IDatabase _db; + private readonly ISearchCommandsAsync _search; + private readonly RedisConfig _config; private readonly ITextEmbeddingGenerator _embeddingGenerator; - private readonly ILogger _log; + private readonly ILogger _logger; /// - /// Create new instance + /// Initializes the instance /// - /// Redis connector configuration - /// Text embedding generator - /// Application logger + /// + /// + /// + /// public RedisMemory( RedisConfig config, + IConnectionMultiplexer multiplexer, ITextEmbeddingGenerator embeddingGenerator, - ILogger? log = null) + ILogger? logger = null) { + this._config = config; this._embeddingGenerator = embeddingGenerator; + this._logger = logger ?? DefaultLogger.Instance; + this._search = multiplexer.GetDatabase().FT(); + this._db = multiplexer.GetDatabase(); + } + + /// + public async Task CreateIndexAsync(string index, int vectorSize, CancellationToken cancellationToken = default) + { + var normalizedIndexName = NormalizeIndexName(index, this._config.AppPrefix); + var schema = new Schema().AddVectorField(EmbeddingFieldName, Schema.VectorField.VectorAlgo.HNSW, new Dictionary() + { + { "TYPE", "FLOAT32" }, + { "DIM", vectorSize }, + { "DISTANCE_METRIC", "COSINE" } + }); + + var ftParams = new FTCreateParams().On(IndexDataType.HASH).Prefix($"{normalizedIndexName}:"); - if (this._embeddingGenerator == null) + foreach (var tag in this._config.Tags) { - throw new RedisException("Embedding generator not configured"); + var fieldName = tag.Key; + var separator = tag.Value ?? DefaultSeparator; + schema.AddTagField(fieldName, separator: separator.ToString()); } - this._log = log ?? DefaultLogger.Instance; + try + { + await this._search.CreateAsync(normalizedIndexName, ftParams, schema).ConfigureAwait(false); + } + catch (RedisServerException ex) + { + if (!ex.Message.Contains("Index already exists", StringComparison.OrdinalIgnoreCase)) + { + throw; + } + } } /// - public Task CreateIndexAsync(string index, int vectorSize, CancellationToken cancellationToken = default) + public async Task> GetIndexesAsync(CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + var result = await this._search._ListAsync().ConfigureAwait(false); + return result.Select(x => (string)x!); } /// - public Task> GetIndexesAsync(CancellationToken cancellationToken = default) + public async Task DeleteIndexAsync(string index, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + var normalizedIndexName = NormalizeIndexName(index, this._config.AppPrefix); + try + { + // we are explicitly dropping all records associated with the index here. + await this._search.DropIndexAsync(normalizedIndexName, dd: true).ConfigureAwait(false); + } + catch (RedisServerException exception) + { + if (!exception.Message.Equals("unknown index name", StringComparison.OrdinalIgnoreCase)) + { + throw; + } + } } /// - public Task DeleteIndexAsync(string index, CancellationToken cancellationToken = default) + public async Task UpsertAsync(string index, MemoryRecord record, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + var normalizedIndexName = NormalizeIndexName(index, this._config.AppPrefix); + var key = Key(normalizedIndexName, record.Id); + var fields = new List(); + fields.Add(new HashEntry(EmbeddingFieldName, record.Vector.VectorBlob())); + foreach (var item in record.Tags) + { + var isIndexed = this._config.Tags.TryGetValue(item.Key, out var c); + var separator = c ?? DefaultSeparator; + if (!isIndexed) + { + this._logger.LogWarning("Inserting un-indexed tag field: {Key}, will not be able to filter on it", item.Key); + } + + if (item.Value.Any(s => s is not null && s.Contains(separator.ToString(), StringComparison.InvariantCulture))) + { + this._logger.LogWarning("Inserting indexed tag field with the selected separator: '{Separator}' in it", separator); + } + + fields.Add(new HashEntry(item.Key, string.Join(separator, item.Value))); + } + + if (record.Payload.Count != 0) + { + fields.Add(new HashEntry(PayloadFieldName, JsonSerializer.Serialize(record.Payload))); // assumption: it's safe to serialize/deserialize the payload to/from JSON. + } + + await this._db.HashSetAsync(key, fields.ToArray()).ConfigureAwait(false); + + return record.Id; } /// - public Task UpsertAsync(string index, MemoryRecord record, CancellationToken cancellationToken = default) + public async IAsyncEnumerable<(MemoryRecord, double)> GetSimilarListAsync(string index, string text, ICollection? filters = null, double minRelevance = 0, int limit = 1, bool withEmbeddings = false, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + var normalizedIndexName = NormalizeIndexName(index, this._config.AppPrefix); + var embedding = await this._embeddingGenerator.GenerateEmbeddingAsync(text, cancellationToken).ConfigureAwait(false); + var blob = embedding.VectorBlob(); + var parameters = new Dictionary + { + { "blob", blob }, + { "limit", limit } + }; + + var sb = new StringBuilder(); + if (filters != null && filters.Any(x => x.Pairs.Any())) + { + foreach ((string key, string? value) in filters.SelectMany(x => x.Pairs)) + { + if (value is null) + { + this._logger.LogWarning("Attempted to perform null check on tag field. This behavior is not supported by Redis"); + } + + sb.Append(CultureInfo.InvariantCulture, $"@{key}:{{{value}}} "); + } + } + else + { + sb.Append('*'); + } + + sb.Append($"=>[KNN $limit @{EmbeddingFieldName} $blob]"); + + var query = new Query(sb.ToString()); + query.Params(parameters); + query.Limit(0, limit); + query.Dialect(2); + + var result = await this._search.SearchAsync(normalizedIndexName, query).ConfigureAwait(false); + foreach (var doc in result.Documents) + { + var next = this.FromDocument(doc, withEmbeddings); + if (1 - next.Item2 > minRelevance) + { + yield return next; + } + } } /// - public IAsyncEnumerable<(MemoryRecord, double)> GetSimilarListAsync(string index, string text, ICollection? filters = null, double minRelevance = 0, int limit = 1, bool withEmbeddings = false, CancellationToken cancellationToken = default) + public async IAsyncEnumerable GetListAsync(string index, ICollection? filters = null, int limit = 1, bool withEmbeddings = false, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + var normalizedIndexName = NormalizeIndexName(index, this._config.AppPrefix); + var sb = new StringBuilder(); + if (filters != null && filters.Any(x => x.Pairs.Any())) + { + foreach ((string key, string? value) in filters.SelectMany(x => x.Pairs)) + { + if (value is null) + { + this._logger.LogWarning("Attempted to perform null check on tag field. This behavior is not supported by Redis"); + } + + sb.Append(CultureInfo.InvariantCulture, $" @{key}:{{{EscapeTagField(value!)}}}"); + } + } + else + { + sb.Append('*'); + } + + var query = new Query(sb.ToString()); + query.Limit(0, limit); + var result = await this._search.SearchAsync(normalizedIndexName, query).ConfigureAwait(false); + foreach (var doc in result.Documents) + { + yield return this.FromDocument(doc, withEmbeddings).Item1; + } } /// - public IAsyncEnumerable GetListAsync(string index, ICollection? filters = null, int limit = 1, bool withEmbeddings = false, CancellationToken cancellationToken = default) + public Task DeleteAsync(string index, MemoryRecord record, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + var normalizedIndexName = NormalizeIndexName(index, this._config.AppPrefix); + var key = Key(normalizedIndexName, record.Id); + return this._db.KeyDeleteAsync(key); } - /// - public Task DeleteAsync(string index, MemoryRecord record, CancellationToken cancellationToken = default) + #region private ================================================================================ + + private const string EmbeddingFieldName = "embedding"; + private const string PayloadFieldName = "payload"; + private const char DefaultSeparator = ','; + private const string DistanceFieldName = $"__{EmbeddingFieldName}_score"; + + /// + /// Characters to escape when serializing a tag expression. + /// + private static readonly char[] s_tagEscapeChars = + { + ',', '.', '<', '>', '{', '}', '[', ']', '"', '\'', ':', ';', + '!', '@', '#', '$', '%', '^', '&', '*', '(', ')', '-', '+', '=', '~', '|', ' ', '/', + }; + + /// + /// Special chars to specifically replace within index names to keep + /// index names consistent with other connectors. + /// + private static readonly Regex s_replaceIndexNameCharsRegex = new(@"[\s|\\|/|.|_|:]"); + + /// + /// Use designated KM separator + /// + private const string KmSeparator = "-"; + + private (MemoryRecord, double) FromDocument(NRedisStack.Search.Document doc, bool withEmbedding) { - throw new NotImplementedException(); + double distance = 0; + var memoryRecord = new MemoryRecord(); + memoryRecord.Id = doc.Id.Split(":", 2)[1]; + foreach (var field in doc.GetProperties()) + { + if (field.Key == EmbeddingFieldName) + { + if (withEmbedding) + { + var floats = ByteArrayToFloatArray((byte[])field.Value!); + memoryRecord.Vector = new Embedding(floats); + } + } + else if (field.Key == PayloadFieldName) + { + var payload = JsonSerializer.Deserialize>(field.Value.ToString()); + memoryRecord.Payload = payload ?? new Dictionary(); + } + else if (field.Key == DistanceFieldName) + { + distance = (double)field.Value; + } + else + { + this._config.Tags.TryGetValue(field.Key, out var c); + var separator = c ?? DefaultSeparator; + var values = ((string)field.Value!).Split(separator); + memoryRecord.Tags.Add(new KeyValuePair>(field.Key, new List(values))); + } + } + + return (memoryRecord, distance); } + + /// + /// Normalizes the provided index name to maintain consistent looking + /// index names across connections. Naturally Redis's index names + /// are binary safe so this is purely for consistency. + /// + private static string NormalizeIndexName(string index, string? prefix = null) + { + if (string.IsNullOrWhiteSpace(index)) + { + index = Constants.DefaultIndex; + } + + var indexWithPrefix = !string.IsNullOrWhiteSpace(prefix) ? $"{prefix}-{index}" : index; + + indexWithPrefix = s_replaceIndexNameCharsRegex.Replace(indexWithPrefix.Trim().ToLowerInvariant(), KmSeparator); + + return indexWithPrefix; + } + + /// + /// Escapes a tag field string. + /// + /// the text toe escape. + /// The Escaped Text. + private static string EscapeTagField(string text) + { + var sb = new StringBuilder(); + foreach (var c in text) + { + if (s_tagEscapeChars.Contains(c)) + { + sb.Append('\\'); + } + + sb.Append(c); + } + + return sb.ToString(); + } + + private static RedisKey Key(string indexWithPrefix, string id) => $"{indexWithPrefix}:{id}"; + + private static float[] ByteArrayToFloatArray(byte[] bytes) + { + if (bytes.Length % 4 != 0) + { + throw new InvalidOperationException("Encountered an unbalanced array of bytes for float array conversion"); + } + + var res = new float[bytes.Length / 4]; + for (int i = 0; i < bytes.Length / 4; i++) + { + res[i] = BitConverter.ToSingle(bytes, i * 4); + } + + return res; + } + + #endregion } diff --git a/service/tests/redis-tests/MockEmbeddingGenerator.cs b/service/tests/redis-tests/MockEmbeddingGenerator.cs new file mode 100644 index 000000000..249ef58ae --- /dev/null +++ b/service/tests/redis-tests/MockEmbeddingGenerator.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.KernelMemory; +using Microsoft.KernelMemory.AI; + +namespace redis_tests; + +internal class MockEmbeddingGenerator : ITextEmbeddingGenerator +{ + private readonly Dictionary _embeddings = new(); + + internal void AddFakeEmbedding(string str, float[] floats) + { + this._embeddings.Add(str, floats); + } + + /// + public int CountTokens(string text) => 0; + + /// + public int MaxTokens => 0; + + /// + public Task GenerateEmbeddingAsync(string text, CancellationToken cancellationToken = default) => + Task.FromResult(new Embedding(this._embeddings[text])); +} diff --git a/service/tests/redis-tests/Program.cs b/service/tests/redis-tests/Program.cs new file mode 100644 index 000000000..5e3aeba85 --- /dev/null +++ b/service/tests/redis-tests/Program.cs @@ -0,0 +1,104 @@ +// See https://aka.ms/new-console-template for more information + +using Microsoft.KernelMemory; +using Microsoft.KernelMemory.MemoryDb.Redis; +using Microsoft.KernelMemory.MemoryStorage; +using StackExchange.Redis; +using redis_tests; + +var muxer = await ConnectionMultiplexer.ConnectAsync("localhost"); +var embeddingGenerator = new MockEmbeddingGenerator(); +var tags = new Dictionary() +{ + { "updated", '|' }, + { "type", '|' } +}; +var memory = new RedisMemory(new RedisConfig(tags: tags), muxer, embeddingGenerator); + +Console.WriteLine("===== DELETE INDEX ====="); + +await memory.DeleteIndexAsync("test"); + +Console.WriteLine("===== CREATE INDEX ====="); + +await memory.CreateIndexAsync("test", 5); + +Console.WriteLine("===== INSERT RECORD 1 ====="); + +const string Text = "test1"; +var embedding = new[] { 0f, 0, 1, 0, 1 }; +embeddingGenerator.AddFakeEmbedding(Text, embedding); + +var memoryRecord1 = new MemoryRecord +{ + Id = "memory 1", + Vector = embedding, + Tags = new TagCollection { { "updated", "no" }, { "type", "email" } }, + Payload = new Dictionary() +}; + +var id1 = await memory.UpsertAsync("test", memoryRecord1); +Console.WriteLine($"Insert 1: {id1} {memoryRecord1.Id}"); + +Console.WriteLine("===== INSERT RECORD 2 ====="); + +var memoryRecord2 = new MemoryRecord +{ + Id = "memory two", + Vector = new[] { 0f, 0, 1, 0, 1 }, + Tags = new TagCollection { { "type", "news" } }, + Payload = new Dictionary() +}; + +var id2 = await memory.UpsertAsync("test", memoryRecord2); +Console.WriteLine($"Insert 2: {id2} {memoryRecord2.Id}"); + +Console.WriteLine("===== UPDATE RECORD 2 ====="); +memoryRecord2.Tags.Add("updated", "yes"); +id2 = await memory.UpsertAsync("test", memoryRecord2); +Console.WriteLine($"Update 2: {id2} {memoryRecord2.Id}"); + +Console.WriteLine("===== SEARCH 1 ====="); + +var similarList = memory.GetSimilarListAsync("test", text: Text, + limit: 10, withEmbeddings: true); +await foreach ((MemoryRecord, double) record in similarList) +{ + Console.WriteLine(record.Item1.Id); + Console.WriteLine(" tags: " + record.Item1.Tags.Count); + Console.WriteLine(" size: " + record.Item1.Vector.Length); +} + +Console.WriteLine("===== SEARCH 2 ====="); + +similarList = memory.GetSimilarListAsync("test", text: Text, + limit: 10, withEmbeddings: true, filters: new List { MemoryFilters.ByTag("type", "email") }); +await foreach ((MemoryRecord, double) record in similarList) +{ + Console.WriteLine(record.Item1.Id); + Console.WriteLine(" type: " + record.Item1.Tags["type"].First()); +} + +Console.WriteLine("===== LIST ====="); + +var list = memory.GetListAsync("test", limit: 10, withEmbeddings: false); +await foreach (MemoryRecord record in list) +{ + Console.WriteLine(record.Id); + Console.WriteLine(" type: " + record.Tags["type"].First()); +} + +Console.WriteLine("===== DELETE ====="); + +await memory.DeleteAsync("test", new MemoryRecord { Id = "memory 1" }); + +Console.WriteLine("===== LIST AFTER DELETE ====="); + +list = memory.GetListAsync("test", limit: 10, withEmbeddings: false); +await foreach (MemoryRecord record in list) +{ + Console.WriteLine(record.Id); + Console.WriteLine(" type: " + record.Tags["type"].First()); +} + +Console.WriteLine("== Done =="); diff --git a/service/tests/redis-tests/redis-tests.csproj b/service/tests/redis-tests/redis-tests.csproj new file mode 100644 index 000000000..0f7a1a871 --- /dev/null +++ b/service/tests/redis-tests/redis-tests.csproj @@ -0,0 +1,19 @@ + + + + Exe + net7.0 + redis_tests + enable + enable + false + false + $(NoWarn);CA1050,CA2007,CA1826,CA1303,CA1307,SKEXP0001 + + + + + + + + diff --git a/tools/README.md b/tools/README.md index 0fa909d9e..b210f5306 100644 --- a/tools/README.md +++ b/tools/README.md @@ -41,4 +41,11 @@ Script to start RabbitMQ using Docker, for local development/debugging. RabbitMQ is used to provides queues for the asynchronous pipelines, as an alternative to -[Azure Queues](https://learn.microsoft.com/azure/storage/queues/storage-queues-introduction). \ No newline at end of file +[Azure Queues](https://learn.microsoft.com/azure/storage/queues/storage-queues-introduction). + +# run-redis.sh + +Script to start Redis using Docker, for local development/debugging. + +Redis is used to store and search vectors, as an alternative to +[Azure AI Search](https://azure.microsoft.com/products/ai-services/ai-search/). \ No newline at end of file diff --git a/tools/run-redis.sh b/tools/run-redis.sh new file mode 100755 index 000000000..56a7a6908 --- /dev/null +++ b/tools/run-redis.sh @@ -0,0 +1 @@ +docker run -p 6379:6379 redis/redis-stack-server \ No newline at end of file