Add support for SQL Server Vector Search (#722)

## Motivation and Context (Why the change? What's the scenario?)

Azure SQL Database now provides native Vector Support, currently in EAP
(https://devblogs.microsoft.com/azure-sql/announcing-eap-native-vector-support-in-azure-sql-database).

This PR extends SQL Server Memory DB with a flag that allows to use this
new feature.

> [!IMPORTANT]  
> Remember that, at this time, Vector Support is available only on Azure
SQL Database. On the other hand, the current SQL Server Memory DB
requires a COLUMNSTORE INDEX that, on Azure, is available only on vCore
databases and Standard databases in S3 and above pricing tiers
(https://azure.microsoft.com/en-us/blog/columnstore-support-in-standard-tier-azure-sql-databases).

---------

Co-authored-by: Devis Lucato <[email protected]>

Author: marcominerva

extensions/SQLServer/SQLServer/DependencyInjection.cs

@@ -14,10 +14,10 @@ namespace Microsoft.KernelMemory;
 public static partial class KernelMemoryBuilderExtensions
 {
     /// <summary>
-    /// Kernel Memory Builder extension method to add SqlServer memory connector.
+    /// Kernel Memory Builder extension method to add SQL Server memory connector.
     /// </summary>
     /// <param name="builder">KM builder instance</param>
-    /// <param name="config">SqlServer configuration</param>
+    /// <param name="config">SQL Server configuration</param>
     public static IKernelMemoryBuilder WithSqlServerMemoryDb(
         this IKernelMemoryBuilder builder,
         SqlServerConfig config)
@@ -27,15 +27,17 @@ public static IKernelMemoryBuilder WithSqlServerMemoryDb(
     }
 
     /// <summary>
-    /// Kernel Memory Builder extension method to add SqlServer memory connector.
+    /// Kernel Memory Builder extension method to add SQL Server memory connector.
     /// </summary>
     /// <param name="builder">KM builder instance</param>
-    /// <param name="connString">SqlServer connection string</param>
+    /// <param name="connString">SQL Server connection string</param>
+    /// <param name="useNativeVectorSearch">Whether to use native vector search or not</param>
     public static IKernelMemoryBuilder WithSqlServerMemoryDb(
         this IKernelMemoryBuilder builder,
-        string connString)
+        string connString,
+        bool useNativeVectorSearch = false)
     {
-        builder.Services.AddSqlServerAsMemoryDb(connString);
+        builder.Services.AddSqlServerAsMemoryDb(connString, useNativeVectorSearch);
         return builder;
     }
 }
@@ -46,10 +48,10 @@ public static IKernelMemoryBuilder WithSqlServerMemoryDb(
 public static partial class DependencyInjection
 {
     /// <summary>
-    /// Inject SqlServer as the default implementation of IMemoryDb
+    /// Inject SQL Server as the default implementation of IMemoryDb
     /// </summary>
     /// <param name="services">Service collection</param>
-    /// <param name="config">Postgres configuration</param>
+    /// <param name="config">SQL Server configuration</param>
     public static IServiceCollection AddSqlServerAsMemoryDb(
         this IServiceCollection services,
         SqlServerConfig config)
@@ -60,15 +62,17 @@ public static IServiceCollection AddSqlServerAsMemoryDb(
     }
 
     /// <summary>
-    /// Inject SqlServer as the default implementation of IMemoryDb
+    /// Inject SQL Server as the default implementation of IMemoryDb
     /// </summary>
     /// <param name="services">Service collection</param>
     /// <param name="connString">SQL Server connection string</param>
+    /// <param name="useNativeVectorSearch">Whether to use native vector search or not</param>
     public static IServiceCollection AddSqlServerAsMemoryDb(
         this IServiceCollection services,
-        string connString)
+        string connString,
+        bool useNativeVectorSearch = false)
     {
-        var config = new SqlServerConfig { ConnectionString = connString };
+        var config = new SqlServerConfig { ConnectionString = connString, UseNativeVectorSearch = useNativeVectorSearch };
         return services.AddSqlServerAsMemoryDb(config);
     }
 }

extensions/SQLServer/SQLServer/QueryProviders/DefaultQueryProvider.cs

@@ -0,0 +1,59 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.Data.SqlClient;
+
+namespace Microsoft.KernelMemory.MemoryDb.SQLServer.QueryProviders;
+
+public interface ISqlServerQueryProvider
+{
+    /// <summary>
+    /// Return SQL used to create a new index
+    /// </summary>
+    public string PrepareCreateIndexQuery(
+        int sqlServerVersion,
+        string index,
+        int vectorSize);
+
+    /// <summary>
+    /// Return SQL used to get a list of indexes
+    /// </summary>
+    public string PrepareGetIndexesQuery();
+
+    /// <summary>
+    /// Return SQL used to delete an index
+    /// </summary>
+    public string PrepareDeleteIndexQuery(string index);
+
+    /// <summary>
+    /// Return SQL used to delete a memory record
+    /// </summary>
+    public string PrepareDeleteRecordQuery(string index);
+
+    /// <summary>
+    /// Return SQL used to get a list of memory records
+    /// </summary>
+    public string PrepareGetRecordsListQuery(
+        string index,
+        ICollection<MemoryFilter>? filters,
+        bool withEmbedding,
+        SqlParameterCollection parameters);
+
+    /// <summary>
+    /// Return SQL used to get a list of similar memory records
+    /// </summary>
+    public string PrepareGetSimilarRecordsListQuery(
+        string index,
+        ICollection<MemoryFilter>? filters,
+        bool withEmbedding,
+        SqlParameterCollection parameters);
+
+    /// <summary>
+    /// Return SQL used to upsert a batch of memory records
+    /// </summary>
+    public string PrepareUpsertRecordsBatchQuery(string index);
+
+    /// <summary>
+    /// Return SQL used to create all supporting tables
+    /// </summary>
+    public string PrepareCreateAllSupportingTablesQuery();
+}

extensions/SQLServer/SQLServer/QueryProviders/ISqlServerQueryProvider.cs

@@ -0,0 +1,85 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Globalization;
+using System.Text;
+using Microsoft.Data.SqlClient;
+
+namespace Microsoft.KernelMemory.MemoryDb.SQLServer.QueryProviders;
+
+internal static class Utils
+{
+    /// <summary>
+    /// Gets the full table name with schema.
+    /// </summary>
+    /// <param name="config">Server settings</param>
+    /// <param name="tableName">The table name.</param>
+    internal static string GetFullTableName(SqlServerConfig config, string tableName)
+    {
+        return $"[{config.Schema}].[{tableName}]";
+    }
+
+    /// <summary>
+    /// Generates the filters as SQL commands and sets the SQL parameters
+    /// </summary>
+    /// <param name="config">Server settings</param>
+    /// <param name="index">The index name.</param>
+    /// <param name="parameters">The SQL parameters to populate.</param>
+    /// <param name="filters">The filters to apply</param>
+    internal static string GenerateFilters(
+        SqlServerConfig config,
+        string index,
+        SqlParameterCollection parameters,
+        ICollection<MemoryFilter>? filters)
+    {
+        var filterBuilder = new StringBuilder();
+
+        if (filters is null || filters.Count <= 0 || filters.All(f => f.Count <= 0))
+        {
+            return string.Empty;
+        }
+
+        filterBuilder.Append("AND ( ");
+
+        for (int i = 0; i < filters.Count; i++)
+        {
+            var filter = filters.ElementAt(i);
+
+            if (i > 0)
+            {
+                filterBuilder.Append(" OR ");
+            }
+
+            for (int j = 0; j < filter.Pairs.Count(); j++)
+            {
+                var value = filter.Pairs.ElementAt(j);
+
+                if (j > 0)
+                {
+                    filterBuilder.Append(" AND ");
+                }
+
+                filterBuilder.Append(" ( ");
+
+                filterBuilder.Append(CultureInfo.CurrentCulture, $@"EXISTS (
+                         SELECT
+	                        1
+                        FROM {GetFullTableName(config, $"{config.TagsTableName}_{index}")} AS [tags]
+                        WHERE
+	                        [tags].[memory_id] = {GetFullTableName(config, config.MemoryTableName)}.[id]
+                            AND [name] = @filter_{i}_{j}_name
+                            AND [value] = @filter_{i}_{j}_value
+                        )
+                    ");
+
+                filterBuilder.Append(" ) ");
+
+                parameters.AddWithValue($"@filter_{i}_{j}_name", value.Key);
+                parameters.AddWithValue($"@filter_{i}_{j}_value", value.Value);
+            }
+        }
+
+        filterBuilder.Append(" )");
+
+        return filterBuilder.ToString();
+    }
+}