Adding ability to configure extensions for Sqlite (#428)

* Adding ability to configure extensions for Sqlite * Loading sqlite extensions from nuget packages * Fixing test * Have to clean up the connection string for the SqliteConnection to be created * DbConnectionStringBuilder lower-cases connection string keys? * Making it more flexible to work with extensions, but also marking the feature as experimental * Adding diagnostics info * Update src/CommunityToolkit.Aspire.Hosting.Sqlite/SqliteResourceBuilderExtensions.cs Co-authored-by: Alireza Baloochi <[email protected]> * Turning metadata class into shared type * Adding test for extension loading * Moving where package is * Fixing test * Adding some logging * Bit more logging * Little more logging * Disabling the current test --------- Co-authored-by: Alireza Baloochi <[email protected]>
CommunityToolkit · Feb 18, 2025 · a8d1717 · a8d1717
1 parent 38897c3
commit a8d1717
Show file tree

Hide file tree

Showing 21 changed files with 474 additions and 15 deletions.
diff --git a/Directory.Packages.props b/Directory.Packages.props
@@ -30,6 +30,8 @@
     </PackageVersion>
     <PackageVersion Include="Microsoft.Extensions.AI" Version="$(MEAIVersion)" />
     <PackageVersion Include="Microsoft.Extensions.Configuration.Binder" Version="$(DotNetExtensionsVersion)" />
+    <PackageVersion Include="Microsoft.DotNet.PlatformAbstractions" Version="3.1.6" />
+    <PackageVersion Include="Microsoft.Extensions.DependencyModel" Version="9.0.1" />
     <PackageVersion Include="Microsoft.Extensions.Hosting.Abstractions" Version="$(DotNetExtensionsVersion)" />
     <PackageVersion Include="Swashbuckle.AspNetCore" Version="7.2.0" />
     <!-- .NET packages -->
@@ -54,6 +56,7 @@
     <PackageVersion Include="xunit.runner.visualstudio" Version="2.8.2" />
     <PackageVersion Include="xunit.extensibility.execution" Version="2.9.2" />
     <PackageVersion Include="Microsoft.DotNet.XUnitExtensions" Version=" 9.0.0-beta.24568.1" />
+    <PackageVersion Include="mod_spatialite" Version="4.3.0.1" />
     <!-- External packages -->
     <PackageVersion Include="Azure.Provisioning.AppContainers" Version="1.0.0" />
     <PackageVersion Include="JsonSchema.Net" Version="7.3.0" />

diff --git a/docs/diagnostics.md b/docs/diagnostics.md
@@ -10,3 +10,6 @@ In these cases, refer to the `<remarks>` docs section of the API for more inform
 
 Once a release of .NET Aspire with that API is available, the API in the .NET Aspire Community Toolkit will be marked as obsolete and will be removed in a future release.
 
+## CTASPIRE002
+
+Support for loading extensions into SQLite requires either a NuGet package or folder path to the library to be provided, and as a result there is some custom logic to load the extension based on the path or NuGet package. This logic will require some experimenting to figure out edge cases, so the feature for extension loading will be kept as experimental until it is proven to be stable.
diff --git a/nuget.config b/nuget.config
@@ -8,7 +8,7 @@
   </packageSources>
   <packageSourceMapping>
     <packageSource key="dotnet-eng">
-      <package pattern="Microsoft.DotNet.*" />
+      <package pattern="Microsoft.DotNet.XUnitExtensions" />
     </packageSource>
     <packageSource key="nuget">
       <package pattern="*" />

diff --git a/src/CommunityToolkit.Aspire.Hosting.Sqlite/CommunityToolkit.Aspire.Hosting.Sqlite.csproj b/src/CommunityToolkit.Aspire.Hosting.Sqlite/CommunityToolkit.Aspire.Hosting.Sqlite.csproj
@@ -11,4 +11,8 @@
   <ItemGroup>
     <InternalsVisibleTo Include="CommunityToolkit.Aspire.Hosting.Sqlite.Tests"></InternalsVisibleTo>
   </ItemGroup>
+
+  <ItemGroup>
+    <Compile Include="$(SharedDir)\Sqlite\SqliteExtensionMetadata.cs" Link="Utils\Sqlite\SqliteExtensionMetadata.cs" />
+  </ItemGroup>
 </Project>
diff --git a/src/CommunityToolkit.Aspire.Hosting.Sqlite/PublicAPI.Unshipped.txt b/src/CommunityToolkit.Aspire.Hosting.Sqlite/PublicAPI.Unshipped.txt
@@ -1,11 +1,24 @@
 #nullable enable
 Aspire.Hosting.ApplicationModel.SqliteResource
 Aspire.Hosting.ApplicationModel.SqliteResource.ConnectionStringExpression.get -> Aspire.Hosting.ApplicationModel.ReferenceExpression!
+Aspire.Hosting.ApplicationModel.SqliteResource.Extensions.get -> System.Collections.Generic.IReadOnlyCollection<Microsoft.Extensions.Hosting.SqliteExtensionMetadata!>!
 Aspire.Hosting.ApplicationModel.SqliteResource.SqliteResource(string! name, string! databasePath, string! databaseFileName) -> void
 Aspire.Hosting.ApplicationModel.SqliteWebResource
 Aspire.Hosting.ApplicationModel.SqliteWebResource.ConnectionStringExpression.get -> Aspire.Hosting.ApplicationModel.ReferenceExpression!
 Aspire.Hosting.ApplicationModel.SqliteWebResource.PrimaryEndpoint.get -> Aspire.Hosting.ApplicationModel.EndpointReference!
 Aspire.Hosting.ApplicationModel.SqliteWebResource.SqliteWebResource(string! name) -> void
 Aspire.Hosting.SqliteResourceBuilderExtensions
 static Aspire.Hosting.SqliteResourceBuilderExtensions.AddSqlite(this Aspire.Hosting.IDistributedApplicationBuilder! builder, string! name, string? databasePath = null, string? databaseFileName = null) -> Aspire.Hosting.ApplicationModel.IResourceBuilder<Aspire.Hosting.ApplicationModel.SqliteResource!>!
+static Aspire.Hosting.SqliteResourceBuilderExtensions.WithLocalExtension(this Aspire.Hosting.ApplicationModel.IResourceBuilder<Aspire.Hosting.ApplicationModel.SqliteResource!>! builder, string! extension, string! extensionPath) -> Aspire.Hosting.ApplicationModel.IResourceBuilder<Aspire.Hosting.ApplicationModel.SqliteResource!>!
+static Aspire.Hosting.SqliteResourceBuilderExtensions.WithNuGetExtension(this Aspire.Hosting.ApplicationModel.IResourceBuilder<Aspire.Hosting.ApplicationModel.SqliteResource!>! builder, string! extension, string? packageName = null) -> Aspire.Hosting.ApplicationModel.IResourceBuilder<Aspire.Hosting.ApplicationModel.SqliteResource!>!
 static Aspire.Hosting.SqliteResourceBuilderExtensions.WithSqliteWeb(this Aspire.Hosting.ApplicationModel.IResourceBuilder<Aspire.Hosting.ApplicationModel.SqliteResource!>! builder, string? containerName = null) -> Aspire.Hosting.ApplicationModel.IResourceBuilder<Aspire.Hosting.ApplicationModel.SqliteResource!>!
+Microsoft.Extensions.Hosting.SqliteExtensionMetadata
+Microsoft.Extensions.Hosting.SqliteExtensionMetadata.Extension.get -> string!
+Microsoft.Extensions.Hosting.SqliteExtensionMetadata.Extension.init -> void
+Microsoft.Extensions.Hosting.SqliteExtensionMetadata.ExtensionFolder.get -> string?
+Microsoft.Extensions.Hosting.SqliteExtensionMetadata.ExtensionFolder.init -> void
+Microsoft.Extensions.Hosting.SqliteExtensionMetadata.IsNuGetPackage.get -> bool
+Microsoft.Extensions.Hosting.SqliteExtensionMetadata.IsNuGetPackage.init -> void
+Microsoft.Extensions.Hosting.SqliteExtensionMetadata.PackageName.get -> string?
+Microsoft.Extensions.Hosting.SqliteExtensionMetadata.PackageName.init -> void
+Microsoft.Extensions.Hosting.SqliteExtensionMetadata.SqliteExtensionMetadata(string! Extension, string? PackageName, bool IsNuGetPackage, string? ExtensionFolder) -> void
diff --git a/src/CommunityToolkit.Aspire.Hosting.Sqlite/SqliteResource.cs b/src/CommunityToolkit.Aspire.Hosting.Sqlite/SqliteResource.cs
@@ -1,3 +1,6 @@
+using Microsoft.Extensions.Hosting;
+using System.Text.Json;
+
 namespace Aspire.Hosting.ApplicationModel;
 
 /// <summary>
@@ -15,5 +18,18 @@ public class SqliteResource(string name, string databasePath, string databaseFil
     internal string DatabaseFilePath => Path.Combine(DatabasePath, DatabaseFileName);
 
     /// <inheritdoc/>
-    public ReferenceExpression ConnectionStringExpression => ReferenceExpression.Create($"Data Source={DatabaseFilePath};Cache=Shared;Mode=ReadWriteCreate;");
+    public ReferenceExpression ConnectionStringExpression => ReferenceExpression.Create($"Data Source={DatabaseFilePath};Cache=Shared;Mode=ReadWriteCreate;Extensions={JsonSerializer.Serialize(Extensions)}");
+
+    private readonly List<SqliteExtensionMetadata> extensions = [];
+
+    /// <summary>
+    /// Gets the extensions to be loaded into the database.
+    /// </summary>
+    /// <remarks>
+    /// Extensions are not loaded by the hosting integration, the information is provided for the client to load the extensions.
+    /// </remarks>
+    public IReadOnlyCollection<SqliteExtensionMetadata> Extensions => extensions;
+
+    internal void AddExtension(SqliteExtensionMetadata extension) => extensions.Add(extension);
 }
+
diff --git a/src/CommunityToolkit.Aspire.Hosting.Sqlite/SqliteResourceBuilderExtensions.cs b/src/CommunityToolkit.Aspire.Hosting.Sqlite/SqliteResourceBuilderExtensions.cs
@@ -1,4 +1,5 @@
 using Aspire.Hosting.ApplicationModel;
+using System.Diagnostics.CodeAnalysis;
 
 namespace Aspire.Hosting;
 
@@ -85,4 +86,51 @@ public static IResourceBuilder<SqliteResource> WithSqliteWeb(this IResourceBuild
 
         return builder;
     }
+
+    /// <summary>
+    /// Adds an extension to the Sqlite resource that will be loaded from a NuGet package.
+    /// </summary>
+    /// <param name="builder">The resource builder.</param>
+    /// <param name="extension">The name of the extension file with to add, eg: vec0, without file extension.</param>
+    /// <param name="packageName">The name of the NuGet package. If this is set to null, the value of <paramref name="extension"/> is used.</param>
+    /// <returns>The resource builder.</returns>
+    /// <remarks>
+    /// Extensions are not loaded by the hosting integration, the information is provided for the client to load the extensions.
+    /// 
+    /// This extension is experimental while the final design of extension loading is decided.
+    /// </remarks>
+    [Experimental("CTASPIRE002", UrlFormat = "https://aka.ms/communitytoolkit/aspire/diagnostics#{0}")]
+    public static IResourceBuilder<SqliteResource> WithNuGetExtension(this IResourceBuilder<SqliteResource> builder, string extension, string? packageName = null)
+    {
+        ArgumentNullException.ThrowIfNull(builder, nameof(builder));
+        ArgumentException.ThrowIfNullOrEmpty(extension, nameof(extension));
+
+        builder.Resource.AddExtension(new(extension, packageName ?? extension, IsNuGetPackage: true, ExtensionFolder: null));
+
+        return builder;
+    }
+
+    /// <summary>
+    /// Adds an extension to the Sqlite resource that will be loaded from a local path.
+    /// </summary>
+    /// <param name="builder">The resource builder.</param>
+    /// <param name="extension">The name of the extension file with to add, eg: vec0, without file extension.</param>
+    /// <param name="extensionPath">The path to the extension file.</param>
+    /// <returns>The resource builder.</returns>
+    /// <remarks>
+    /// Extensions are not loaded by the hosting integration, the information is provided for the client to load the extensions.
+    /// 
+    /// This extension is experimental while the final design of extension loading is decided.
+    /// </remarks>
+    [Experimental("CTASPIRE002", UrlFormat = "https://aka.ms/communitytoolkit/aspire/diagnostics#{0}")]
+    public static IResourceBuilder<SqliteResource> WithLocalExtension(this IResourceBuilder<SqliteResource> builder, string extension, string extensionPath)
+    {
+        ArgumentNullException.ThrowIfNull(builder, nameof(builder));
+        ArgumentException.ThrowIfNullOrEmpty(extension, nameof(extension));
+        ArgumentException.ThrowIfNullOrEmpty(extensionPath, nameof(extensionPath));
+
+        builder.Resource.AddExtension(new(extension, PackageName: null, IsNuGetPackage: false, extensionPath));
+
+        return builder;
+    }
 }
diff --git a/src/CommunityToolkit.Aspire.Microsoft.Data.Sqlite/AspireSqliteExtensions.cs b/src/CommunityToolkit.Aspire.Microsoft.Data.Sqlite/AspireSqliteExtensions.cs
@@ -3,7 +3,13 @@
 using Microsoft.Data.Sqlite;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.DependencyModel;
 using Microsoft.Extensions.Diagnostics.HealthChecks;
+using Microsoft.Extensions.Logging;
+using System.Data.Common;
+using System.Runtime.InteropServices;
+using System.Text.Json;
+using RuntimeEnvironment = Microsoft.DotNet.PlatformAbstractions.RuntimeEnvironment;
 
 namespace Microsoft.Extensions.Hosting;
 
@@ -61,6 +67,15 @@ private static void AddSqliteClient(
             settings.ConnectionString = connectionString;
         }
 
+        if (!string.IsNullOrEmpty(settings.ConnectionString))
+        {
+            var cbs = new DbConnectionStringBuilder { ConnectionString = settings.ConnectionString };
+            if (cbs.TryGetValue("Extensions", out var extensions))
+            {
+                settings.Extensions = JsonSerializer.Deserialize<IEnumerable<SqliteExtensionMetadata>>((string)extensions) ?? [];
+            }
+        }
+
         configureSettings?.Invoke(settings);
 
         builder.RegisterSqliteServices(settings, connectionName, serviceKey);
@@ -100,8 +115,181 @@ private static void RegisterSqliteServices(
 
         SqliteConnection CreateConnection(IServiceProvider sp, object? key)
         {
+            var logger = sp.GetRequiredService<ILogger<SqliteConnection>>();
             ConnectionStringValidation.ValidateConnectionString(settings.ConnectionString, connectionName, DefaultConfigSectionName);
-            return new SqliteConnection(settings.ConnectionString);
+            var csb = new DbConnectionStringBuilder { ConnectionString = settings.ConnectionString };
+            if (csb.ContainsKey("Extensions"))
+            {
+                csb.Remove("Extensions");
+            }
+            var connection = new SqliteConnection(csb.ConnectionString);
+
+            foreach (var extension in settings.Extensions)
+            {
+                if (extension.IsNuGetPackage)
+                {
+                    if (string.IsNullOrEmpty(extension.PackageName))
+                    {
+                        throw new InvalidOperationException("PackageName is required when loading an extension from a NuGet package.");
+                    }
+
+                    EnsureLoadableFromNuGet(extension.Extension, extension.PackageName, logger);
+                }
+                else
+                {
+                    if (string.IsNullOrEmpty(extension.ExtensionFolder))
+                    {
+                        throw new InvalidOperationException("ExtensionFolder is required when loading an extension from a folder.");
+                    }
+
+                    EnsureLoadableFromLocalPath(extension.Extension, extension.ExtensionFolder);
+                }
+                connection.LoadExtension(extension.Extension);
+            }
+
+            return connection;
+        }
+    }
+
+    // Adapted from https://github.com/dotnet/docs/blob/dbbeda13bf016a6ff76b0baab1488c927a64ff24/samples/snippets/standard/data/sqlite/ExtensionsSample/Program.cs#L40
+    internal static void EnsureLoadableFromNuGet(string package, string library, ILogger<SqliteConnection> logger)
+    {
+        var runtimeLibrary = DependencyContext.Default?.RuntimeLibraries.FirstOrDefault(l => l.Name == package);
+        if (runtimeLibrary is null)
+        {
+            logger.LogInformation("Could not find the runtime library for package {Package}", package);
+            return;
+        }
+
+        string sharedLibraryExtension;
+        string pathVariableName = "PATH";
+        if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+        {
+            sharedLibraryExtension = ".dll";
+        }
+        else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
+        {
+            sharedLibraryExtension = ".so";
+            pathVariableName = "LD_LIBRARY_PATH";
+        }
+        else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
+        {
+            sharedLibraryExtension = ".dylib";
+            pathVariableName = "DYLD_LIBRARY_PATH";
+        }
+        else
+        {
+            throw new NotSupportedException("Unsupported OS platform");
+        }
+
+        var candidateAssets = new Dictionary<(string? Package, string Asset), int>();
+        var rid = RuntimeEnvironment.GetRuntimeIdentifier();
+        var rids = DependencyContext.Default?.RuntimeGraph.First(g => g.Runtime == rid).Fallbacks.ToList() ?? [];
+        rids.Insert(0, rid);
+
+        logger.LogInformation("Looking for {Library} in {Package} runtime assets", library, package);
+        logger.LogInformation("Possible runtime identifiers: {Rids}", string.Join(", ", rids));
+
+        foreach (var group in runtimeLibrary.NativeLibraryGroups)
+        {
+            foreach (var file in group.RuntimeFiles)
+            {
+                if (string.Equals(
+                    Path.GetFileName(file.Path),
+                    library + sharedLibraryExtension,
+                    StringComparison.OrdinalIgnoreCase))
+                {
+                    var fallbacks = rids.IndexOf(group.Runtime);
+                    if (fallbacks != -1)
+                    {
+                        logger.LogInformation("Found {Library} in {Package} runtime assets at {Path}", library, package, file.Path);
+                        candidateAssets.Add((runtimeLibrary.Path, file.Path), fallbacks);
+                    }
+                }
+            }
+        }
+
+        var assetPath = candidateAssets
+            .OrderBy(p => p.Value)
+            .Select(p => p.Key)
+            .FirstOrDefault();
+        if (assetPath != default)
+        {
+            string? assetDirectory = null;
+            if (File.Exists(Path.Combine(AppContext.BaseDirectory, assetPath.Asset)))
+            {
+                // NB: Framework-dependent deployments copy assets to the application base directory
+                assetDirectory = Path.Combine(
+                    AppContext.BaseDirectory,
+                    Path.GetDirectoryName(assetPath.Asset.Replace('/', Path.DirectorySeparatorChar))!);
+
+                logger.LogInformation("Found {Library} in {Package} runtime assets at {Path}", library, package, assetPath.Asset);
+            }
+            else
+            {
+                string? assetFullPath = null;
+                var probingDirectories = ((string?)AppDomain.CurrentDomain.GetData("PROBING_DIRECTORIES"))?
+                    .Split(Path.PathSeparator) ?? [];
+                foreach (var directory in probingDirectories)
+                {
+                    var candidateFullPath = Path.Combine(
+                        directory,
+                        assetPath.Package ?? "",
+                        assetPath.Asset);
+                    if (File.Exists(candidateFullPath))
+                    {
+                        assetFullPath = candidateFullPath;
+                    }
+                }
+
+                assetDirectory = Path.GetDirectoryName(assetFullPath);
+                logger.LogInformation("Found {Library} in {Package} runtime assets at {Path} (using PROBING_DIRECTORIES: {ProbingDirectories})", library, package, assetFullPath, string.Join(",", probingDirectories));
+            }
+
+            var path = new HashSet<string>(Environment.GetEnvironmentVariable(pathVariableName)!.Split(Path.PathSeparator));
+
+            if (assetDirectory is not null && path.Add(assetDirectory))
+            {
+                logger.LogInformation("Adding {AssetDirectory} to {PathVariableName}", assetDirectory, pathVariableName);
+                Environment.SetEnvironmentVariable(pathVariableName, string.Join(Path.PathSeparator, path));
+                logger.LogInformation("Set {PathVariableName} to: {PathVariableValue}", pathVariableName, Environment.GetEnvironmentVariable(pathVariableName));
+            }
+        }
+        else
+        {
+            logger.LogInformation("Could not find {Library} in {Package} runtime assets", library, package);
+        }
+    }
+
+    internal static void EnsureLoadableFromLocalPath(string library, string assetDirectory)
+    {
+        string sharedLibraryExtension;
+        string pathVariableName = "PATH";
+        if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+        {
+            sharedLibraryExtension = ".dll";
+        }
+        else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
+        {
+            sharedLibraryExtension = ".so";
+            pathVariableName = "LD_LIBRARY_PATH";
+        }
+        else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
+        {
+            sharedLibraryExtension = ".dylib";
+            pathVariableName = "DYLD_LIBRARY_PATH";
+        }
+        else
+        {
+            throw new NotSupportedException("Unsupported OS platform");
+        }
+
+        if (File.Exists(Path.Combine(assetDirectory, library + sharedLibraryExtension)))
+        {
+            var path = new HashSet<string>(Environment.GetEnvironmentVariable(pathVariableName)!.Split(Path.PathSeparator));
+
+            if (assetDirectory is not null && path.Add(assetDirectory))
+                Environment.SetEnvironmentVariable(pathVariableName, string.Join(Path.PathSeparator, path));
         }
     }
 }
diff --git a/...Toolkit.Aspire.Microsoft.Data.Sqlite/CommunityToolkit.Aspire.Microsoft.Data.Sqlite.csproj b/...Toolkit.Aspire.Microsoft.Data.Sqlite/CommunityToolkit.Aspire.Microsoft.Data.Sqlite.csproj
@@ -8,6 +8,8 @@
   <ItemGroup>
     <PackageReference Include="AspNetCore.HealthChecks.Sqlite" />
     <PackageReference Include="Microsoft.Data.Sqlite" />
+    <PackageReference Include="Microsoft.DotNet.PlatformAbstractions" />
+    <PackageReference Include="Microsoft.Extensions.DependencyModel" />
     <PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks" />
     <PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Binder" />
@@ -16,6 +18,7 @@
   <ItemGroup>
     <Compile Include="$(SharedDir)\HealthChecksExtensions.cs" Link="Utils\HealthChecksExtensions.cs" />
     <Compile Include="$(SharedDir)\ConnectionStringValidation.cs" Link="Utils\ConnectionStringValidation.cs" />
+    <Compile Include="$(SharedDir)\Sqlite\SqliteExtensionMetadata.cs" Link="Utils\Sqlite\SqliteExtensionMetadata.cs" />
   </ItemGroup>
 
 </Project>
Original file line number	Diff line number	Diff line change
Expand Up		@@ -10,3 +10,6 @@ In these cases, refer to the `<remarks>` docs section of the API for more inform

		Once a release of .NET Aspire with that API is available, the API in the .NET Aspire Community Toolkit will be marked as obsolete and will be removed in a future release.

		## CTASPIRE002

		Support for loading extensions into SQLite requires either a NuGet package or folder path to the library to be provided, and as a result there is some custom logic to load the extension based on the path or NuGet package. This logic will require some experimenting to figure out edge cases, so the feature for extension loading will be kept as experimental until it is proven to be stable.