Merge pull request #118 from Nexus-Mods/ro-rocksdb-mode

Add readonly helpers, compaction, and zstd compression

Author: halgari

CHANGELOG.md

@@ -1,5 +1,10 @@
 ## Changelog
 
+### 0.9.99 - 21/1/2025
+* Provide a way to open a RocksDB backend in read-only mode
+* Default to zstd compression for RocksDB vs the previous snappy compression
+* Provide a `FlushAndCompact` method on the connection so that the user can manually trigger a compaction of the database
+
 ### 0.9.98 - 16/1/2025
 * Massively improve performance of the `ObserveDatoms` function. It is now ~200x faster than the previous version
 * Clean up the logging in the inner transacting loop, switch to high performance logging for those few critical messages

src/NexusMods.MnemonicDB.Abstractions/IConnection.cs

@@ -75,6 +75,11 @@ public interface IConnection
     /// of datoms that were excised.
     /// </summary>
     public Task<ICommitResult> Excise(EntityId[] entityIds);
+    
+    /// <summary>
+    /// Flushes the in-memory transaction log to the database, and compacts the database to remove any unused space.
+    /// </summary>
+    public Task<ICommitResult> FlushAndCompact();
 
     /// <summary>
     /// Update the database's schema with the given attributes.

src/NexusMods.MnemonicDB/Connection.cs

@@ -244,6 +244,14 @@ public async Task<ICommitResult> Excise(EntityId[] entityIds)
         return await tx.Commit();
     }
 
+    /// <inheritdoc />
+    public async Task<ICommitResult> FlushAndCompact()
+    {
+        var tx = new Transaction(this);
+        tx.Set(new FlushAndCompact());
+        return await tx.Commit();
+    }
+
     /// <inheritdoc />
     public Task UpdateSchema(params IAttribute[] attribute)
     {

src/NexusMods.MnemonicDB/InternalTxFunctions/FlushAndCompact.cs

@@ -0,0 +1,14 @@
+using NexusMods.MnemonicDB.Storage;
+
+namespace NexusMods.MnemonicDB.InternalTxFunctions;
+
+/// <summary>
+/// Performs a flush and compact operation on the backend.
+/// </summary>
+internal class FlushAndCompact : AInternalFn
+{
+    public override void Execute(DatomStore store)
+    {
+        store.Backend.FlushAndCompact();
+    }
+}

src/NexusMods.MnemonicDB/Storage/Abstractions/IStoreBackend.cs

@@ -34,5 +34,10 @@ public interface IStoreBackend : IDisposable
     ///     during calls to GetIterator
     /// </summary>
     public ISnapshot GetSnapshot();
-    
+
+    /// <summary>
+    /// Flushes all the logs to disk, and performs a compaction, recommended if you want to archive the database
+    /// and move it somewhere else.
+    /// </summary>
+    public void FlushAndCompact();
 }

src/NexusMods.MnemonicDB/Storage/InMemoryBackend/Backend.cs

@@ -51,6 +51,12 @@ public ISnapshot GetSnapshot()
         return new Snapshot(_index, AttributeCache);
     }
 
+    /// <inheritdoc />
+    public void FlushAndCompact()
+    {
+        // No need to do anything
+    }
+
     /// <inheritdoc />
     public void Dispose() { }
 }

src/NexusMods.MnemonicDB/Storage/RocksDbBackend/Backend.cs

@@ -14,12 +14,14 @@ public class Backend : IStoreBackend
 {
     internal RocksDb? Db = null!;
     private IntPtr _comparator;
+    private readonly bool _isReadOnly;
 
     /// <summary>
     /// Default constructor
     /// </summary>
-    public Backend()
+    public Backend(bool readOnly = false)
     {
+        _isReadOnly = readOnly;
         AttributeCache = new AttributeCache();
     }
 
@@ -50,10 +52,25 @@ public void Init(AbsolutePath location)
         var options = new DbOptions()
             .SetCreateIfMissing()
             .SetCreateMissingColumnFamilies()
-            .SetCompression(Compression.Lz4)
+            .SetCompression(Compression.Zstd)
             .SetComparator(_comparator);
 
-        Db = RocksDb.Open(options, location.ToString());
+        Native.Instance.rocksdb_options_set_bottommost_compression(options.Handle, (int)Compression.Zstd);
+        
+        if (_isReadOnly)
+            Db = RocksDb.OpenReadOnly(options, location.ToString(), false);
+        else 
+            Db = RocksDb.Open(options, location.ToString());
+    }
+
+    /// <summary>
+    /// Flushes all the logs to disk, and performs a compaction, recommended if you want to archive the database
+    /// and move it somewhere else.
+    /// </summary>
+    public void FlushAndCompact()
+    {
+        Db?.Flush(new FlushOptions().SetWaitForFlush(true));
+        Db?.CompactRange([0x00], [0xFF]);
     }
 
     /// <inheritdoc />

tests/NexusMods.MnemonicDB.Storage.Tests/NullConnection.cs

@@ -37,6 +37,11 @@ public Task<ICommitResult> Excise(EntityId[] entityIds)
         throw new NotSupportedException();
     }
 
+    public Task<ICommitResult> FlushAndCompact()
+    {
+        throw new NotSupportedException();
+    }
+
     public Task UpdateSchema(params IAttribute[] attribute)
     {
         throw new NotSupportedException();

tests/NexusMods.MnemonicDB.Tests/DbTests.cs

@@ -1233,4 +1233,23 @@ public async Task CanHandleLargeNumbersOfSubscribers()
             sub.Dispose();
         }
     }
+
+    [Fact]
+    public async Task CanFlushAndCompactTheDB()
+    {
+        var tx = Connection.BeginTransaction();
+
+        for (int i = 0; i < 1000; i++)
+        {
+            _ = new Mod.New(tx)
+            {
+                Name = "Test Mod " + i,
+                Source = new Uri("http://test.com"),
+                LoadoutId = EntityId.From(0)
+            };
+        }
+        await tx.Commit();
+
+        await Connection.FlushAndCompact();
+    }
 }