From aab3c0cdd66fce6b1c8d5736ce262caa372b6607 Mon Sep 17 00:00:00 2001 From: halgari Date: Tue, 21 Jan 2025 12:03:05 -0700 Subject: [PATCH] Add readonly helpers, compaction, and zstd compression --- CHANGELOG.md | 5 ++++ .../IConnection.cs | 5 ++++ src/NexusMods.MnemonicDB/Connection.cs | 8 +++++++ .../InternalTxFunctions/FlushAndCompact.cs | 14 +++++++++++ .../Storage/Abstractions/IStoreBackend.cs | 7 +++++- .../Storage/InMemoryBackend/Backend.cs | 6 +++++ .../Storage/RocksDbBackend/Backend.cs | 23 ++++++++++++++++--- .../NullConnection.cs | 5 ++++ tests/NexusMods.MnemonicDB.Tests/DbTests.cs | 19 +++++++++++++++ 9 files changed, 88 insertions(+), 4 deletions(-) create mode 100644 src/NexusMods.MnemonicDB/InternalTxFunctions/FlushAndCompact.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b7010c7..0ab3f3a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ ## Changelog +### 0.9.99 - 21/1/2025 +* Provide a way to open a RocksDB backend in read-only mode +* Default to zstd compression for RocksDB vs the previous snappy compression +* Provide a `FlushAndCompact` method on the connection so that the user can manually trigger a compaction of the database + ### 0.9.98 - 16/1/2025 * Massively improve performance of the `ObserveDatoms` function. It is now ~200x faster than the previous version * Clean up the logging in the inner transacting loop, switch to high performance logging for those few critical messages diff --git a/src/NexusMods.MnemonicDB.Abstractions/IConnection.cs b/src/NexusMods.MnemonicDB.Abstractions/IConnection.cs index 02c354d0..bce4b330 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/IConnection.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/IConnection.cs @@ -75,6 +75,11 @@ public interface IConnection /// of datoms that were excised. /// public Task Excise(EntityId[] entityIds); + + /// + /// Flushes the in-memory transaction log to the database, and compacts the database to remove any unused space. + /// + public Task FlushAndCompact(); /// /// Update the database's schema with the given attributes. diff --git a/src/NexusMods.MnemonicDB/Connection.cs b/src/NexusMods.MnemonicDB/Connection.cs index df84767a..45a22ac8 100644 --- a/src/NexusMods.MnemonicDB/Connection.cs +++ b/src/NexusMods.MnemonicDB/Connection.cs @@ -244,6 +244,14 @@ public async Task Excise(EntityId[] entityIds) return await tx.Commit(); } + /// + public async Task FlushAndCompact() + { + var tx = new Transaction(this); + tx.Set(new FlushAndCompact()); + return await tx.Commit(); + } + /// public Task UpdateSchema(params IAttribute[] attribute) { diff --git a/src/NexusMods.MnemonicDB/InternalTxFunctions/FlushAndCompact.cs b/src/NexusMods.MnemonicDB/InternalTxFunctions/FlushAndCompact.cs new file mode 100644 index 00000000..7ae2633d --- /dev/null +++ b/src/NexusMods.MnemonicDB/InternalTxFunctions/FlushAndCompact.cs @@ -0,0 +1,14 @@ +using NexusMods.MnemonicDB.Storage; + +namespace NexusMods.MnemonicDB.InternalTxFunctions; + +/// +/// Performs a flush and compact operation on the backend. +/// +internal class FlushAndCompact : AInternalFn +{ + public override void Execute(DatomStore store) + { + store.Backend.FlushAndCompact(); + } +} diff --git a/src/NexusMods.MnemonicDB/Storage/Abstractions/IStoreBackend.cs b/src/NexusMods.MnemonicDB/Storage/Abstractions/IStoreBackend.cs index 45d77e8b..4c57bc0c 100644 --- a/src/NexusMods.MnemonicDB/Storage/Abstractions/IStoreBackend.cs +++ b/src/NexusMods.MnemonicDB/Storage/Abstractions/IStoreBackend.cs @@ -34,5 +34,10 @@ public interface IStoreBackend : IDisposable /// during calls to GetIterator /// public ISnapshot GetSnapshot(); - + + /// + /// Flushes all the logs to disk, and performs a compaction, recommended if you want to archive the database + /// and move it somewhere else. + /// + public void FlushAndCompact(); } diff --git a/src/NexusMods.MnemonicDB/Storage/InMemoryBackend/Backend.cs b/src/NexusMods.MnemonicDB/Storage/InMemoryBackend/Backend.cs index 21ce8751..6d537b69 100644 --- a/src/NexusMods.MnemonicDB/Storage/InMemoryBackend/Backend.cs +++ b/src/NexusMods.MnemonicDB/Storage/InMemoryBackend/Backend.cs @@ -51,6 +51,12 @@ public ISnapshot GetSnapshot() return new Snapshot(_index, AttributeCache); } + /// + public void FlushAndCompact() + { + // No need to do anything + } + /// public void Dispose() { } } diff --git a/src/NexusMods.MnemonicDB/Storage/RocksDbBackend/Backend.cs b/src/NexusMods.MnemonicDB/Storage/RocksDbBackend/Backend.cs index 431cc749..2a6dcd78 100644 --- a/src/NexusMods.MnemonicDB/Storage/RocksDbBackend/Backend.cs +++ b/src/NexusMods.MnemonicDB/Storage/RocksDbBackend/Backend.cs @@ -14,12 +14,14 @@ public class Backend : IStoreBackend { internal RocksDb? Db = null!; private IntPtr _comparator; + private readonly bool _isReadOnly; /// /// Default constructor /// - public Backend() + public Backend(bool readOnly = false) { + _isReadOnly = readOnly; AttributeCache = new AttributeCache(); } @@ -50,10 +52,25 @@ public void Init(AbsolutePath location) var options = new DbOptions() .SetCreateIfMissing() .SetCreateMissingColumnFamilies() - .SetCompression(Compression.Lz4) + .SetCompression(Compression.Zstd) .SetComparator(_comparator); - Db = RocksDb.Open(options, location.ToString()); + Native.Instance.rocksdb_options_set_bottommost_compression(options.Handle, (int)Compression.Zstd); + + if (_isReadOnly) + Db = RocksDb.OpenReadOnly(options, location.ToString(), false); + else + Db = RocksDb.Open(options, location.ToString()); + } + + /// + /// Flushes all the logs to disk, and performs a compaction, recommended if you want to archive the database + /// and move it somewhere else. + /// + public void FlushAndCompact() + { + Db?.Flush(new FlushOptions().SetWaitForFlush(true)); + Db?.CompactRange([0x00], [0xFF]); } /// diff --git a/tests/NexusMods.MnemonicDB.Storage.Tests/NullConnection.cs b/tests/NexusMods.MnemonicDB.Storage.Tests/NullConnection.cs index c0f66f6a..a8798047 100644 --- a/tests/NexusMods.MnemonicDB.Storage.Tests/NullConnection.cs +++ b/tests/NexusMods.MnemonicDB.Storage.Tests/NullConnection.cs @@ -37,6 +37,11 @@ public Task Excise(EntityId[] entityIds) throw new NotSupportedException(); } + public Task FlushAndCompact() + { + throw new NotSupportedException(); + } + public Task UpdateSchema(params IAttribute[] attribute) { throw new NotSupportedException(); diff --git a/tests/NexusMods.MnemonicDB.Tests/DbTests.cs b/tests/NexusMods.MnemonicDB.Tests/DbTests.cs index c2be2ebd..49da0831 100644 --- a/tests/NexusMods.MnemonicDB.Tests/DbTests.cs +++ b/tests/NexusMods.MnemonicDB.Tests/DbTests.cs @@ -1233,4 +1233,23 @@ public async Task CanHandleLargeNumbersOfSubscribers() sub.Dispose(); } } + + [Fact] + public async Task CanFlushAndCompactTheDB() + { + var tx = Connection.BeginTransaction(); + + for (int i = 0; i < 1000; i++) + { + _ = new Mod.New(tx) + { + Name = "Test Mod " + i, + Source = new Uri("http://test.com"), + LoadoutId = EntityId.From(0) + }; + } + await tx.Commit(); + + await Connection.FlushAndCompact(); + } }