Rework once again of the datom store, taking hints from DataHike and leveraging RocksDB even more

benchmarks/NexusMods.EventSourcing.Benchmarks/Benchmarks/ABenchmark.cs

@@ -0,0 +1,52 @@
+using System;
+using System.Threading.Tasks;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using NexusMods.EventSourcing.Abstractions;
+using NexusMods.EventSourcing.Storage;
+using NexusMods.EventSourcing.Storage.Abstractions;
+using NexusMods.EventSourcing.Storage.InMemoryBackend;
+using NexusMods.EventSourcing.TestModel;
+using NexusMods.Paths;
+using Xunit;
+
+namespace NexusMods.EventSourcing.Benchmarks.Benchmarks;
+
+public class ABenchmark : IAsyncLifetime
+{
+    private IHost _host = null!;
+    protected IConnection Connection = null!;
+
+    public IServiceProvider Services => _host.Services;
+
+    public async Task InitializeAsync()
+    {
+        var builder = Host.CreateDefaultBuilder()
+            .ConfigureServices(services =>
+            {
+                services.AddEventSourcingStorage()
+                    .AddRocksDbBackend()
+                    .AddEventSourcing()
+                    .AddTestModel()
+                    .AddDatomStoreSettings(new DatomStoreSettings
+                    {
+                        Path = FileSystem.Shared.FromUnsanitizedFullPath("benchmarks" + Guid.NewGuid())
+                    });
+            });
+
+        _host = builder.Build();
+        Connection = await NexusMods.EventSourcing.Connection.Start(Services);
+    }
+
+    public async Task DisposeAsync()
+    {
+        var path = Services.GetRequiredService<DatomStoreSettings>().Path;
+        await _host.StopAsync();
+        _host.Dispose();
+
+        if (path.DirectoryExists())
+            path.DeleteDirectory();
+        if (path.FileExists)
+            path.Delete();
+    }
+}

benchmarks/NexusMods.EventSourcing.Benchmarks/Benchmarks/ReadTests.cs

@@ -13,30 +13,22 @@
 namespace NexusMods.EventSourcing.Benchmarks.Benchmarks;
 
 [MemoryDiagnoser]
-public class ReadTests : IAsyncLifetime
+public class ReadTests : ABenchmark
 {
-    private IConnection _connection = null!;
-    private List<EntityId> _entityIdsAscending = null!;
-    private List<EntityId> _entityIdsDescending = null!;
-    private List<EntityId> _entityIdsRandom = null!;
-    private readonly IServiceProvider _services;
     private EntityId _readId;
     private IDb _db = null!;
+    private EntityId[] _entityIds = null!;
 
-    public ReadTests()
-    {
-        _services = AppHost.Create();
-    }
 
     private const int MaxCount = 10000;
 
     [GlobalSetup]
     public async Task Setup()
     {
         await InitializeAsync();
-        var tx = _connection.BeginTransaction();
+        var tx = Connection.BeginTransaction();
         var entityIds = new List<EntityId>();
-        for (var i = 0; i < MaxCount; i++)
+        for (var i = 0; i < Count; i++)
         {
             var file = new File(tx)
             {
@@ -48,42 +40,16 @@ public async Task Setup()
         }
         var result = await tx.Commit();
 
-        entityIds = entityIds.Select(e => result[e]).ToList();
-        _entityIdsAscending = entityIds.OrderBy(id => id.Value).ToList();
-        _entityIdsDescending = entityIds.OrderByDescending(id => id.Value).ToList();
+        _entityIds = entityIds.Select(e => result[e]).ToArray();
 
-        var idArray = entityIds.ToArray();
-        Random.Shared.Shuffle(idArray);
-        _entityIdsRandom = idArray.ToList();
+        _readId = _entityIds[_entityIds.Length / 2];
 
-        _readId = Ids.Take(Count).Skip(Count / 2).First();
-
-        _db = _connection.Db;
+        _db = Connection.Db;
     }
 
-
     [Params(1, 1000, MaxCount)]
     public int Count { get; set; } = MaxCount;
 
-    public enum SortOrder
-    {
-        Ascending,
-        Descending,
-        Random
-    }
-
-
-    //[Params(SortOrder.Ascending, SortOrder.Descending, SortOrder.Random)]
-    public SortOrder Order { get; set; } = SortOrder.Descending;
-
-    public List<EntityId> Ids => Order switch
-    {
-        SortOrder.Ascending => _entityIdsAscending,
-        SortOrder.Descending => _entityIdsDescending,
-        SortOrder.Random => _entityIdsRandom,
-        _ => throw new ArgumentOutOfRangeException()
-    };
-
     [Benchmark]
     public ulong ReadFiles()
     {
@@ -92,13 +58,10 @@ public ulong ReadFiles()
         return sum;
     }
 
-    public async Task InitializeAsync()
-    {
-        _connection = await AppHost.CreateConnection(_services);
-    }
-
-    public Task DisposeAsync()
+    [Benchmark]
+    public long ReadAll()
     {
-        return Task.CompletedTask;
+        return _db.Get<File>(_entityIds)
+            .Sum(e => (long)e.Index);
     }
 }

benchmarks/NexusMods.EventSourcing.Benchmarks/Benchmarks/WriteTests.cs

@@ -11,6 +11,7 @@ namespace NexusMods.EventSourcing.Benchmarks.Benchmarks;
 
 public class WriteTests
 {
+    /*
     private readonly IConnection _connection;
 
     public WriteTests()
@@ -51,5 +52,6 @@ public async Task AddFiles()
             loaded.Should().NotBeNull("the entity should be in the database");
         }
     }
+    */
 
 }

benchmarks/NexusMods.EventSourcing.Benchmarks/NexusMods.EventSourcing.Benchmarks.csproj

@@ -9,7 +9,8 @@
 
     <ItemGroup>
       <PackageReference Include="BenchmarkDotNet" Version="0.13.12" />
-      <PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.0" />
+      <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
+      <PackageReference Update="JetBrains.Annotations" Version="2023.3.0" />
     </ItemGroup>
 
     <ItemGroup>

benchmarks/NexusMods.EventSourcing.Benchmarks/Program.cs

@@ -6,7 +6,7 @@
 using NexusMods.EventSourcing.Benchmarks.Benchmarks;
 
 
-#if DEBUG
+//#if DEBUG
 
 var benchmark = new ReadTests
 {
@@ -15,14 +15,20 @@
 
 var sw = Stopwatch.StartNew();
 await benchmark.Setup();
-ulong result = 0;
-for (var i = 0; i < 1000000; i++)
+long result = 0;
+for (var i = 0; i < 10000; i++)
 {
-    result = benchmark.ReadFiles();
+    result = benchmark.ReadAll();
 }
 Console.WriteLine("Elapsed: " + sw.Elapsed + " Result: " + result);
+
+
+
+/*
 #else
 
 BenchmarkRunner.Run<ReadTests>();
 
 #endif
+*/
+

benchmarks/OneBillionDatomsTest/OneBillionDatomsTest.csproj

@@ -8,7 +8,8 @@
     </PropertyGroup>
 
     <ItemGroup>
-      <PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.0" />
+      <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
+      <PackageReference Update="JetBrains.Annotations" Version="2023.3.0" />
     </ItemGroup>
 
     <ItemGroup>

benchmarks/OneBillionDatomsTest/Program.cs

@@ -16,6 +16,7 @@
     {
         s.AddEventSourcingStorage()
             .AddEventSourcing()
+            .AddRocksDbBackend()
             .AddTestModel()
             .AddSingleton<DatomStoreSettings>(_ => new DatomStoreSettings
             {

docs/IndexFormat.md

@@ -5,9 +5,7 @@ hide:
 
 ## Index Format
 
-The index format of the framework follows fairly closely to one found in Datomic, although differences likely exist due it having
-a different set of constraints and requirements. The base format of the system is a sorted set of tuples. However each tuple
-exists in multiple indexes with a different sorting and conflict resolution strategy for each.
+The index format of the framework will look familiar to those who have used Datomic in the past, although there's some very critical
 
 In general data flows through 4 core indexes:
 

src/NexusMods.EventSourcing.Abstractions/AttributeId.cs

@@ -13,4 +13,9 @@ public readonly partial struct AttributeId
     /// </summary>
     /// <returns></returns>
     public EntityId ToEntityId() => EntityId.From(Value);
+
+    /// <summary>
+    /// Minimum value for an AttributeId.
+    /// </summary>
+    public static AttributeId Min => new(ulong.MinValue);
 }