From 3e3a4c27d7563273bc84815fcace066cef635905 Mon Sep 17 00:00:00 2001 From: halgari Date: Wed, 14 Feb 2024 10:07:38 -0700 Subject: [PATCH 1/8] WIP on code generator improvements --- .../AttributeDefinitionsBuilder.cs | 15 ++- .../ModelGeneration/ModelDefinitionBuilder.cs | 20 +++ .../AttributeData.cs | 2 + .../ModelGenerator.cs | 120 ++++++++++++------ .../Model/ArchiveFile.cs | 15 +++ .../Model/File.cs | 1 - .../NexusMods.EventSourcing.Tests/DbTests.cs | 28 ++++ 7 files changed, 162 insertions(+), 39 deletions(-) create mode 100644 tests/NexusMods.EventSourcing.TestModel/Model/ArchiveFile.cs diff --git a/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/AttributeDefinitionsBuilder.cs b/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/AttributeDefinitionsBuilder.cs index a9cbab2e..59cb847d 100644 --- a/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/AttributeDefinitionsBuilder.cs +++ b/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/AttributeDefinitionsBuilder.cs @@ -13,6 +13,18 @@ public AttributeDefinitionsBuilder() } + /// + /// Defines a new attribute with the given name and description + /// + /// + /// + /// + /// + public AttributeDefinitionsBuilder Define(string name, string description) + { + return this; + } + /// /// Defines a new attribute with the given name and description /// @@ -20,7 +32,8 @@ public AttributeDefinitionsBuilder() /// /// /// - public AttributeDefinitionsBuilder Define(string name, string description) + public AttributeDefinitionsBuilder Include() + where TAttr : IAttribute { return this; } diff --git a/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/ModelDefinitionBuilder.cs b/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/ModelDefinitionBuilder.cs index c42f8636..93c7bc25 100644 --- a/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/ModelDefinitionBuilder.cs +++ b/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/ModelDefinitionBuilder.cs @@ -14,4 +14,24 @@ public ModelDefinitionBuilder() } + /// + /// Includes the attribute in the model definition + /// + /// + /// + public ModelDefinitionBuilder Include() + where TAttr : IAttribute + { + return this; + } + + /// + /// Builds the definition into a model with the given name + /// + /// + /// + public ModelDefinition Build(string name) + { + return new ModelDefinition(); + } } diff --git a/src/NexusMods.EventSourcing.SourceGenerator/AttributeData.cs b/src/NexusMods.EventSourcing.SourceGenerator/AttributeData.cs index c37f5276..14927af1 100644 --- a/src/NexusMods.EventSourcing.SourceGenerator/AttributeData.cs +++ b/src/NexusMods.EventSourcing.SourceGenerator/AttributeData.cs @@ -9,6 +9,8 @@ public class AttributeData public string Description { get; set; } = ""; public string Namespace { get; set; } = ""; public string Entity { get; set; } = ""; + + public bool IsInclude { get; set; } = false; } public class AttributeGroup diff --git a/src/NexusMods.EventSourcing.SourceGenerator/ModelGenerator.cs b/src/NexusMods.EventSourcing.SourceGenerator/ModelGenerator.cs index a8220d17..19f6ab2e 100644 --- a/src/NexusMods.EventSourcing.SourceGenerator/ModelGenerator.cs +++ b/src/NexusMods.EventSourcing.SourceGenerator/ModelGenerator.cs @@ -83,6 +83,7 @@ private string GenerateSource(IEnumerable attributeData) foreach (var attribute in group.Attributes) { + if (attribute.IsInclude) continue; sb.ClassComment(attribute.Description); var withoutQuotes = attribute.Name.Replace("\"", ""); var uniqueName = $"{attribute.Namespace}.{attribute.Entity}/{withoutQuotes}"; @@ -103,6 +104,7 @@ private string GenerateSource(IEnumerable attributeData) foreach (var attribute in group.Attributes) { + if (attribute.IsInclude) continue; var withoutQuotes = attribute.Name.Replace("\"", ""); sb.Line($"services.AddAttribute<{withoutQuotes}>();"); } @@ -137,6 +139,7 @@ private void EmitReadModel(AttributeGroup group, CodeWriter sb) foreach (var attribute in group.Attributes) { + if (attribute.IsInclude) continue; var withoutQuotes = attribute.Name.Replace("\"", ""); sb.ClassComment(attribute.Description); sb.Line($"public {attribute.AttributeType} {withoutQuotes} {{get; private set; }} = default!;"); @@ -151,6 +154,7 @@ private void EmitReadModel(AttributeGroup group, CodeWriter sb) sb.Line("{"); foreach (var attribute in group.Attributes) { + if (attribute.IsInclude) continue; var withoutQuotes = attribute.Name.Replace("\"", ""); sb.Line($"case {group.Namespace}.{group.Entity}.{withoutQuotes} a:"); sb.Line("{"); @@ -183,6 +187,7 @@ private void EmitReadModel(AttributeGroup group, CodeWriter sb) sb.Line("public Type[] Attributes => new Type[] {"); foreach (var attribute in group.Attributes) { + if (attribute.IsInclude) continue; var withoutQuotes = attribute.Name.Replace("\"", ""); sb.Line($"typeof({group.Namespace}.{group.Entity}.{withoutQuotes}),"); } @@ -217,43 +222,8 @@ private static IEnumerable FindAttributes(Compilation compilation var builderExpression = initializer.Value as InvocationExpressionSyntax; if (builderExpression != null) { - var defineCalls = builderExpression.DescendantNodesAndSelf() - .OfType() - .Where(invocation => invocation.Expression is MemberAccessExpressionSyntax - { - Name.Identifier.Text: "Define" - }); - - foreach (var defineCall in defineCalls) - { - var genericDefineCall = - ((MemberAccessExpressionSyntax)defineCall.Expression).Name as GenericNameSyntax; - if (genericDefineCall == null) - { - continue; - } - - var typeArgument = genericDefineCall.TypeArgumentList.Arguments.First(); - var typeSymbol = semanticModel.GetSymbolInfo(typeArgument).Symbol; - Console.WriteLine($"Type: {typeSymbol}"); - - - var args = defineCall.ArgumentList.Arguments.Select(argument => - { - var name = argument.Expression as LiteralExpressionSyntax; - - return name; - }).ToArray(); - - foundAttributes.Add(new AttributeData - { - Name = args[0]!.Token.ValueText, - Entity = declaredSymbol!.Name, - AttributeType = typeSymbol?.ToString() ?? "", - Description = args[1]!.Token.ValueText, - Namespace = declaredSymbol?.ContainingNamespace.ToString() ?? "" - }); - } + ExtractDefines(builderExpression, semanticModel, foundAttributes, declaredSymbol); + ExtractIncludes(builderExpression, semanticModel, foundAttributes, declaredSymbol); } } } @@ -263,4 +233,80 @@ private static IEnumerable FindAttributes(Compilation compilation return foundAttributes; } + + private static void ExtractDefines(InvocationExpressionSyntax builderExpression, SemanticModel semanticModel, + List foundAttributes, ISymbol? declaredSymbol) + { + var defineCalls = builderExpression.DescendantNodesAndSelf() + .OfType() + .Where(invocation => invocation.Expression is MemberAccessExpressionSyntax + { + Name.Identifier.Text: "Define" + }); + + foreach (var defineCall in defineCalls) + { + var genericDefineCall = + ((MemberAccessExpressionSyntax)defineCall.Expression).Name as GenericNameSyntax; + if (genericDefineCall == null) + { + continue; + } + + var typeArgument = genericDefineCall.TypeArgumentList.Arguments.First(); + var typeSymbol = semanticModel.GetSymbolInfo(typeArgument).Symbol; + + var args = defineCall.ArgumentList.Arguments.Select(argument => + { + var name = argument.Expression as LiteralExpressionSyntax; + + return name; + }).ToArray(); + + foundAttributes.Add(new AttributeData + { + Name = args[0]!.Token.ValueText, + Entity = declaredSymbol!.Name, + AttributeType = typeSymbol?.ToString() ?? "", + Description = args[1]!.Token.ValueText, + Namespace = declaredSymbol?.ContainingNamespace.ToString() ?? "" + }); + } + } + + private static void ExtractIncludes(InvocationExpressionSyntax builderExpression, SemanticModel semanticModel, + List foundAttributes, ISymbol? declaredSymbol) + { + var defineCalls = builderExpression.DescendantNodesAndSelf() + .OfType() + .Where(invocation => invocation.Expression is MemberAccessExpressionSyntax + { + Name.Identifier.Text: "Include" + }); + + foreach (var defineCall in defineCalls) + { + var genericDefineCall = + ((MemberAccessExpressionSyntax)defineCall.Expression).Name as GenericNameSyntax; + if (genericDefineCall == null) + { + continue; + } + + var typeArgument = genericDefineCall.TypeArgumentList.Arguments.First(); + var typeSymbol = semanticModel.GetSymbolInfo(typeArgument).Symbol; + + + foundAttributes.Add(new AttributeData + { + Name = typeArgument.ToString(), + AttributeType = typeSymbol?.ToString() ?? "", + IsInclude = true, + Namespace = declaredSymbol?.ContainingNamespace.ToString() ?? "", + Entity = declaredSymbol!.Name, + Description = "" + }); + + } + } } diff --git a/tests/NexusMods.EventSourcing.TestModel/Model/ArchiveFile.cs b/tests/NexusMods.EventSourcing.TestModel/Model/ArchiveFile.cs new file mode 100644 index 00000000..1f7833ed --- /dev/null +++ b/tests/NexusMods.EventSourcing.TestModel/Model/ArchiveFile.cs @@ -0,0 +1,15 @@ +using NexusMods.EventSourcing.Abstractions.ModelGeneration; + +namespace NexusMods.EventSourcing.TestModel.Model; + +[ModelDefinition] +public static partial class ArchiveFile +{ + public static AttributeDefinitions Attributes = new AttributeDefinitionsBuilder() + .Include() + .Include() + .Define("Index", "A index value for testing purposes") + .Define("ArchivePath", "The path of the archive file") + .Build(); + +} diff --git a/tests/NexusMods.EventSourcing.TestModel/Model/File.cs b/tests/NexusMods.EventSourcing.TestModel/Model/File.cs index d57e6406..87244503 100644 --- a/tests/NexusMods.EventSourcing.TestModel/Model/File.cs +++ b/tests/NexusMods.EventSourcing.TestModel/Model/File.cs @@ -10,5 +10,4 @@ public static partial class File .Define("Hash", "The hash of the file") .Define("Index", "A index value for testing purposes") .Build(); - } diff --git a/tests/NexusMods.EventSourcing.Tests/DbTests.cs b/tests/NexusMods.EventSourcing.Tests/DbTests.cs index b8a6b530..da7dc04e 100644 --- a/tests/NexusMods.EventSourcing.Tests/DbTests.cs +++ b/tests/NexusMods.EventSourcing.Tests/DbTests.cs @@ -90,4 +90,32 @@ public void DbIsImmutable() } } + [Fact] + public void ReadModelsCanHaveExtraAttributes() + { + var tx = Connection.BeginTransaction(); + var fileId = tx.TempId(); + File.Path.Assert(fileId, "C:\\test.txt", tx); + File.Hash.Assert(fileId, 0xDEADBEEF, tx); + File.Index.Assert(fileId, 77, tx); + ArchiveFile.Index.Assert(fileId, 42, tx); + ArchiveFile.ArchivePath.Assert(fileId, "C:\\archive.zip", tx); + var result = tx.Commit(); + + var realId = result[fileId]; + var db = Connection.Db; + var readModel = db.Get([realId]).First(); + readModel.Path.Should().Be("C:\\test.txt"); + readModel.Hash.Should().Be(0xDEADBEEF); + readModel.Index.Should().Be(77); + + var archiveReadModel = db.Get([realId]).First(); + archiveReadModel.Path.Should().Be("C:\\test.txt"); + archiveReadModel.Hash.Should().Be(0xDEADBEEF); + archiveReadModel.Index.Should().Be(42); + archiveReadModel.ArchivePath.Should().Be("C:\\archive.zip"); + + + } + } From 07ff6ce11bd47ab52de0d109e40051717fb73b10 Mon Sep 17 00:00:00 2001 From: halgari Date: Wed, 14 Feb 2024 14:45:24 -0700 Subject: [PATCH 2/8] WIP docs --- docs/AttributeDefinitions.md | 159 +++++++++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 docs/AttributeDefinitions.md diff --git a/docs/AttributeDefinitions.md b/docs/AttributeDefinitions.md new file mode 100644 index 00000000..8236bcf0 --- /dev/null +++ b/docs/AttributeDefinitions.md @@ -0,0 +1,159 @@ +--- +hide: + - toc +--- + + +## Attribute Definitions +The datoms stored in the store require typed attributes with unique name and serializer tagging. Storing this information in +a way that provides typed access to attribues is strangely difficult. There are several ways to tackle this issue, which we will talk about here. + +### Symbolic Names +The simplest approach is to do what Datomic does and use symbolic names for attributes. For example a `:loadout/name` attribute +would be registered as a string type: + +```csharp + + System.RegisterAttribute("loadout/name"); + var nameSymbol = Symbol.Intern("loadout/name"); + + tx.Add(eid, nameSymbol, "My Loadout"); +``` + +The problem with this approach is there is nothing stopping someone from using the wrong type for an attribute. The error happens +at runtime instead of at compile time. In addition loading values from the store may result in boxing unless care is taken. + +This approach does allow for the use of fairly dynamic queries however. + +```csharp +var query = from e in db.Entities + where e[Loadout_Name] == "My Loadout" + select e.Pull(Loadout_Name, Loadout_Version); + +// What is the type of query result? +var results = query.ToList(); +``` + +!!!info + Dapper uses this approach, as does Datomic, but it assumes a dynamic query result. This is not a bad thing, but it does +reduce the ability to use the type system to catch errors. + +If we want to predefine a query model, we end up with something even more complex + +```csharp + +interface QueryResult +{ + string Name { get; } + int Version { get; } +} + + +var results = from e in db.Entities() + where e.Name == "My Loadout" + select e; +```` + +The question here is how we map the symbolic names to the result type. We can use attributes, but attributes must have +constant values as arguments, so we can't symbolc names and must use strings or some sort of `nameof` expression. + +```csharp +interface QueryResult +{ + [From("loadout/name")] + string Name { get; } + [From(nameof(NexusMods.Model.Loadout_Version))] + int Version { get; } +} +``` + +Since `nameof` only names the specific type, we have to give it a fully qualified name. This is also suboptimal. + +### Attributes as Types + +Another approach would be to use the type system to define the attributes. This has the advantage of being able to use the types +to provide strict type checking, and we can use attribues to provide the symbolic names. + +```csharp +namespace Loadout { + public class Name : Attribute(); + public class Version : Attribute(); +} + +public class QueryResult { + [From] + public string Name { get; } + [From] + public int Version { get; } +} +``` + +Unfortunately in this approach we have to make sure to use the correct type for the getter in the read model. There's nothing +stopping us from accidentally defining `Name` as an `int` for example. This would not result in a compile time error. If we pre-register +all our read models (like `QueryResult`) we can use reflection to check the types and at least we get a startup time error. + +Another problem with this approach is that C#'s inference system is not good at resolving complex constraints, for example: + +```csharp + +// This requires us to know at usage time that Name is a string attribute, and to make sure that the type is correct. +tx.Add("foo"); + +// What we can do, is put the `.Bar` method in a static extension method + +tx.Add("foo"); + +public static class AttributeExtensions +{ + public static void Add(this Transaction tx, T attribute, string value) + where T : Attribute + { + tx.Add(attribute, value); + } + + public static void Add(this Transaction tx, T attribute, float value) + where T : Attribute + { + tx.Add(attribute, value); + } +} +``` + +This works quite well, and we only need to perform the operation once per attribute value type. After we do this we can easily define +models that use the attributes. + +```csharp + +record ReadModel +{ + [From] + public string Name { get; } + [From] + public int Version { get; } +} + +record WriteModel +{ + [From] + public required string Name { get; init;} + [From] + public required int Version { get; init;} +} + +/// Define a model that responds to new transactions and fires INotifyPropertyChanged events +public class ActiveModel : ActiveModel +{ + [From] + public string Name { get; set; } + [From] + public int Version { get; set; } +} + +/// Perform an ad-hoc query +/// Find files with the path of "c:\temp\foo.txt" and look up the name and version of the mods that contain them. +var results = from f in db.Where(@"c:\temp\foo.txt") + from m in db.Where(f.EntityId) + select db.Pull(m.EntityId); +``` + + From 9aaa105da169968604c1981be3e0888238ea9319 Mon Sep 17 00:00:00 2001 From: halgari Date: Thu, 15 Feb 2024 10:53:15 -0700 Subject: [PATCH 3/8] Can read models as well now --- NexusMods.EventSourcing.sln | 7 - .../NexusMods.EventSourcing.Benchmarks.csproj | 1 - .../DependencyInjectionExtensions.cs | 14 +- .../IAttribute.cs | 11 +- .../IDatomStore.cs | 11 ++ .../IDb.cs | 1 + .../IEntityIterator.cs | 32 +++- .../IReadModel.cs | 22 --- .../IReadModelBuilder.cs | 24 --- .../IReadModelFactory.cs | 23 --- .../ITransaction.cs | 21 +++ .../ModelGeneration/AttributeDefinitions.cs | 6 - .../AttributeDefinitionsBuilder.cs | 50 ------ .../ModelGeneration/ModelDefinition.cs | 9 -- .../ModelDefinitionAttribute.cs | 9 -- .../ModelGeneration/ModelDefinitionBuilder.cs | 37 ----- .../Models/AReadModel.cs | 30 ++++ .../Models/FromAttribute.cs | 17 +++ .../Models/IFromAttribute.cs | 14 ++ .../Models/IReadModel.cs | 11 ++ .../ScalarAttribute.cs | 13 +- .../Symbol.cs | 3 +- .../TransactionExtensionMethods.cs | 6 + .../AttributeRegistry.cs | 22 ++- .../Indexes/EATVIndex.cs | 21 ++- .../RocksDBDatomStore.cs | 6 + src/NexusMods.EventSourcing/Connection.cs | 10 +- src/NexusMods.EventSourcing/Db.cs | 20 +-- src/NexusMods.EventSourcing/ModelReflector.cs | 143 ++++++++++++++++++ src/NexusMods.EventSourcing/Transaction.cs | 19 +++ .../ADatomStoreTest.cs | 4 +- .../DatomStoreSetupTests.cs | 5 +- .../Model/ArchiveFile.cs | 15 -- .../Model/Attributes/File.cs | 13 ++ .../Model/File.cs | 30 +++- .../Model/Loadout.cs | 12 -- .../Model/Mod.cs | 13 -- .../NexusMods.EventSourcing.TestModel.csproj | 1 - .../Services.cs | 17 ++- .../AEventSourcingTest.cs | 4 +- .../NexusMods.EventSourcing.Tests/DbTests.cs | 24 +-- 41 files changed, 443 insertions(+), 308 deletions(-) delete mode 100644 src/NexusMods.EventSourcing.Abstractions/IReadModel.cs delete mode 100644 src/NexusMods.EventSourcing.Abstractions/IReadModelBuilder.cs delete mode 100644 src/NexusMods.EventSourcing.Abstractions/IReadModelFactory.cs delete mode 100644 src/NexusMods.EventSourcing.Abstractions/ModelGeneration/AttributeDefinitions.cs delete mode 100644 src/NexusMods.EventSourcing.Abstractions/ModelGeneration/AttributeDefinitionsBuilder.cs delete mode 100644 src/NexusMods.EventSourcing.Abstractions/ModelGeneration/ModelDefinition.cs delete mode 100644 src/NexusMods.EventSourcing.Abstractions/ModelGeneration/ModelDefinitionAttribute.cs delete mode 100644 src/NexusMods.EventSourcing.Abstractions/ModelGeneration/ModelDefinitionBuilder.cs create mode 100644 src/NexusMods.EventSourcing.Abstractions/Models/AReadModel.cs create mode 100644 src/NexusMods.EventSourcing.Abstractions/Models/FromAttribute.cs create mode 100644 src/NexusMods.EventSourcing.Abstractions/Models/IFromAttribute.cs create mode 100644 src/NexusMods.EventSourcing.Abstractions/Models/IReadModel.cs create mode 100644 src/NexusMods.EventSourcing.Abstractions/TransactionExtensionMethods.cs create mode 100644 src/NexusMods.EventSourcing/ModelReflector.cs delete mode 100644 tests/NexusMods.EventSourcing.TestModel/Model/ArchiveFile.cs create mode 100644 tests/NexusMods.EventSourcing.TestModel/Model/Attributes/File.cs delete mode 100644 tests/NexusMods.EventSourcing.TestModel/Model/Loadout.cs delete mode 100644 tests/NexusMods.EventSourcing.TestModel/Model/Mod.cs diff --git a/NexusMods.EventSourcing.sln b/NexusMods.EventSourcing.sln index add754d6..31b0183c 100644 --- a/NexusMods.EventSourcing.sln +++ b/NexusMods.EventSourcing.sln @@ -35,8 +35,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.EventSourcing.Tes EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.EventSourcing.DatomStore.Tests", "tests\NexusMods.EventSourcing.DatomStore.Tests\NexusMods.EventSourcing.DatomStore.Tests.csproj", "{81CCE07D-818D-4153-8486-5D2A860C4D9D}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.EventSourcing.SourceGenerator", "src\NexusMods.EventSourcing.SourceGenerator\NexusMods.EventSourcing.SourceGenerator.csproj", "{F2A6F6B9-5D36-4416-BDD8-C7D30EE3ED4A}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.EventSourcing.Tests", "tests\NexusMods.EventSourcing.Tests\NexusMods.EventSourcing.Tests.csproj", "{07E2C578-8644-474D-8F07-B25CFEB28408}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.EventSourcing.Benchmarks", "benchmarks\NexusMods.EventSourcing.Benchmarks\NexusMods.EventSourcing.Benchmarks.csproj", "{930B3AB7-56EA-48D6-B603-24D79C7DD00A}" @@ -52,7 +50,6 @@ Global {F2C1FB09-D01D-4E8B-B6BE-B548AB00187B} = {0377EBE6-F147-4233-86AD-32C821B9567E} {EC1570A4-18B9-4A76-84FF-275BAA76A357} = {6ED01F9D-5E12-4EB2-9601-64A2ADC719DE} {81CCE07D-818D-4153-8486-5D2A860C4D9D} = {6ED01F9D-5E12-4EB2-9601-64A2ADC719DE} - {F2A6F6B9-5D36-4416-BDD8-C7D30EE3ED4A} = {0377EBE6-F147-4233-86AD-32C821B9567E} {07E2C578-8644-474D-8F07-B25CFEB28408} = {6ED01F9D-5E12-4EB2-9601-64A2ADC719DE} {930B3AB7-56EA-48D6-B603-24D79C7DD00A} = {72AFE85F-8C12-436A-894E-638ED2C92A76} EndGlobalSection @@ -77,10 +74,6 @@ Global {81CCE07D-818D-4153-8486-5D2A860C4D9D}.Debug|Any CPU.Build.0 = Debug|Any CPU {81CCE07D-818D-4153-8486-5D2A860C4D9D}.Release|Any CPU.ActiveCfg = Release|Any CPU {81CCE07D-818D-4153-8486-5D2A860C4D9D}.Release|Any CPU.Build.0 = Release|Any CPU - {F2A6F6B9-5D36-4416-BDD8-C7D30EE3ED4A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {F2A6F6B9-5D36-4416-BDD8-C7D30EE3ED4A}.Debug|Any CPU.Build.0 = Debug|Any CPU - {F2A6F6B9-5D36-4416-BDD8-C7D30EE3ED4A}.Release|Any CPU.ActiveCfg = Release|Any CPU - {F2A6F6B9-5D36-4416-BDD8-C7D30EE3ED4A}.Release|Any CPU.Build.0 = Release|Any CPU {07E2C578-8644-474D-8F07-B25CFEB28408}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {07E2C578-8644-474D-8F07-B25CFEB28408}.Debug|Any CPU.Build.0 = Debug|Any CPU {07E2C578-8644-474D-8F07-B25CFEB28408}.Release|Any CPU.ActiveCfg = Release|Any CPU diff --git a/benchmarks/NexusMods.EventSourcing.Benchmarks/NexusMods.EventSourcing.Benchmarks.csproj b/benchmarks/NexusMods.EventSourcing.Benchmarks/NexusMods.EventSourcing.Benchmarks.csproj index 8652ecf5..374c8591 100644 --- a/benchmarks/NexusMods.EventSourcing.Benchmarks/NexusMods.EventSourcing.Benchmarks.csproj +++ b/benchmarks/NexusMods.EventSourcing.Benchmarks/NexusMods.EventSourcing.Benchmarks.csproj @@ -14,7 +14,6 @@ - diff --git a/src/NexusMods.EventSourcing.Abstractions/DependencyInjectionExtensions.cs b/src/NexusMods.EventSourcing.Abstractions/DependencyInjectionExtensions.cs index aa3444f4..913bf18a 100644 --- a/src/NexusMods.EventSourcing.Abstractions/DependencyInjectionExtensions.cs +++ b/src/NexusMods.EventSourcing.Abstractions/DependencyInjectionExtensions.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.DependencyInjection; +using NexusMods.EventSourcing.Abstractions.Models; namespace NexusMods.EventSourcing.Abstractions; @@ -33,17 +34,4 @@ public static IServiceCollection AddValueSerializer(this IServ services.AddSingleton(); return services; } - - /// - /// Registers the specified read model factory type with the service collection. - /// - /// - /// - /// - public static IServiceCollection AddReadModelFactory(this IServiceCollection services) - where TReadModelFactory : class, IReadModelFactory - { - services.AddSingleton(); - return services; - } } diff --git a/src/NexusMods.EventSourcing.Abstractions/IAttribute.cs b/src/NexusMods.EventSourcing.Abstractions/IAttribute.cs index 0dbc4db0..ee0be7a3 100644 --- a/src/NexusMods.EventSourcing.Abstractions/IAttribute.cs +++ b/src/NexusMods.EventSourcing.Abstractions/IAttribute.cs @@ -50,7 +50,7 @@ public interface IAttribute /// Typed variant of IAttribute /// /// -public interface IAttribute : IAttribute +public interface IAttribute : IAttribute { /// @@ -60,4 +60,13 @@ public interface IAttribute : IAttribute /// public TVal Read(ReadOnlySpan buffer); + + /// + /// Creates a new assertion datom for the given entity and value + /// + /// + /// + /// + public static abstract void Add(ITransaction tx, EntityId entity, TVal value); + } diff --git a/src/NexusMods.EventSourcing.Abstractions/IDatomStore.cs b/src/NexusMods.EventSourcing.Abstractions/IDatomStore.cs index 717fd8d3..2fac6cd8 100644 --- a/src/NexusMods.EventSourcing.Abstractions/IDatomStore.cs +++ b/src/NexusMods.EventSourcing.Abstractions/IDatomStore.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq.Expressions; namespace NexusMods.EventSourcing.Abstractions; @@ -31,4 +32,14 @@ public interface IDatomStore : IDisposable /// /// void RegisterAttributes(IEnumerable newAttrs); + + /// + /// Gets the attributeId for the given attribute. And returns an expression that reads the attribute + /// value from the expression valueSpan. + /// + /// + /// + /// + /// + Expression GetValueReadExpression(Type attribute, Expression valueSpan, out ulong attributeId); } diff --git a/src/NexusMods.EventSourcing.Abstractions/IDb.cs b/src/NexusMods.EventSourcing.Abstractions/IDb.cs index 24a5df91..ab0f0d03 100644 --- a/src/NexusMods.EventSourcing.Abstractions/IDb.cs +++ b/src/NexusMods.EventSourcing.Abstractions/IDb.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using NexusMods.EventSourcing.Abstractions.Models; namespace NexusMods.EventSourcing.Abstractions; diff --git a/src/NexusMods.EventSourcing.Abstractions/IEntityIterator.cs b/src/NexusMods.EventSourcing.Abstractions/IEntityIterator.cs index 6acc1f1c..a90c0460 100644 --- a/src/NexusMods.EventSourcing.Abstractions/IEntityIterator.cs +++ b/src/NexusMods.EventSourcing.Abstractions/IEntityIterator.cs @@ -1,9 +1,12 @@ -namespace NexusMods.EventSourcing.Abstractions; +using System; +using NexusMods.EventSourcing.Abstractions.Models; + +namespace NexusMods.EventSourcing.Abstractions; /// /// Represents an iterator over a set of datoms. /// -public interface IEntityIterator +public interface IEntityIterator : IDisposable { /// /// Move to the next datom for the current entity @@ -12,10 +15,10 @@ public interface IEntityIterator public bool Next(); /// - /// Sets the current entity id, this implicitly resets the iterator. + /// Seeks to the data for the given Entity Id, this implicitly resets the iterator. /// /// - public void SetEntityId(EntityId entityId); + public void SeekTo(EntityId entityId); /// /// Gets the current datom as a distinct value. @@ -23,9 +26,22 @@ public interface IEntityIterator public IDatom Current { get; } /// - /// Sends the current datom to the read model. + /// Gets the current datom's value + /// + /// + /// + /// + public TValue GetValue() + where TAttribute : IAttribute; + + /// + /// Gets the current datom's attribute id + /// + public ulong AttributeId { get; } + + /// + /// Gets the current datom's value as a span, valid until the next call to Next() + /// or SetEntityId() /// - /// - /// - public void SetOn(TModel model) where TModel : IReadModel; + public ReadOnlySpan ValueSpan { get; } } diff --git a/src/NexusMods.EventSourcing.Abstractions/IReadModel.cs b/src/NexusMods.EventSourcing.Abstractions/IReadModel.cs deleted file mode 100644 index 45187d6f..00000000 --- a/src/NexusMods.EventSourcing.Abstractions/IReadModel.cs +++ /dev/null @@ -1,22 +0,0 @@ -using System; - -namespace NexusMods.EventSourcing.Abstractions; - -/// -/// A read model is a set of attributes grouped together with a common entity id -/// -public interface IReadModel -{ - /// - /// The unique identifier of the entity in the read model - /// - public EntityId Id { get; } - - /// - /// Sets the value of an attribute in the model - /// - /// - /// - /// - public void Set(IAttribute attribute, ReadOnlySpan value); -} diff --git a/src/NexusMods.EventSourcing.Abstractions/IReadModelBuilder.cs b/src/NexusMods.EventSourcing.Abstractions/IReadModelBuilder.cs deleted file mode 100644 index 954ea499..00000000 --- a/src/NexusMods.EventSourcing.Abstractions/IReadModelBuilder.cs +++ /dev/null @@ -1,24 +0,0 @@ -using System; - -namespace NexusMods.EventSourcing.Abstractions; - -/// -/// A push-based system for building up a IReadModel state -/// -public interface IReadModelBuilder -{ - /// - /// Sets the value of the given attribute to the given value - /// - /// - /// - /// - public void Set(IAttribute attr, ReadOnlySpan span); - - /// - /// Builds the collected data into a IReadModel - /// - /// - public IReadModel Build(EntityId id); -} - diff --git a/src/NexusMods.EventSourcing.Abstractions/IReadModelFactory.cs b/src/NexusMods.EventSourcing.Abstractions/IReadModelFactory.cs deleted file mode 100644 index 7e671d49..00000000 --- a/src/NexusMods.EventSourcing.Abstractions/IReadModelFactory.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System; - -namespace NexusMods.EventSourcing.Abstractions; - -/// -/// A factory for creating read models. The attribute list is used to optimize -/// reading so that only the requested attributes have to be loaded from the store. -/// -public interface IReadModelFactory -{ - public Type ModelType { get; } - - /// - /// A collection of all the attributes in the model. - /// - public Type[] Attributes { get; } - - /// - /// Creates a new instance of the read model - /// - /// - public IReadModel Create(EntityId id); -} diff --git a/src/NexusMods.EventSourcing.Abstractions/ITransaction.cs b/src/NexusMods.EventSourcing.Abstractions/ITransaction.cs index d1971453..efa2de60 100644 --- a/src/NexusMods.EventSourcing.Abstractions/ITransaction.cs +++ b/src/NexusMods.EventSourcing.Abstractions/ITransaction.cs @@ -1,4 +1,5 @@ using System; +using NexusMods.EventSourcing.Abstractions.Models; namespace NexusMods.EventSourcing.Abstractions; @@ -15,6 +16,26 @@ public interface ITransaction /// void Add(IDatom datom); + + /// + /// Adds a new read model to the transaction, the datoms are extracted from the read model + /// as asserts for each property with the FromAttribute + /// + /// + TReadModel Add(TReadModel model) + where TReadModel : IReadModel; + + /// + /// Adds a new datom to the transaction + /// + /// + /// + /// + /// + /// + void Add(EntityId entityId, TVal val, bool isAssert = true) + where TAttribute : IAttribute; + /// /// Commits the transaction /// diff --git a/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/AttributeDefinitions.cs b/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/AttributeDefinitions.cs deleted file mode 100644 index 8bce64f4..00000000 --- a/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/AttributeDefinitions.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace NexusMods.EventSourcing.Abstractions.ModelGeneration; - -public class AttributeDefinitions -{ - public static AttributeDefinitions Instance = new(); -} diff --git a/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/AttributeDefinitionsBuilder.cs b/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/AttributeDefinitionsBuilder.cs deleted file mode 100644 index 59cb847d..00000000 --- a/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/AttributeDefinitionsBuilder.cs +++ /dev/null @@ -1,50 +0,0 @@ -namespace NexusMods.EventSourcing.Abstractions.ModelGeneration; - -/// -/// Placeholder for a model definition for source generators -/// -public class AttributeDefinitionsBuilder -{ - /// - /// Placeholder for a model definition for source generators - /// - public AttributeDefinitionsBuilder() - { - - } - - /// - /// Defines a new attribute with the given name and description - /// - /// - /// - /// - /// - public AttributeDefinitionsBuilder Define(string name, string description) - { - return this; - } - - /// - /// Defines a new attribute with the given name and description - /// - /// - /// - /// - /// - public AttributeDefinitionsBuilder Include() - where TAttr : IAttribute - { - return this; - } - - /// - /// Builds the final model definition - /// - /// - public AttributeDefinitions Build() - { - return AttributeDefinitions.Instance; - } - -} diff --git a/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/ModelDefinition.cs b/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/ModelDefinition.cs deleted file mode 100644 index 2d0135fd..00000000 --- a/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/ModelDefinition.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace NexusMods.EventSourcing.Abstractions.ModelGeneration; - -/// -/// Placeholder class for the final result of the model generation -/// -public class ModelDefinition -{ - -} diff --git a/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/ModelDefinitionAttribute.cs b/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/ModelDefinitionAttribute.cs deleted file mode 100644 index a4798c91..00000000 --- a/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/ModelDefinitionAttribute.cs +++ /dev/null @@ -1,9 +0,0 @@ -using System; - -namespace NexusMods.EventSourcing.Abstractions.ModelGeneration; - -/// -/// Marks a class as a model definition -/// -[AttributeUsage(AttributeTargets.Class)] -public class ModelDefinitionAttribute : Attribute; diff --git a/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/ModelDefinitionBuilder.cs b/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/ModelDefinitionBuilder.cs deleted file mode 100644 index 93c7bc25..00000000 --- a/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/ModelDefinitionBuilder.cs +++ /dev/null @@ -1,37 +0,0 @@ -namespace NexusMods.EventSourcing.Abstractions.ModelGeneration; - -/// -/// Placeholder class for generating model definitions for source generators -/// -public class ModelDefinitionBuilder -{ - /// - /// Placeholder class for generating model definitions for source generators - /// - public ModelDefinitionBuilder() - { - - } - - - /// - /// Includes the attribute in the model definition - /// - /// - /// - public ModelDefinitionBuilder Include() - where TAttr : IAttribute - { - return this; - } - - /// - /// Builds the definition into a model with the given name - /// - /// - /// - public ModelDefinition Build(string name) - { - return new ModelDefinition(); - } -} diff --git a/src/NexusMods.EventSourcing.Abstractions/Models/AReadModel.cs b/src/NexusMods.EventSourcing.Abstractions/Models/AReadModel.cs new file mode 100644 index 00000000..3bc59f1b --- /dev/null +++ b/src/NexusMods.EventSourcing.Abstractions/Models/AReadModel.cs @@ -0,0 +1,30 @@ +namespace NexusMods.EventSourcing.Abstractions.Models; + +/// +/// Base class for all read models. +/// +/// +public abstract class AReadModel : IReadModel +where TOuter : AReadModel, IReadModel +{ + /// + /// Creates a new read model with a temporary id + /// + /// + protected AReadModel(ITransaction? tx) + { + if (tx is null) return; + Id = tx.TempId(); + tx.Add((TOuter)this); + } + + internal AReadModel(EntityId id) + { + Id = id; + } + + /// + /// The base identifier for the entity. + /// + public EntityId Id { get; internal set; } +} diff --git a/src/NexusMods.EventSourcing.Abstractions/Models/FromAttribute.cs b/src/NexusMods.EventSourcing.Abstractions/Models/FromAttribute.cs new file mode 100644 index 00000000..b4eaadc2 --- /dev/null +++ b/src/NexusMods.EventSourcing.Abstractions/Models/FromAttribute.cs @@ -0,0 +1,17 @@ +using System; + +namespace NexusMods.EventSourcing.Abstractions.Models; + +/// +/// Marks a property as being derived from an attribute. +/// +/// +[AttributeUsage(AttributeTargets.Property)] +public class FromAttribute : Attribute, IFromAttribute + where TAttribute : IAttribute +{ + /// + /// Gets the type of the attribute. + /// + public Type AttributeType => typeof(TAttribute); +} diff --git a/src/NexusMods.EventSourcing.Abstractions/Models/IFromAttribute.cs b/src/NexusMods.EventSourcing.Abstractions/Models/IFromAttribute.cs new file mode 100644 index 00000000..b907871e --- /dev/null +++ b/src/NexusMods.EventSourcing.Abstractions/Models/IFromAttribute.cs @@ -0,0 +1,14 @@ +using System; + +namespace NexusMods.EventSourcing.Abstractions.Models; + +/// +/// Base interface for all from attributes. +/// +public interface IFromAttribute +{ + /// + /// The attribute tag of this from attribute. + /// + public Type AttributeType { get; } +} diff --git a/src/NexusMods.EventSourcing.Abstractions/Models/IReadModel.cs b/src/NexusMods.EventSourcing.Abstractions/Models/IReadModel.cs new file mode 100644 index 00000000..b6f3b1bc --- /dev/null +++ b/src/NexusMods.EventSourcing.Abstractions/Models/IReadModel.cs @@ -0,0 +1,11 @@ +namespace NexusMods.EventSourcing.Abstractions.Models; + +/// +/// Base interface for all read models. The AReadModel class takes a generic parameter +/// which makes it hard to use as a base class for all read models in cases where the user +/// doesn't care about the type of the read model. +/// +public interface IReadModel +{ + public EntityId Id { get; } +} diff --git a/src/NexusMods.EventSourcing.Abstractions/ScalarAttribute.cs b/src/NexusMods.EventSourcing.Abstractions/ScalarAttribute.cs index 61f8906d..2c71b428 100644 --- a/src/NexusMods.EventSourcing.Abstractions/ScalarAttribute.cs +++ b/src/NexusMods.EventSourcing.Abstractions/ScalarAttribute.cs @@ -1,6 +1,5 @@ using System; using System.Runtime.CompilerServices; -using System.Transactions; namespace NexusMods.EventSourcing.Abstractions; @@ -8,6 +7,7 @@ namespace NexusMods.EventSourcing.Abstractions; /// Interface for a specific attribute /// /// +/// public class ScalarAttribute : IAttribute where TAttribute : IAttribute { @@ -17,8 +17,10 @@ public class ScalarAttribute : IAttribute /// Create a new attribute /// /// - protected ScalarAttribute(string uniqueName) + protected ScalarAttribute(string uniqueName = "") { + if (uniqueName == "") + uniqueName = typeof(TAttribute).FullName!; Id = Symbol.Intern(uniqueName); } @@ -37,6 +39,13 @@ public TValueType Read(ReadOnlySpan buffer) return val; } + + /// + public static void Add(ITransaction tx, EntityId entity, TValueType value) + { + tx.Add(entity, value); + } + public void SetSerializer(IValueSerializer serializer) { if (serializer is not IValueSerializer valueSerializer) diff --git a/src/NexusMods.EventSourcing.Abstractions/Symbol.cs b/src/NexusMods.EventSourcing.Abstractions/Symbol.cs index b50e8781..daabea2b 100644 --- a/src/NexusMods.EventSourcing.Abstractions/Symbol.cs +++ b/src/NexusMods.EventSourcing.Abstractions/Symbol.cs @@ -14,8 +14,9 @@ public class Symbol /// private Symbol(string nsAndName) { + nsAndName = nsAndName.Replace("+", "."); Id = nsAndName; - var splitOn = nsAndName.LastIndexOf('/'); + var splitOn = nsAndName.LastIndexOf('.'); Name = nsAndName[(splitOn + 1)..]; Namespace = nsAndName[..splitOn]; } diff --git a/src/NexusMods.EventSourcing.Abstractions/TransactionExtensionMethods.cs b/src/NexusMods.EventSourcing.Abstractions/TransactionExtensionMethods.cs new file mode 100644 index 00000000..e4281922 --- /dev/null +++ b/src/NexusMods.EventSourcing.Abstractions/TransactionExtensionMethods.cs @@ -0,0 +1,6 @@ +namespace NexusMods.EventSourcing.Abstractions; + +public static class TransactionExtensionMethods +{ + +} diff --git a/src/NexusMods.EventSourcing.DatomStore/AttributeRegistry.cs b/src/NexusMods.EventSourcing.DatomStore/AttributeRegistry.cs index 6804158f..5325a7bf 100644 --- a/src/NexusMods.EventSourcing.DatomStore/AttributeRegistry.cs +++ b/src/NexusMods.EventSourcing.DatomStore/AttributeRegistry.cs @@ -2,7 +2,9 @@ using System.Buffers; using System.Collections.Generic; using System.Linq; +using System.Linq.Expressions; using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Abstractions.Models; namespace NexusMods.EventSourcing.DatomStore; @@ -84,11 +86,23 @@ public IDatom ReadDatom(ref KeyHeader header, ReadOnlySpan valueSpan) return attribute.Read(header.Entity, header.Tx, header.IsAssert, valueSpan); } - public void SetOn(TModel model, ref KeyHeader key, ReadOnlySpan sliceFast) where TModel : IReadModel + public TValue ReadValue(ref KeyHeader currentHeader, ReadOnlySpan currentValue) + where TAttribute : IAttribute { - var attrId = key.AttributeId; + var attrId = currentHeader.AttributeId; var dbAttribute = _dbAttributesByEntityId[attrId]; - var attribute = _attributesById[dbAttribute.UniqueId]; - model.Set(attribute, sliceFast); + var attribute = (TAttribute)_attributesById[dbAttribute.UniqueId]; + return attribute.Read(currentValue); + } + + public Expression GetReadExpression(Type attributeType, Expression valueSpan, out ulong attributeId) + { + var attr = _attributesByType[attributeType]; + attributeId = _dbAttributesByUniqueId[attr.Id].AttrEntityId; + var serializer = _valueSerializersByNativeType[attr.ValueType]; + var readMethod = serializer.GetType().GetMethod("Read")!; + var valueExpr = Expression.Parameter(attr.ValueType, "retVal"); + var readExpression = Expression.Call(Expression.Constant(serializer), readMethod, valueSpan, valueExpr); + return Expression.Block([valueExpr], readExpression, valueExpr); } } diff --git a/src/NexusMods.EventSourcing.DatomStore/Indexes/EATVIndex.cs b/src/NexusMods.EventSourcing.DatomStore/Indexes/EATVIndex.cs index dbd488db..6b4e23a1 100644 --- a/src/NexusMods.EventSourcing.DatomStore/Indexes/EATVIndex.cs +++ b/src/NexusMods.EventSourcing.DatomStore/Indexes/EATVIndex.cs @@ -1,6 +1,7 @@ using System; using System.Runtime.InteropServices; using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Abstractions.Models; using Reloaded.Memory.Extensions; using RocksDbSharp; @@ -76,7 +77,7 @@ public EATVIterator(ulong txId, AttributeRegistry registry, EATVIndex idx) } - public void SetEntityId(EntityId entityId) + public void SeekTo(EntityId entityId) { _key->Entity = entityId.Value; _key->AttributeId = ulong.MaxValue; @@ -95,14 +96,28 @@ public IDatom Current } } - public void SetOn(TModel model) where TModel : IReadModel + public TValue GetValue() + where TAttribute : IAttribute { var span = _iterator.GetKeySpan(); var currentHeader = MemoryMarshal.AsRef(span); var currentValue = span.SliceFast(KeyHeader.Size); - _registry.SetOn(model, ref currentHeader, currentValue); + return _registry.ReadValue(ref currentHeader, currentValue); + } + public ulong AttributeId + { + get + { + var span = _iterator.GetKeySpan(); + var currentHeader = MemoryMarshal.AsRef(span); + return currentHeader.AttributeId; + } + } + + public ReadOnlySpan ValueSpan => _iterator.GetKeySpan().SliceFast(KeyHeader.Size); + public bool Next() { TOP: diff --git a/src/NexusMods.EventSourcing.DatomStore/RocksDBDatomStore.cs b/src/NexusMods.EventSourcing.DatomStore/RocksDBDatomStore.cs index bb6b40df..a74c31af 100644 --- a/src/NexusMods.EventSourcing.DatomStore/RocksDBDatomStore.cs +++ b/src/NexusMods.EventSourcing.DatomStore/RocksDBDatomStore.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Linq.Expressions; using System.Runtime.InteropServices; using Microsoft.Extensions.Logging; using NexusMods.EventSourcing.Abstractions; @@ -136,6 +137,11 @@ public void RegisterAttributes(IEnumerable newAttrs) _registry.Populate(newAttrs.ToArray()); } + public Expression GetValueReadExpression(Type attribute, Expression valueSpan, out ulong attributeId) + { + return _registry.GetReadExpression(attribute, valueSpan, out attributeId); + } + public void Dispose() { _db.Dispose(); diff --git a/src/NexusMods.EventSourcing/Connection.cs b/src/NexusMods.EventSourcing/Connection.cs index 6a9065d5..1b8f5474 100644 --- a/src/NexusMods.EventSourcing/Connection.cs +++ b/src/NexusMods.EventSourcing/Connection.cs @@ -16,16 +16,16 @@ public class Connection : IConnection private ulong _nextEntityId = Ids.MinId(Ids.Partition.Entity); private readonly IDatomStore _store; private readonly IAttribute[] _declaredAttributes; - private readonly Dictionary _factories; + internal readonly ModelReflector ModelReflector; /// /// Main connection class, co-ordinates writes and immutable reads /// - public Connection(IDatomStore store, IEnumerable declaredAttributes, IEnumerable serializers, IEnumerable factories) + public Connection(IDatomStore store, IEnumerable declaredAttributes, IEnumerable serializers) { _store = store; _declaredAttributes = declaredAttributes.ToArray(); - _factories = factories.ToDictionary(f => f.ModelType); + ModelReflector = new ModelReflector(store); AddMissingAttributes(serializers); } @@ -68,7 +68,7 @@ private IEnumerable ExistingAttributes() var entIterator = _store.EntityIterator(tx); while (attrIterator.Next()) { - entIterator.SetEntityId(attrIterator.EntityId); + entIterator.SeekTo(attrIterator.EntityId); var serializerId = UInt128.Zero; Symbol uniqueId = null!; @@ -92,7 +92,7 @@ private IEnumerable ExistingAttributes() /// - public IDb Db => new Db(_store, this, TxId, _factories); + public IDb Db => new Db(_store, this, TxId); /// diff --git a/src/NexusMods.EventSourcing/Db.cs b/src/NexusMods.EventSourcing/Db.cs index 4da096fb..03ca4d66 100644 --- a/src/NexusMods.EventSourcing/Db.cs +++ b/src/NexusMods.EventSourcing/Db.cs @@ -2,10 +2,11 @@ using System.Collections.Generic; using System.Linq; using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Abstractions.Models; namespace NexusMods.EventSourcing; -public class Db(IDatomStore store, IConnection connection, TxId txId, IDictionary factories) : IDb +public class Db(IDatomStore store, Connection connection, TxId txId) : IDb { public TxId BasisTxId => txId; @@ -22,20 +23,13 @@ public IIterator Where(EntityId id) public IEnumerable Get(IEnumerable ids) where TModel : IReadModel { - var factory = factories[typeof(TModel)]; - - var iterator = store.EntityIterator(txId); - + using var iterator = store.EntityIterator(txId); + var reader = connection.ModelReflector.GetReader(); foreach (var id in ids) { - var readModel = (TModel)factory.Create(id); - iterator.SetEntityId(id); - while (iterator.Next()) - { - iterator.SetOn(readModel); - } - - yield return readModel; + iterator.SeekTo(id); + var model = reader(id, iterator); + yield return model; } } } diff --git a/src/NexusMods.EventSourcing/ModelReflector.cs b/src/NexusMods.EventSourcing/ModelReflector.cs new file mode 100644 index 00000000..09e4e575 --- /dev/null +++ b/src/NexusMods.EventSourcing/ModelReflector.cs @@ -0,0 +1,143 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Linq.Expressions; +using System.Reflection; +using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Abstractions.Models; + +namespace NexusMods.EventSourcing; + +/// +/// Reflects over models and creates reader/writer functions for them. +/// +internal class ModelReflector +where TTransaction : ITransaction +{ + private readonly ConcurrentDictionary _emitters; + private readonly IDatomStore _store; + + public ModelReflector(IDatomStore store) + { + _emitters = new ConcurrentDictionary(); + _store = store; + + } + + private delegate void EmitterFn(TTransaction tx, TReadModel model) + where TReadModel : IReadModel; + + internal delegate TReadModel ReaderFn(EntityId id, IEntityIterator iterator) + where TReadModel : IReadModel; + + public void Add(TTransaction tx, IReadModel model) + { + EmitterFn emitterFn; + var modelType = model.GetType(); + if (!_emitters.TryGetValue(model.GetType(), out var found)) + { + emitterFn = CreateEmitter(modelType); + _emitters.TryAdd(modelType, emitterFn); + } + else + { + emitterFn = (EmitterFn)found; + } + emitterFn(tx, model); + } + + /// + /// Reflects over + /// + /// + /// + private EmitterFn CreateEmitter(Type readModel) + { + var properties = GetModelProperties(readModel); + + var entityParameter = Expression.Parameter(typeof(IReadModel), "entity"); + var txParameter = Expression.Parameter(typeof(TTransaction), "tx"); + + var exprs = new List(); + var idVariable = Expression.Variable(typeof(EntityId), "entityId"); + var castedVariable = Expression.Variable(readModel, "castedEntity"); + + exprs.Add(Expression.Assign(castedVariable, Expression.Convert(entityParameter, readModel))); + exprs.Add(Expression.Assign(idVariable, Expression.Property(entityParameter, "Id"))); + + exprs.AddRange(from property in properties + let method = property.Attribute.GetMethod("Add", BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy)! + let value = Expression.Property(castedVariable, property.Property) + select Expression.Call(null, method, txParameter, idVariable, value)); + + var blockExpr = Expression.Block(new[] { idVariable, castedVariable}, exprs); + + var lambda = Expression.Lambda>(blockExpr, txParameter, entityParameter); + return lambda.Compile(); + } + + private static IEnumerable<(Type Attribute, PropertyInfo Property)> GetModelProperties(Type readModel) + { + var properties = readModel + .GetProperties(BindingFlags.Public | BindingFlags.Instance | BindingFlags.FlattenHierarchy) + .Where(p => p.GetCustomAttributes(typeof(FromAttribute<>), true).Any()) + .Select(p => ( + (p.GetCustomAttributes(typeof(FromAttribute<>), true).First() as IFromAttribute)!.AttributeType, + p)) + .ToList(); + return properties; + } + + public ReaderFn GetReader() where TModel : IReadModel + { + var properties = GetModelProperties(typeof(TModel)); + + var exprs = new List(); + + var whileTopLabel = Expression.Label("whileTop"); + var exitLabel = Expression.Label("exit"); + + + + + var entityIdParameter = Expression.Parameter(typeof(EntityId), "entityId"); + var iteratorParameter = Expression.Parameter(typeof(IEntityIterator), "iterator"); + var newModelExpr = Expression.Variable(typeof(TModel), "newModel"); + + var spanExpr = Expression.Property(iteratorParameter, "ValueSpan"); + var ctor = typeof(TModel).GetConstructor([typeof(ITransaction)])!; + + + exprs.Add(Expression.Assign(newModelExpr, Expression.New(ctor, Expression.Constant(null, typeof(ITransaction))))); + exprs.Add(Expression.Assign(Expression.Property(newModelExpr, "Id"), entityIdParameter)); + + exprs.Add(Expression.Label(whileTopLabel)); + exprs.Add(Expression.IfThen( + Expression.Not(Expression.Call(iteratorParameter, typeof(IEntityIterator).GetMethod("Next")!)), + Expression.Break(exitLabel))); + + var cases = new List(); + + foreach (var (attribute, property) in properties) + { + var readSpanExpr = _store.GetValueReadExpression(attribute, spanExpr, out var attributeId); + + var assigned = Expression.Assign(Expression.Property(newModelExpr, property), readSpanExpr); + + cases.Add(Expression.SwitchCase(Expression.Block([assigned, Expression.Goto(whileTopLabel)]), + Expression.Constant(attributeId))); + } + + exprs.Add(Expression.Switch(Expression.Property(iteratorParameter, "AttributeId"), cases.ToArray())); + + exprs.Add(Expression.Goto(whileTopLabel)); + exprs.Add(Expression.Label(exitLabel)); + exprs.Add(newModelExpr); + + var block = Expression.Block(new[] {newModelExpr}, exprs); + + var lambda = Expression.Lambda>(block, entityIdParameter, iteratorParameter); + return lambda.Compile(); + } +} diff --git a/src/NexusMods.EventSourcing/Transaction.cs b/src/NexusMods.EventSourcing/Transaction.cs index c5c29eb2..6def27bb 100644 --- a/src/NexusMods.EventSourcing/Transaction.cs +++ b/src/NexusMods.EventSourcing/Transaction.cs @@ -2,6 +2,7 @@ using System.Collections.Concurrent; using System.Threading; using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Abstractions.Models; namespace NexusMods.EventSourcing; @@ -9,6 +10,7 @@ public class Transaction(Connection connection) : ITransaction { private ulong _tempId = Ids.MinId(Ids.Partition.Tmp); private ConcurrentBag _datoms = new(); + private ConcurrentBag _models = new(); /// public EntityId TempId() @@ -21,8 +23,25 @@ public void Add(IDatom datom) _datoms.Add(datom); } + /// + public TReadModel Add(TReadModel model) + where TReadModel : IReadModel + { + _models.Add(model); + return model; + } + + public void Add(EntityId entityId, TVal val, bool isAssert = true) where TAttribute : IAttribute + { + _datoms.Add(new AssertDatom(entityId.Value, val)); + } + public ICommitResult Commit() { + foreach (var model in _models) + { + connection.ModelReflector.Add(this, model); + } return connection.Transact(_datoms); } } diff --git a/tests/NexusMods.EventSourcing.DatomStore.Tests/ADatomStoreTest.cs b/tests/NexusMods.EventSourcing.DatomStore.Tests/ADatomStoreTest.cs index bb86fb1f..5dbcdfb2 100644 --- a/tests/NexusMods.EventSourcing.DatomStore.Tests/ADatomStoreTest.cs +++ b/tests/NexusMods.EventSourcing.DatomStore.Tests/ADatomStoreTest.cs @@ -12,7 +12,7 @@ public abstract class ADatomStoreTest : IDisposable protected readonly RocksDBDatomStore Store; protected readonly Connection Connection; - protected ADatomStoreTest(IEnumerable valueSerializers, IEnumerable attributes, IEnumerable factories) + protected ADatomStoreTest(IEnumerable valueSerializers, IEnumerable attributes) { _tmpPath = FileSystem.Shared.GetKnownPath(KnownPath.TempDirectory).Combine(Guid.NewGuid() + ".rocksdb"); var dbSettings = new DatomStoreSettings() @@ -21,7 +21,7 @@ protected ADatomStoreTest(IEnumerable valueSerializers, IEnume }; _registry = new AttributeRegistry(valueSerializers, attributes); Store = new RocksDBDatomStore(new NullLogger(), _registry, dbSettings); - Connection = new Connection(Store, attributes, valueSerializers, factories); + Connection = new Connection(Store, attributes, valueSerializers); } public void Dispose() diff --git a/tests/NexusMods.EventSourcing.DatomStore.Tests/DatomStoreSetupTests.cs b/tests/NexusMods.EventSourcing.DatomStore.Tests/DatomStoreSetupTests.cs index af3f34c6..0d6200c2 100644 --- a/tests/NexusMods.EventSourcing.DatomStore.Tests/DatomStoreSetupTests.cs +++ b/tests/NexusMods.EventSourcing.DatomStore.Tests/DatomStoreSetupTests.cs @@ -4,9 +4,8 @@ namespace NexusMods.EventSourcing.DatomStore.Tests; public class DatomStoreSetupTests : ADatomStoreTest { - public DatomStoreSetupTests(IEnumerable valueSerializers, IEnumerable attributes, - IEnumerable factories) - : base(valueSerializers, attributes, factories) + public DatomStoreSetupTests(IEnumerable valueSerializers, IEnumerable attributes) + : base(valueSerializers, attributes) { } diff --git a/tests/NexusMods.EventSourcing.TestModel/Model/ArchiveFile.cs b/tests/NexusMods.EventSourcing.TestModel/Model/ArchiveFile.cs deleted file mode 100644 index 1f7833ed..00000000 --- a/tests/NexusMods.EventSourcing.TestModel/Model/ArchiveFile.cs +++ /dev/null @@ -1,15 +0,0 @@ -using NexusMods.EventSourcing.Abstractions.ModelGeneration; - -namespace NexusMods.EventSourcing.TestModel.Model; - -[ModelDefinition] -public static partial class ArchiveFile -{ - public static AttributeDefinitions Attributes = new AttributeDefinitionsBuilder() - .Include() - .Include() - .Define("Index", "A index value for testing purposes") - .Define("ArchivePath", "The path of the archive file") - .Build(); - -} diff --git a/tests/NexusMods.EventSourcing.TestModel/Model/Attributes/File.cs b/tests/NexusMods.EventSourcing.TestModel/Model/Attributes/File.cs new file mode 100644 index 00000000..c036f80d --- /dev/null +++ b/tests/NexusMods.EventSourcing.TestModel/Model/Attributes/File.cs @@ -0,0 +1,13 @@ +using NexusMods.EventSourcing.Abstractions; + +namespace NexusMods.EventSourcing.TestModel.Model.Attributes; + +public static class ModFileAttributes +{ + public class Hash : ScalarAttribute; + + public class Path : ScalarAttribute; + + public class Index : ScalarAttribute; + +} diff --git a/tests/NexusMods.EventSourcing.TestModel/Model/File.cs b/tests/NexusMods.EventSourcing.TestModel/Model/File.cs index 87244503..8dd28a42 100644 --- a/tests/NexusMods.EventSourcing.TestModel/Model/File.cs +++ b/tests/NexusMods.EventSourcing.TestModel/Model/File.cs @@ -1,13 +1,27 @@ -using NexusMods.EventSourcing.Abstractions.ModelGeneration; +using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Abstractions.Models; +using NexusMods.EventSourcing.TestModel.Model.Attributes; namespace NexusMods.EventSourcing.TestModel.Model; -[ModelDefinition] -public static partial class File + +public class File(ITransaction? tx) : AReadModel(tx) { - public static AttributeDefinitions Attributes = new AttributeDefinitionsBuilder() - .Define("Path", "The path of the file") - .Define("Hash", "The hash of the file") - .Define("Index", "A index value for testing purposes") - .Build(); + /// + /// Base attribute + /// + [From] + public required string Path { get; init; } + + /// + /// Example of "inheritance" of attributes from other namespaces + /// + [From] + public required ulong Hash { get; init; } + + /// + /// The index of the file in the archive used for debugging purposes + /// + [From] + public required ulong Index { get; init; } } diff --git a/tests/NexusMods.EventSourcing.TestModel/Model/Loadout.cs b/tests/NexusMods.EventSourcing.TestModel/Model/Loadout.cs deleted file mode 100644 index 396b53c8..00000000 --- a/tests/NexusMods.EventSourcing.TestModel/Model/Loadout.cs +++ /dev/null @@ -1,12 +0,0 @@ -using NexusMods.EventSourcing.Abstractions.ModelGeneration; - -namespace NexusMods.EventSourcing.TestModel.Model; - -[ModelDefinition] -public static partial class Loadout -{ - public static AttributeDefinitions AttributeDefinitions = new AttributeDefinitionsBuilder() - .Define("Name", "The name of the loadout") - .Build(); - -} diff --git a/tests/NexusMods.EventSourcing.TestModel/Model/Mod.cs b/tests/NexusMods.EventSourcing.TestModel/Model/Mod.cs deleted file mode 100644 index a4f19544..00000000 --- a/tests/NexusMods.EventSourcing.TestModel/Model/Mod.cs +++ /dev/null @@ -1,13 +0,0 @@ -using NexusMods.EventSourcing.Abstractions.ModelGeneration; - -namespace NexusMods.EventSourcing.TestModel.Model; - -[ModelDefinition] -public static partial class Mod -{ - public static AttributeDefinitions AttributeDefinitions = new AttributeDefinitionsBuilder() - .Define("Name", "The name of the mod") - .Define("Enabled", "Whether the mod is enabled") - .Build(); - -} diff --git a/tests/NexusMods.EventSourcing.TestModel/NexusMods.EventSourcing.TestModel.csproj b/tests/NexusMods.EventSourcing.TestModel/NexusMods.EventSourcing.TestModel.csproj index fd36c77c..a4bb5538 100644 --- a/tests/NexusMods.EventSourcing.TestModel/NexusMods.EventSourcing.TestModel.csproj +++ b/tests/NexusMods.EventSourcing.TestModel/NexusMods.EventSourcing.TestModel.csproj @@ -8,7 +8,6 @@ - diff --git a/tests/NexusMods.EventSourcing.TestModel/Services.cs b/tests/NexusMods.EventSourcing.TestModel/Services.cs index 389d7529..c9120cb1 100644 --- a/tests/NexusMods.EventSourcing.TestModel/Services.cs +++ b/tests/NexusMods.EventSourcing.TestModel/Services.cs @@ -1,12 +1,19 @@ using Microsoft.Extensions.DependencyInjection; -using NexusMods.EventSourcing.TestModel.Model; +using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.TestModel.Model.Attributes; -namespace NexusMods.EventSourcing.TestModel; public static class Services { public static IServiceCollection AddTestModel(this IServiceCollection services) => - services.AddModModel() - .AddFileModel() - .AddLoadoutModel(); + services.AddAttribute() + .AddAttribute() + .AddAttribute() + + ; + /* + .AddAttribute() + .AddAttribute();*/ + //.AddReadModel() + //.AddReadModel(); } diff --git a/tests/NexusMods.EventSourcing.Tests/AEventSourcingTest.cs b/tests/NexusMods.EventSourcing.Tests/AEventSourcingTest.cs index 4605bc69..1b9c0696 100644 --- a/tests/NexusMods.EventSourcing.Tests/AEventSourcingTest.cs +++ b/tests/NexusMods.EventSourcing.Tests/AEventSourcingTest.cs @@ -13,7 +13,7 @@ public class AEventSourcingTest : IDisposable protected readonly Connection Connection; protected AEventSourcingTest(IEnumerable valueSerializers, - IEnumerable attributes, IEnumerable factories) + IEnumerable attributes) { _tmpPath = FileSystem.Shared.GetKnownPath(KnownPath.TempDirectory).Combine(Guid.NewGuid() + ".rocksdb"); var dbSettings = new DatomStoreSettings() @@ -22,7 +22,7 @@ protected AEventSourcingTest(IEnumerable valueSerializers, }; _registry = new AttributeRegistry(valueSerializers, attributes); Store = new RocksDBDatomStore(new NullLogger(), _registry, dbSettings); - Connection = new Connection(Store, attributes, valueSerializers, factories); + Connection = new Connection(Store, attributes, valueSerializers); } public void Dispose() diff --git a/tests/NexusMods.EventSourcing.Tests/DbTests.cs b/tests/NexusMods.EventSourcing.Tests/DbTests.cs index da7dc04e..2a7d5ef9 100644 --- a/tests/NexusMods.EventSourcing.Tests/DbTests.cs +++ b/tests/NexusMods.EventSourcing.Tests/DbTests.cs @@ -7,8 +7,8 @@ namespace NexusMods.EventSourcing.Tests; public class DbTests : AEventSourcingTest { - public DbTests(IEnumerable valueSerializers, IEnumerable attributes, IEnumerable factories) - : base(valueSerializers, attributes, factories) + public DbTests(IEnumerable valueSerializers, IEnumerable attributes) + : base(valueSerializers, attributes) { } @@ -21,13 +21,15 @@ public void ReadDatomsForEntity() var ids = new List(); - for (ulong i = 0; i < TOTAL_COUNT; i++) + for (ulong idx = 0; idx < TOTAL_COUNT; idx++) { - var fileId = tx.TempId(); - ids.Add(fileId); - File.Path.Assert(fileId, $"C:\\test_{i}.txt", tx); - File.Hash.Assert(fileId, i + 0xDEADBEEF, tx); - File.Index.Assert(fileId, i, tx); + var file = new File(tx) + { + Path = $"C:\\test_{idx}.txt", + Hash = idx + 0xDEADBEEF, + Index = idx + }; + ids.Add(file.Id); } var oldTx = Connection.TxId; @@ -36,7 +38,7 @@ public void ReadDatomsForEntity() result.NewTx.Value.Should().Be(oldTx.Value + 1, "transaction id should be incremented by 1"); var db = Connection.Db; - var resolved = db.Get(ids.Select(id => result[id])).ToArray(); + var resolved = db.Get(ids.Select(id => result[id])).ToArray(); resolved.Should().HaveCount(TOTAL_COUNT); foreach (var readModel in resolved) @@ -47,6 +49,8 @@ public void ReadDatomsForEntity() } } + /* + [Fact] public void DbIsImmutable() { @@ -118,4 +122,6 @@ public void ReadModelsCanHaveExtraAttributes() } + */ + } From 038e5b2ad6473ced4a32e0400386f43e70c36555 Mon Sep 17 00:00:00 2001 From: halgari Date: Thu, 15 Feb 2024 13:12:09 -0700 Subject: [PATCH 4/8] Benchmarking and small optimizations, can hit 1.2mil datoms per second read on my machine --- .../AppHost.cs | 3 +- .../Benchmarks/ReadTests.cs | 72 ++++++++++++++----- .../Benchmarks/WriteTests.cs | 3 +- .../Model/File.cs | 14 ---- .../NexusMods.EventSourcing.Benchmarks.csproj | 1 + .../Program.cs | 2 +- .../Indexes/AETVIndex.cs | 7 +- .../Indexes/AIndexDefinition.cs | 24 +++++-- .../Indexes/EATVIndex.cs | 9 +-- .../Indexes/TxIndex.cs | 6 +- .../KeyHeader.cs | 21 +++--- .../RocksDBDatomStore.cs | 20 ++---- src/NexusMods.EventSourcing/ModelReflector.cs | 28 ++++---- .../NexusMods.EventSourcing.Tests/DbTests.cs | 1 + 14 files changed, 125 insertions(+), 86 deletions(-) delete mode 100644 benchmarks/NexusMods.EventSourcing.Benchmarks/Model/File.cs diff --git a/benchmarks/NexusMods.EventSourcing.Benchmarks/AppHost.cs b/benchmarks/NexusMods.EventSourcing.Benchmarks/AppHost.cs index b558205b..5b16a922 100644 --- a/benchmarks/NexusMods.EventSourcing.Benchmarks/AppHost.cs +++ b/benchmarks/NexusMods.EventSourcing.Benchmarks/AppHost.cs @@ -1,7 +1,6 @@ using System; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; -using NexusMods.EventSourcing.Benchmarks.Model; using NexusMods.EventSourcing.DatomStore; using NexusMods.Paths; @@ -16,7 +15,7 @@ public static IServiceProvider Create() { services.AddDatomStore() .AddEventSourcing() - .AddFileModel() + .AddTestModel() .AddSingleton(new DatomStoreSettings() { Path = FileSystem.Shared.GetKnownPath(KnownPath.TempDirectory).Combine(Guid.NewGuid() + ".rocksdb") diff --git a/benchmarks/NexusMods.EventSourcing.Benchmarks/Benchmarks/ReadTests.cs b/benchmarks/NexusMods.EventSourcing.Benchmarks/Benchmarks/ReadTests.cs index cde53e36..1933ad70 100644 --- a/benchmarks/NexusMods.EventSourcing.Benchmarks/Benchmarks/ReadTests.cs +++ b/benchmarks/NexusMods.EventSourcing.Benchmarks/Benchmarks/ReadTests.cs @@ -1,51 +1,89 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Linq; using BenchmarkDotNet.Attributes; using Microsoft.Extensions.DependencyInjection; using NexusMods.EventSourcing.Abstractions; -using NexusMods.EventSourcing.Benchmarks.Model; +using NexusMods.EventSourcing.TestModel.Model; namespace NexusMods.EventSourcing.Benchmarks.Benchmarks; +[MemoryDiagnoser] public class ReadTests { private readonly IConnection _connection; - private readonly List _entityIds; + private List _entityIdsAscending = null!; + private List _entityIdsDescending = null!; + private List _entityIdsRandom = null!; public ReadTests() { var services = AppHost.Create(); _connection = services.GetRequiredService(); - _entityIds = new List(); } + public const int MaxCount = 10000; + [GlobalSetup] public void Setup() { var tx = _connection.BeginTransaction(); - _entityIds.Clear(); - for (var i = 0; i < Count; i++) + var entityIds = new List(); + for (var i = 0; i < MaxCount; i++) { - var id = Ids.MakeId(Ids.Partition.Entity, (ulong)i); - File.Hash.Assert(tx.TempId(), (ulong)i, tx); - File.Path.Assert(tx.TempId(), $"C:\\test_{i}.txt", tx); - File.Index.Assert(tx.TempId(), (ulong)i, tx); - _entityIds.Add(EntityId.From(id)); + var file = new File(tx) + { + Hash = (ulong)i, + Path = $"C:\\test_{i}.txt", + Index = (ulong)i + }; + entityIds.Add(file.Id); } - tx.Commit(); + var result = tx.Commit(); + + entityIds = entityIds.Select(e => result[e]).ToList(); + _entityIdsAscending = entityIds.OrderBy(id => id.Value).ToList(); + _entityIdsDescending = entityIds.OrderByDescending(id => id.Value).ToList(); + + var idArray = entityIds.ToArray(); + Random.Shared.Shuffle(idArray); + _entityIdsRandom = idArray.ToList(); + } + + + [Params(1, 1000, MaxCount)] + public int Count { get; set; } = MaxCount; + + public enum SortOrder + { + Ascending, + Descending, + Random } - [Params(1, 10, 100, 1000)] - public int Count { get; set; } = 1000; + //[Params(SortOrder.Ascending, SortOrder.Descending, SortOrder.Random)] + public SortOrder Order { get; set; } = SortOrder.Descending; + + public List Ids => Order switch + { + SortOrder.Ascending => _entityIdsAscending, + SortOrder.Descending => _entityIdsDescending, + SortOrder.Random => _entityIdsRandom, + _ => throw new ArgumentOutOfRangeException() + }; [Benchmark] - public int ReadFiles() + public ulong ReadFiles() { var db = _connection.Db; - var read = db.Get(_entityIds).ToList(); - return read.Count; + ulong sum = 0; + foreach (var itm in db.Get(Ids.Take(Count))) + { + sum += itm.Index; + } + return (ulong)sum; } } diff --git a/benchmarks/NexusMods.EventSourcing.Benchmarks/Benchmarks/WriteTests.cs b/benchmarks/NexusMods.EventSourcing.Benchmarks/Benchmarks/WriteTests.cs index 1cafa79b..d523783c 100644 --- a/benchmarks/NexusMods.EventSourcing.Benchmarks/Benchmarks/WriteTests.cs +++ b/benchmarks/NexusMods.EventSourcing.Benchmarks/Benchmarks/WriteTests.cs @@ -1,7 +1,6 @@ using BenchmarkDotNet.Attributes; using Microsoft.Extensions.DependencyInjection; using NexusMods.EventSourcing.Abstractions; -using NexusMods.EventSourcing.Benchmarks.Model; namespace NexusMods.EventSourcing.Benchmarks.Benchmarks; @@ -23,6 +22,7 @@ public WriteTests() [Benchmark] public void AddFiles() { + /* var tx = _connection.BeginTransaction(); for (var i = 0; i < Count; i++) { @@ -32,6 +32,7 @@ public void AddFiles() File.Index.Assert(tx.TempId(), (ulong)i, tx); } tx.Commit(); + */ } } diff --git a/benchmarks/NexusMods.EventSourcing.Benchmarks/Model/File.cs b/benchmarks/NexusMods.EventSourcing.Benchmarks/Model/File.cs deleted file mode 100644 index ef4dadb9..00000000 --- a/benchmarks/NexusMods.EventSourcing.Benchmarks/Model/File.cs +++ /dev/null @@ -1,14 +0,0 @@ -using NexusMods.EventSourcing.Abstractions.ModelGeneration; - -namespace NexusMods.EventSourcing.Benchmarks.Model; - -[ModelDefinition] -public static partial class File -{ - public static AttributeDefinitions Attributes = new AttributeDefinitionsBuilder() - .Define("Path", "The path of the file") - .Define("Hash", "The hash of the file") - .Define("Index", "A index value for testing purposes") - .Build(); - -} diff --git a/benchmarks/NexusMods.EventSourcing.Benchmarks/NexusMods.EventSourcing.Benchmarks.csproj b/benchmarks/NexusMods.EventSourcing.Benchmarks/NexusMods.EventSourcing.Benchmarks.csproj index 374c8591..b9cbe311 100644 --- a/benchmarks/NexusMods.EventSourcing.Benchmarks/NexusMods.EventSourcing.Benchmarks.csproj +++ b/benchmarks/NexusMods.EventSourcing.Benchmarks/NexusMods.EventSourcing.Benchmarks.csproj @@ -15,6 +15,7 @@ + diff --git a/benchmarks/NexusMods.EventSourcing.Benchmarks/Program.cs b/benchmarks/NexusMods.EventSourcing.Benchmarks/Program.cs index 21dfbcdd..96992f4b 100644 --- a/benchmarks/NexusMods.EventSourcing.Benchmarks/Program.cs +++ b/benchmarks/NexusMods.EventSourcing.Benchmarks/Program.cs @@ -9,7 +9,7 @@ #if DEBUG var benchmark = new ReadTests(); -benchmark.Count = 1; +benchmark.Count = 1000; var sw = Stopwatch.StartNew(); benchmark.Setup(); diff --git a/src/NexusMods.EventSourcing.DatomStore/Indexes/AETVIndex.cs b/src/NexusMods.EventSourcing.DatomStore/Indexes/AETVIndex.cs index c7d446bc..89f3c4a2 100644 --- a/src/NexusMods.EventSourcing.DatomStore/Indexes/AETVIndex.cs +++ b/src/NexusMods.EventSourcing.DatomStore/Indexes/AETVIndex.cs @@ -6,8 +6,9 @@ namespace NexusMods.EventSourcing.DatomStore.Indexes; -public class AETVIndex(AttributeRegistry registry) : AIndexDefinition(registry, "aetv") { - public override unsafe int Compare(KeyHeader* a, uint aLength, KeyHeader* b, uint bLength) +public class AETVIndex(AttributeRegistry registry) : AIndexDefinition(registry, "aetv"), IComparatorIndex +{ + public static unsafe int Compare(AIndexDefinition idx, KeyHeader* a, uint aLength, KeyHeader* b, uint bLength) { // TX, Entity, Attribute, IsAssert, Value var cmp = KeyHeader.CompareAttribute(a, b); @@ -18,7 +19,7 @@ public override unsafe int Compare(KeyHeader* a, uint aLength, KeyHeader* b, uin if (cmp != 0) return cmp; cmp = KeyHeader.CompareIsAssert(a, b); if (cmp != 0) return cmp; - return KeyHeader.CompareValues(Registry, a, aLength, b, bLength); + return KeyHeader.CompareValues(idx.Registry, a, aLength, b, bLength); } diff --git a/src/NexusMods.EventSourcing.DatomStore/Indexes/AIndexDefinition.cs b/src/NexusMods.EventSourcing.DatomStore/Indexes/AIndexDefinition.cs index 23c49dc3..a2293984 100644 --- a/src/NexusMods.EventSourcing.DatomStore/Indexes/AIndexDefinition.cs +++ b/src/NexusMods.EventSourcing.DatomStore/Indexes/AIndexDefinition.cs @@ -1,12 +1,22 @@ using System; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using NexusMods.EventSourcing.Abstractions; using RocksDbSharp; namespace NexusMods.EventSourcing.DatomStore.Indexes; -public abstract class AIndexDefinition(AttributeRegistry registry, string columnFamilyName) : IDisposable +public interface IComparatorIndex where TOuter : IComparatorIndex { - protected readonly AttributeRegistry Registry = registry; + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static abstract unsafe int Compare(AIndexDefinition instance, KeyHeader* a, uint aLength, + KeyHeader* b, uint bLength); +} + +public abstract class AIndexDefinition(AttributeRegistry registry, string columnFamilyName) +where TOuter : IComparatorIndex +{ + protected internal readonly AttributeRegistry Registry = registry; private ColumnFamilyOptions? _options; protected ColumnFamilyHandle? ColumnFamilyHandle; @@ -29,7 +39,7 @@ public void Init(RocksDb db) { unsafe { - return Compare((KeyHeader*)a, (uint)alen, (KeyHeader*)b, (uint)blen); + return TOuter.Compare(this, (KeyHeader*)a, (uint)alen, (KeyHeader*)b, (uint)blen); } }; _comparator = Native.Instance.rocksdb_comparator_create(IntPtr.Zero, _destructorDelegate, _comparatorDelegate, _nameDelegate); @@ -39,8 +49,6 @@ public void Init(RocksDb db) public string ColumnFamilyName { get; } = columnFamilyName; - public abstract unsafe int Compare(KeyHeader *a, uint aLength, KeyHeader *b, uint bLength); - public void Dispose() { if (_comparator != IntPtr.Zero) @@ -55,6 +63,12 @@ public void Dispose() _namePtr = IntPtr.Zero; } } + + public static unsafe int Compare(TOuter outer, KeyHeader* a, uint aLength, KeyHeader* b, uint bLength) + { + throw new NotImplementedException(); + } + public void Put(WriteBatch batch, ReadOnlySpan span) { batch.Put(span, ReadOnlySpan.Empty, ColumnFamilyHandle); diff --git a/src/NexusMods.EventSourcing.DatomStore/Indexes/EATVIndex.cs b/src/NexusMods.EventSourcing.DatomStore/Indexes/EATVIndex.cs index 6b4e23a1..be064381 100644 --- a/src/NexusMods.EventSourcing.DatomStore/Indexes/EATVIndex.cs +++ b/src/NexusMods.EventSourcing.DatomStore/Indexes/EATVIndex.cs @@ -1,15 +1,16 @@ using System; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using NexusMods.EventSourcing.Abstractions; -using NexusMods.EventSourcing.Abstractions.Models; using Reloaded.Memory.Extensions; using RocksDbSharp; namespace NexusMods.EventSourcing.DatomStore.Indexes; -public class EATVIndex(AttributeRegistry registry) : AIndexDefinition(registry, "eatv") +public class EATVIndex(AttributeRegistry registry) : AIndexDefinition(registry, "eatv"), IComparatorIndex { - public override unsafe int Compare(KeyHeader* a, uint aLength, KeyHeader* b, uint bLength) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static unsafe int Compare(AIndexDefinition idx, KeyHeader* a, uint aLength, KeyHeader* b, uint bLength) { // TX, Entity, Attribute, IsAssert, Value var cmp = KeyHeader.CompareEntity(a, b); @@ -20,7 +21,7 @@ public override unsafe int Compare(KeyHeader* a, uint aLength, KeyHeader* b, uin if (cmp != 0) return cmp; cmp = KeyHeader.CompareIsAssert(a, b); if (cmp != 0) return cmp; - return KeyHeader.CompareValues(Registry, a, aLength, b, bLength); + return KeyHeader.CompareValues(idx.Registry, a, aLength, b, bLength); } diff --git a/src/NexusMods.EventSourcing.DatomStore/Indexes/TxIndex.cs b/src/NexusMods.EventSourcing.DatomStore/Indexes/TxIndex.cs index da791237..8efe3a0f 100644 --- a/src/NexusMods.EventSourcing.DatomStore/Indexes/TxIndex.cs +++ b/src/NexusMods.EventSourcing.DatomStore/Indexes/TxIndex.cs @@ -1,8 +1,8 @@ namespace NexusMods.EventSourcing.DatomStore.Indexes; -public class TxIndex(AttributeRegistry registry) : AIndexDefinition(registry, "txLog") +public class TxIndex(AttributeRegistry registry) : AIndexDefinition(registry, "txLog"), IComparatorIndex { - public override unsafe int Compare(KeyHeader* a, uint aLength, KeyHeader* b, uint bLength) + public static unsafe int Compare(AIndexDefinition idx, KeyHeader* a, uint aLength, KeyHeader* b, uint bLength) { // TX, Entity, Attribute, IsAssert, Value var cmp = KeyHeader.CompareTx(a, b); @@ -13,6 +13,6 @@ public override unsafe int Compare(KeyHeader* a, uint aLength, KeyHeader* b, uin if (cmp != 0) return cmp; cmp = KeyHeader.CompareIsAssert(a, b); if (cmp != 0) return cmp; - return KeyHeader.CompareValues(Registry, a, aLength, b, bLength); + return KeyHeader.CompareValues(idx.Registry, a, aLength, b, bLength); } } diff --git a/src/NexusMods.EventSourcing.DatomStore/KeyHeader.cs b/src/NexusMods.EventSourcing.DatomStore/KeyHeader.cs index 887d73b4..bb921611 100644 --- a/src/NexusMods.EventSourcing.DatomStore/KeyHeader.cs +++ b/src/NexusMods.EventSourcing.DatomStore/KeyHeader.cs @@ -49,22 +49,24 @@ public bool IsRetraction [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int CompareEntity(KeyHeader* a, KeyHeader* b) { - if (a->Entity < b->Entity) return -1; - return a->Entity > b->Entity ? 1 : 0; + if (a->_entity < b->_entity) return -1; + return a->_entity > b->_entity ? 1 : 0; } [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int CompareAttribute(KeyHeader* a, KeyHeader* b) { - if (a->AttributeId < b->AttributeId) return -1; - return a->AttributeId > b->AttributeId ? 1 : 0; + var aAttrId = a->_attrAndExtra & 0x7FFF; + var bAttrId = b->_attrAndExtra & 0x7FFF; + if (aAttrId < bAttrId) return -1; + return aAttrId > bAttrId ? 1 : 0; } [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int CompareTx(KeyHeader* a, KeyHeader* b) { - if (a->Tx < b->Tx) return -1; - return a->Tx > b->Tx ? 1 : 0; + if (a->_tx < b->_tx) return -1; + return a->_tx > b->_tx ? 1 : 0; } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -79,8 +81,11 @@ public static int CompareIsAssert(KeyHeader* a, KeyHeader* b) [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int CompareValues(AttributeRegistry registry, KeyHeader* a, uint aLength, KeyHeader* b, uint bLength) { - if (a->AttributeId < b->AttributeId) return 1; - if (a->AttributeId > b->AttributeId) return -1; + var aAttrId = a->_attrAndExtra & 0x7FFF; + var bAttrId = b->_attrAndExtra & 0x7FFF; + if (aAttrId < bAttrId) return -1; + if (aAttrId > bAttrId) + return 1; var aVal = (byte*) a + Size; var bVal = (byte*) b + Size; diff --git a/src/NexusMods.EventSourcing.DatomStore/RocksDBDatomStore.cs b/src/NexusMods.EventSourcing.DatomStore/RocksDBDatomStore.cs index a74c31af..036d66ee 100644 --- a/src/NexusMods.EventSourcing.DatomStore/RocksDBDatomStore.cs +++ b/src/NexusMods.EventSourcing.DatomStore/RocksDBDatomStore.cs @@ -20,7 +20,6 @@ public class RocksDBDatomStore : IDatomStore private readonly RocksDb _db; private readonly PooledMemoryBufferWriter _pooledWriter; private readonly AttributeRegistry _registry; - private readonly AIndexDefinition[] _indexes; private ulong _tx; private readonly AETVIndex _avetIndex; private readonly EATVIndex _eatvIndex; @@ -40,18 +39,9 @@ public RocksDBDatomStore(ILogger logger, AttributeRegistry re _db = RocksDb.Open(_options, _settings.Path.ToString(), new ColumnFamilies()); _eatvIndex = new EATVIndex(_registry); + _eatvIndex.Init(_db); _avetIndex = new AETVIndex(_registry); - _indexes = - [ - //new TxIndex(_registry), - _eatvIndex, - _avetIndex, - ]; - - foreach (var index in _indexes) - { - index.Init(_db); - } + _avetIndex.Init(_db); _pooledWriter = new PooledMemoryBufferWriter(128); @@ -76,10 +66,8 @@ private void Serialize(WriteBatch batch, ulong e, TVal val, ulong t _registry.WriteValue(val, in _pooledWriter); var span = _pooledWriter.GetWrittenSpan(); - foreach (var index in _indexes) - { - index.Put(batch, span); - } + _eatvIndex.Put(batch, span); + _avetIndex.Put(batch, span); } private struct TransactSink(RocksDBDatomStore store, WriteBatch batch, ulong tx) : IDatomSink diff --git a/src/NexusMods.EventSourcing/ModelReflector.cs b/src/NexusMods.EventSourcing/ModelReflector.cs index 09e4e575..ab5754b1 100644 --- a/src/NexusMods.EventSourcing/ModelReflector.cs +++ b/src/NexusMods.EventSourcing/ModelReflector.cs @@ -12,18 +12,11 @@ namespace NexusMods.EventSourcing; /// /// Reflects over models and creates reader/writer functions for them. /// -internal class ModelReflector -where TTransaction : ITransaction +internal class ModelReflector(IDatomStore store) + where TTransaction : ITransaction { - private readonly ConcurrentDictionary _emitters; - private readonly IDatomStore _store; - - public ModelReflector(IDatomStore store) - { - _emitters = new ConcurrentDictionary(); - _store = store; - - } + private readonly ConcurrentDictionary _emitters = new(); + private readonly ConcurrentDictionary _readers = new(); private delegate void EmitterFn(TTransaction tx, TReadModel model) where TReadModel : IReadModel; @@ -90,6 +83,17 @@ private EmitterFn CreateEmitter(Type readModel) } public ReaderFn GetReader() where TModel : IReadModel + { + var modelType = typeof(TModel); + if (_readers.TryGetValue(modelType, out var found)) + return (ReaderFn)found; + + var readerFn = MakeReader(); + _readers.TryAdd(modelType, readerFn); + return readerFn; + } + + public ReaderFn MakeReader() where TModel : IReadModel { var properties = GetModelProperties(typeof(TModel)); @@ -121,7 +125,7 @@ public ReaderFn GetReader() where TModel : IReadModel foreach (var (attribute, property) in properties) { - var readSpanExpr = _store.GetValueReadExpression(attribute, spanExpr, out var attributeId); + var readSpanExpr = store.GetValueReadExpression(attribute, spanExpr, out var attributeId); var assigned = Expression.Assign(Expression.Property(newModelExpr, property), readSpanExpr); diff --git a/tests/NexusMods.EventSourcing.Tests/DbTests.cs b/tests/NexusMods.EventSourcing.Tests/DbTests.cs index 2a7d5ef9..ca4a2d86 100644 --- a/tests/NexusMods.EventSourcing.Tests/DbTests.cs +++ b/tests/NexusMods.EventSourcing.Tests/DbTests.cs @@ -46,6 +46,7 @@ public void ReadDatomsForEntity() var idx = readModel.Index; readModel.Hash.Should().Be(idx + 0xDEADBEEF); readModel.Path.Should().Be($"C:\\test_{idx}.txt"); + readModel.Index.Should().Be(idx); } } From ea1db5bae009669c77134ee19700877d612be1de Mon Sep 17 00:00:00 2001 From: halgari Date: Sat, 17 Feb 2024 08:56:27 -0700 Subject: [PATCH 5/8] Fix the EAVTIterator --- .../IEntityIterator.cs | 5 +- .../Indexes/EATVIndex.cs | 62 +++++++++---------- .../KeyHeader.cs | 5 ++ src/NexusMods.EventSourcing/Connection.cs | 3 +- src/NexusMods.EventSourcing/Db.cs | 2 +- .../NexusMods.EventSourcing.Tests/DbTests.cs | 26 ++++---- 6 files changed, 55 insertions(+), 48 deletions(-) diff --git a/src/NexusMods.EventSourcing.Abstractions/IEntityIterator.cs b/src/NexusMods.EventSourcing.Abstractions/IEntityIterator.cs index a90c0460..ffda6137 100644 --- a/src/NexusMods.EventSourcing.Abstractions/IEntityIterator.cs +++ b/src/NexusMods.EventSourcing.Abstractions/IEntityIterator.cs @@ -15,10 +15,11 @@ public interface IEntityIterator : IDisposable public bool Next(); /// - /// Seeks to the data for the given Entity Id, this implicitly resets the iterator. + /// Sets the EntityId for the iterator, the next call to Next() will return the first datom for the given entity + /// that is less than or equal to the txId given to the iterator at creation. /// /// - public void SeekTo(EntityId entityId); + public void Set(EntityId entityId); /// /// Gets the current datom as a distinct value. diff --git a/src/NexusMods.EventSourcing.DatomStore/Indexes/EATVIndex.cs b/src/NexusMods.EventSourcing.DatomStore/Indexes/EATVIndex.cs index be064381..3206a7c3 100644 --- a/src/NexusMods.EventSourcing.DatomStore/Indexes/EATVIndex.cs +++ b/src/NexusMods.EventSourcing.DatomStore/Indexes/EATVIndex.cs @@ -1,4 +1,5 @@ using System; +using System.Diagnostics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using NexusMods.EventSourcing.Abstractions; @@ -56,16 +57,15 @@ public bool MaxId(Ids.Partition partition, out ulong o) public unsafe struct EATVIterator : IEntityIterator, IDisposable { - private readonly EATVIndex _idx; - private KeyHeader* _key; + private readonly KeyHeader* _key; + private KeyHeader* _current; + private UIntPtr _currentLength; private readonly Iterator _iterator; - private readonly ulong _attrId; private readonly AttributeRegistry _registry; - private bool _justSet; + private bool _needsSeek; public EATVIterator(ulong txId, AttributeRegistry registry, EATVIndex idx) { - _idx = idx; _registry = registry; _iterator = idx.Db.NewIterator(idx.ColumnFamilyHandle); _key = (KeyHeader*)Marshal.AllocHGlobal(KeyHeader.Size); @@ -73,37 +73,33 @@ public EATVIterator(ulong txId, AttributeRegistry registry, EATVIndex idx) _key->AttributeId = ulong.MaxValue; _key->Tx = txId; _key->IsAssert = true; - _iterator.SeekForPrev((byte*)_key, KeyHeader.Size); - _justSet = true; + _needsSeek = true; } - public void SeekTo(EntityId entityId) + public void Set(EntityId entityId) { _key->Entity = entityId.Value; _key->AttributeId = ulong.MaxValue; - _iterator.SeekForPrev((byte*)_key, KeyHeader.Size); - _justSet = true; + _needsSeek = true; } public IDatom Current { get { - var span = _iterator.GetKeySpan(); - var valueSpan = span.SliceFast(KeyHeader.Size); - var header = MemoryMarshal.AsRef(span); - return _registry.ReadDatom(ref header, valueSpan); + Debug.Assert(!_needsSeek, "Must call Next() before accessing Current"); + var currentValue = new ReadOnlySpan((byte*)_current + KeyHeader.Size, (int)_currentLength - KeyHeader.Size); + return _registry.ReadDatom(ref *_current, currentValue); } } public TValue GetValue() where TAttribute : IAttribute { - var span = _iterator.GetKeySpan(); - var currentHeader = MemoryMarshal.AsRef(span); - var currentValue = span.SliceFast(KeyHeader.Size); - return _registry.ReadValue(ref currentHeader, currentValue); + Debug.Assert(!_needsSeek, "Must call Next() before accessing GetValue"); + var currentValue = new ReadOnlySpan((byte*)_current + KeyHeader.Size, (int)_currentLength - KeyHeader.Size); + return _registry.ReadValue(ref *_current, currentValue); } @@ -111,9 +107,8 @@ public ulong AttributeId { get { - var span = _iterator.GetKeySpan(); - var currentHeader = MemoryMarshal.AsRef(span); - return currentHeader.AttributeId; + Debug.Assert(!_needsSeek, "Must call Next() before accessing AttributeId"); + return _current->AttributeId; } } @@ -121,28 +116,29 @@ public ulong AttributeId public bool Next() { - TOP: - if (!_justSet) + if (_needsSeek) { - _iterator.Prev(); + _iterator.SeekForPrev((byte*)_key, KeyHeader.Size); + _needsSeek = false; } else { - _justSet = false; + if (_current->AttributeId == 0) + return false; + + _key->AttributeId = _current->AttributeId - 1; + _iterator.SeekForPrev((byte*)_key, KeyHeader.Size); } if (!_iterator.Valid()) return false; - var current = _iterator.GetKeySpan(); - var currentHeader = MemoryMarshal.AsRef(current); + _current = (KeyHeader*)Native.Instance.rocksdb_iter_key(_iterator.Handle, out _currentLength); - if (currentHeader.Entity != _key->Entity) return false; + Debug.Assert(_currentLength >= KeyHeader.Size, "Key length is less than KeyHeader.Size"); + + if (_current->Entity != _key->Entity) + return false; - if (currentHeader.IsRetraction) - { - _key->AttributeId = currentHeader.AttributeId - 1; - goto TOP; - } return true; } public void Dispose() diff --git a/src/NexusMods.EventSourcing.DatomStore/KeyHeader.cs b/src/NexusMods.EventSourcing.DatomStore/KeyHeader.cs index bb921611..db2256fa 100644 --- a/src/NexusMods.EventSourcing.DatomStore/KeyHeader.cs +++ b/src/NexusMods.EventSourcing.DatomStore/KeyHeader.cs @@ -87,6 +87,11 @@ public static int CompareValues(AttributeRegistry registry, KeyHeader* a, uint a if (aAttrId > bAttrId) return 1; + // Iterators will pass a value length of 0, and in that case we want to have the valueless key be the "largest" + // so the iterator will seek to the the entry with the value + if (aLength == Size || bLength == Size) + return aLength == bLength ? 0 : aLength == Size ? 1 : -1; + var aVal = (byte*) a + Size; var bVal = (byte*) b + Size; diff --git a/src/NexusMods.EventSourcing/Connection.cs b/src/NexusMods.EventSourcing/Connection.cs index 1b8f5474..6365ef6b 100644 --- a/src/NexusMods.EventSourcing/Connection.cs +++ b/src/NexusMods.EventSourcing/Connection.cs @@ -68,7 +68,7 @@ private IEnumerable ExistingAttributes() var entIterator = _store.EntityIterator(tx); while (attrIterator.Next()) { - entIterator.SeekTo(attrIterator.EntityId); + entIterator.Set(attrIterator.EntityId); var serializerId = UInt128.Zero; Symbol uniqueId = null!; @@ -129,6 +129,7 @@ public ICommitResult Transact(IEnumerable datoms) } } var newTx = _store.Transact(newDatoms); + TxId = newTx; return new CommitResult(newTx, remaps); } } diff --git a/src/NexusMods.EventSourcing/Db.cs b/src/NexusMods.EventSourcing/Db.cs index 03ca4d66..68b9a53a 100644 --- a/src/NexusMods.EventSourcing/Db.cs +++ b/src/NexusMods.EventSourcing/Db.cs @@ -27,7 +27,7 @@ public IEnumerable Get(IEnumerable ids) where TModel : var reader = connection.ModelReflector.GetReader(); foreach (var id in ids) { - iterator.SeekTo(id); + iterator.Set(id); var model = reader(id, iterator); yield return model; } diff --git a/tests/NexusMods.EventSourcing.Tests/DbTests.cs b/tests/NexusMods.EventSourcing.Tests/DbTests.cs index ca4a2d86..551377a0 100644 --- a/tests/NexusMods.EventSourcing.Tests/DbTests.cs +++ b/tests/NexusMods.EventSourcing.Tests/DbTests.cs @@ -1,5 +1,6 @@ using NexusMods.EventSourcing.Abstractions; using NexusMods.EventSourcing.TestModel.Model; +using NexusMods.EventSourcing.TestModel.Model.Attributes; using File = NexusMods.EventSourcing.TestModel.Model.File; namespace NexusMods.EventSourcing.Tests; @@ -50,7 +51,6 @@ public void ReadDatomsForEntity() } } - /* [Fact] public void DbIsImmutable() @@ -59,18 +59,20 @@ public void DbIsImmutable() // Insert some data var tx = Connection.BeginTransaction(); - var fileId = tx.TempId(); - File.Path.Assert(fileId, "C:\\test.txt_mutate", tx); - File.Hash.Assert(fileId, 0xDEADBEEF, tx); - File.Index.Assert(fileId, 0, tx); + var file = new File(tx) + { + Path = "C:\\test.txt_mutate", + Hash = 0xDEADBEEF, + Index = 0 + }; var result = tx.Commit(); - var realId = result[fileId]; + var realId = result[file.Id]; var originalDb = Connection.Db; // Validate the data - var found = originalDb.Get([realId]).First(); + var found = originalDb.Get([realId]).First(); found.Path.Should().Be("C:\\test.txt_mutate"); found.Hash.Should().Be(0xDEADBEEF); found.Index.Should().Be(0); @@ -79,22 +81,24 @@ public void DbIsImmutable() for (var i = 0; i < TIMES; i++) { var newTx = Connection.BeginTransaction(); - File.Path.Assert(fileId, $"C:\\test_{i}.txt_mutate", newTx); + ModFileAttributes.Path.Add(newTx, realId, $"C:\\test_{i}.txt_mutate"); var newResult = newTx.Commit(); // Validate the data var newDb = Connection.Db; - var newId = newResult[fileId]; - var newFound = newDb.Get([newId]).First(); + newDb.BasisTxId.Value.Should().Be(originalDb.BasisTxId.Value + 1UL + (ulong)i, "transaction id should be incremented by 1 for each mutation"); + + var newFound = newDb.Get([realId]).First(); newFound.Path.Should().Be($"C:\\test_{i}.txt_mutate"); // Validate the original data - var orignalFound = originalDb.Get([realId]).First(); + var orignalFound = originalDb.Get([realId]).First(); orignalFound.Path.Should().Be("C:\\test.txt_mutate"); } } + /* [Fact] public void ReadModelsCanHaveExtraAttributes() { From c9b39ddff7d343868967369be1b9c0284edc0d61 Mon Sep 17 00:00:00 2001 From: halgari Date: Sat, 17 Feb 2024 09:20:08 -0700 Subject: [PATCH 6/8] Example test showing combining of attributes together to create two different read models --- .../Indexes/EATVIndex.cs | 33 ++++++++++++----- .../Model/ArchiveFile.cs | 32 ++++++++++++++++ .../Model/Attributes/ArchiveFileAttributes.cs | 17 +++++++++ .../{File.cs => ModFileAttributes.cs} | 0 .../Services.cs | 13 ++----- .../NexusMods.EventSourcing.Tests/DbTests.cs | 37 +++++++++++-------- 6 files changed, 97 insertions(+), 35 deletions(-) create mode 100644 tests/NexusMods.EventSourcing.TestModel/Model/ArchiveFile.cs create mode 100644 tests/NexusMods.EventSourcing.TestModel/Model/Attributes/ArchiveFileAttributes.cs rename tests/NexusMods.EventSourcing.TestModel/Model/Attributes/{File.cs => ModFileAttributes.cs} (100%) diff --git a/src/NexusMods.EventSourcing.DatomStore/Indexes/EATVIndex.cs b/src/NexusMods.EventSourcing.DatomStore/Indexes/EATVIndex.cs index 3206a7c3..e0f18969 100644 --- a/src/NexusMods.EventSourcing.DatomStore/Indexes/EATVIndex.cs +++ b/src/NexusMods.EventSourcing.DatomStore/Indexes/EATVIndex.cs @@ -123,23 +123,36 @@ public bool Next() } else { - if (_current->AttributeId == 0) - return false; - _key->AttributeId = _current->AttributeId - 1; - _iterator.SeekForPrev((byte*)_key, KeyHeader.Size); + _iterator.Prev(); } - if (!_iterator.Valid()) return false; + while (true) + { - _current = (KeyHeader*)Native.Instance.rocksdb_iter_key(_iterator.Handle, out _currentLength); + if (!_iterator.Valid()) return false; - Debug.Assert(_currentLength >= KeyHeader.Size, "Key length is less than KeyHeader.Size"); + _current = (KeyHeader*)Native.Instance.rocksdb_iter_key(_iterator.Handle, out _currentLength); - if (_current->Entity != _key->Entity) - return false; + Debug.Assert(_currentLength < KeyHeader.Size, "Key length is less than KeyHeader.Size"); - return true; + if (_current->Entity != _key->Entity) + return false; + + if (_current->Tx > _key->Tx) + { + _iterator.Prev(); + continue; + } + + if (_current->AttributeId > _key->AttributeId) + { + _iterator.Prev(); + continue; + } + + return true; + } } public void Dispose() { diff --git a/tests/NexusMods.EventSourcing.TestModel/Model/ArchiveFile.cs b/tests/NexusMods.EventSourcing.TestModel/Model/ArchiveFile.cs new file mode 100644 index 00000000..82023fd5 --- /dev/null +++ b/tests/NexusMods.EventSourcing.TestModel/Model/ArchiveFile.cs @@ -0,0 +1,32 @@ +using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Abstractions.Models; +using NexusMods.EventSourcing.TestModel.Model.Attributes; + +namespace NexusMods.EventSourcing.TestModel.Model; + +public class ArchiveFile(ITransaction? tx) : AReadModel(tx) +{ + /// + /// Base attribute + /// + [From] + public required string ModPath { get; init; } + + /// + /// Base attribute + /// + [From] + public required string Path { get; init; } + + /// + /// Example of "inheritance" of attributes from other namespaces + /// + [From] + public required ulong Hash { get; init; } + + /// + /// The index of the file in the archive used for debugging purposes + /// + [From] + public required ulong Index { get; init; } +} diff --git a/tests/NexusMods.EventSourcing.TestModel/Model/Attributes/ArchiveFileAttributes.cs b/tests/NexusMods.EventSourcing.TestModel/Model/Attributes/ArchiveFileAttributes.cs new file mode 100644 index 00000000..42e5770d --- /dev/null +++ b/tests/NexusMods.EventSourcing.TestModel/Model/Attributes/ArchiveFileAttributes.cs @@ -0,0 +1,17 @@ +using NexusMods.EventSourcing.Abstractions; + +namespace NexusMods.EventSourcing.TestModel.Model.Attributes; + +public static class ArchiveFileAttributes +{ + /// + /// Extra attribute with a different name + /// + public class ArchiveHash : ScalarAttribute; + + /// + /// Overlapping name with ModFileAttributes.Path + /// + public class Path : ScalarAttribute; + +} diff --git a/tests/NexusMods.EventSourcing.TestModel/Model/Attributes/File.cs b/tests/NexusMods.EventSourcing.TestModel/Model/Attributes/ModFileAttributes.cs similarity index 100% rename from tests/NexusMods.EventSourcing.TestModel/Model/Attributes/File.cs rename to tests/NexusMods.EventSourcing.TestModel/Model/Attributes/ModFileAttributes.cs diff --git a/tests/NexusMods.EventSourcing.TestModel/Services.cs b/tests/NexusMods.EventSourcing.TestModel/Services.cs index c9120cb1..6be3d6db 100644 --- a/tests/NexusMods.EventSourcing.TestModel/Services.cs +++ b/tests/NexusMods.EventSourcing.TestModel/Services.cs @@ -7,13 +7,8 @@ public static class Services { public static IServiceCollection AddTestModel(this IServiceCollection services) => services.AddAttribute() - .AddAttribute() - .AddAttribute() - - ; - /* - .AddAttribute() - .AddAttribute();*/ - //.AddReadModel() - //.AddReadModel(); + .AddAttribute() + .AddAttribute() + .AddAttribute() + .AddAttribute(); } diff --git a/tests/NexusMods.EventSourcing.Tests/DbTests.cs b/tests/NexusMods.EventSourcing.Tests/DbTests.cs index 551377a0..b670d1ad 100644 --- a/tests/NexusMods.EventSourcing.Tests/DbTests.cs +++ b/tests/NexusMods.EventSourcing.Tests/DbTests.cs @@ -98,35 +98,40 @@ public void DbIsImmutable() } } - /* + [Fact] public void ReadModelsCanHaveExtraAttributes() { + // Insert some data var tx = Connection.BeginTransaction(); - var fileId = tx.TempId(); - File.Path.Assert(fileId, "C:\\test.txt", tx); - File.Hash.Assert(fileId, 0xDEADBEEF, tx); - File.Index.Assert(fileId, 77, tx); - ArchiveFile.Index.Assert(fileId, 42, tx); - ArchiveFile.ArchivePath.Assert(fileId, "C:\\archive.zip", tx); + var file = new File(tx) + { + Path = "C:\\test.txt", + Hash = 0xDEADBEEF, + Index = 77 + }; + // Attach extra attributes to the entity + ArchiveFileAttributes.Path.Add(tx, file.Id, "C:\\test.zip"); + ArchiveFileAttributes.ArchiveHash.Add(tx, file.Id, 0xFEEDBEEF); var result = tx.Commit(); - var realId = result[fileId]; + + var realId = result[file.Id]; var db = Connection.Db; - var readModel = db.Get([realId]).First(); + // Original data exists + var readModel = db.Get([realId]).First(); readModel.Path.Should().Be("C:\\test.txt"); readModel.Hash.Should().Be(0xDEADBEEF); readModel.Index.Should().Be(77); - var archiveReadModel = db.Get([realId]).First(); - archiveReadModel.Path.Should().Be("C:\\test.txt"); - archiveReadModel.Hash.Should().Be(0xDEADBEEF); - archiveReadModel.Index.Should().Be(42); - archiveReadModel.ArchivePath.Should().Be("C:\\archive.zip"); - + // Extra data exists and can be read with a different read model + var archiveReadModel = db.Get([realId]).First(); + archiveReadModel.ModPath.Should().Be("C:\\test.txt"); + archiveReadModel.Path.Should().Be("C:\\test.zip"); + archiveReadModel.Hash.Should().Be(0xFEEDBEEF); + archiveReadModel.Index.Should().Be(77); } - */ } From e00df597e33fcaa57202b93f62766bf632650190 Mon Sep 17 00:00:00 2001 From: halgari Date: Mon, 19 Feb 2024 14:00:16 -0700 Subject: [PATCH 7/8] Can get an observable of changes from the connection --- .../ICommitResult.cs | 9 ++++- .../IConnection.cs | 8 +++- src/NexusMods.EventSourcing/CommitResult.cs | 5 ++- src/NexusMods.EventSourcing/Connection.cs | 14 ++++++- .../NexusMods.EventSourcing.Tests/DbTests.cs | 38 +++++++++++++++++++ 5 files changed, 69 insertions(+), 5 deletions(-) diff --git a/src/NexusMods.EventSourcing.Abstractions/ICommitResult.cs b/src/NexusMods.EventSourcing.Abstractions/ICommitResult.cs index 5c0fe101..0e277483 100644 --- a/src/NexusMods.EventSourcing.Abstractions/ICommitResult.cs +++ b/src/NexusMods.EventSourcing.Abstractions/ICommitResult.cs @@ -1,4 +1,6 @@ -namespace NexusMods.EventSourcing.Abstractions; +using System.Collections.Generic; + +namespace NexusMods.EventSourcing.Abstractions; /// /// The result of a transaction commit, contains metadata useful for looking up the results of the transaction @@ -16,4 +18,9 @@ public interface ICommitResult /// Gets the new TxId after the commit /// public TxId NewTx { get; } + + /// + /// The datoms that were added to the store as a result of the transaction + /// + public IEnumerable Datoms { get; } } diff --git a/src/NexusMods.EventSourcing.Abstractions/IConnection.cs b/src/NexusMods.EventSourcing.Abstractions/IConnection.cs index 364e88f7..4964e142 100644 --- a/src/NexusMods.EventSourcing.Abstractions/IConnection.cs +++ b/src/NexusMods.EventSourcing.Abstractions/IConnection.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; namespace NexusMods.EventSourcing.Abstractions; @@ -29,4 +30,9 @@ public interface IConnection /// /// public ITransaction BeginTransaction(); + + /// + /// A sequential stream of commits to the database. + /// + public IObservable Commits { get; } } diff --git a/src/NexusMods.EventSourcing/CommitResult.cs b/src/NexusMods.EventSourcing/CommitResult.cs index 48aa0c80..7adf0cbc 100644 --- a/src/NexusMods.EventSourcing/CommitResult.cs +++ b/src/NexusMods.EventSourcing/CommitResult.cs @@ -4,7 +4,7 @@ namespace NexusMods.EventSourcing; /// -public class CommitResult(TxId newTxId, IDictionary remaps) : ICommitResult +public class CommitResult(TxId newTxId, IDictionary remaps, Connection connection, IDatom[] datoms) : ICommitResult { /// public EntityId this[EntityId id] => @@ -12,4 +12,7 @@ public class CommitResult(TxId newTxId, IDictionary remaps) : ICom /// public TxId NewTx => newTxId; + + /// + public IEnumerable Datoms => datoms; } diff --git a/src/NexusMods.EventSourcing/Connection.cs b/src/NexusMods.EventSourcing/Connection.cs index 6365ef6b..ddd24259 100644 --- a/src/NexusMods.EventSourcing/Connection.cs +++ b/src/NexusMods.EventSourcing/Connection.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using System.Reactive.Subjects; using NexusMods.EventSourcing.Abstractions; using NexusMods.EventSourcing.DatomStore; @@ -17,6 +18,7 @@ public class Connection : IConnection private readonly IDatomStore _store; private readonly IAttribute[] _declaredAttributes; internal readonly ModelReflector ModelReflector; + private readonly Subject _updates; /// /// Main connection class, co-ordinates writes and immutable reads @@ -27,6 +29,8 @@ public Connection(IDatomStore store, IEnumerable declaredAttributes, _declaredAttributes = declaredAttributes.ToArray(); ModelReflector = new ModelReflector(store); + _updates = new Subject(); + AddMissingAttributes(serializers); } @@ -103,11 +107,12 @@ private IEnumerable ExistingAttributes() public ICommitResult Transact(IEnumerable datoms) { var remaps = new Dictionary(); + var datomsArray = datoms.ToArray(); lock (_lock) { var newDatoms = new List(); - foreach (var datom in datoms) + foreach (var datom in datomsArray) { var eid = datom.E; if (Ids.GetPartition(eid) == Ids.Partition.Tmp) @@ -130,7 +135,9 @@ public ICommitResult Transact(IEnumerable datoms) } var newTx = _store.Transact(newDatoms); TxId = newTx; - return new CommitResult(newTx, remaps); + var result = new CommitResult(newTx, remaps, this, datomsArray); + _updates.OnNext(result); + return result; } } @@ -139,4 +146,7 @@ public ITransaction BeginTransaction() { return new Transaction(this); } + + /// + public IObservable Commits => _updates; } diff --git a/tests/NexusMods.EventSourcing.Tests/DbTests.cs b/tests/NexusMods.EventSourcing.Tests/DbTests.cs index b670d1ad..759a43ce 100644 --- a/tests/NexusMods.EventSourcing.Tests/DbTests.cs +++ b/tests/NexusMods.EventSourcing.Tests/DbTests.cs @@ -130,6 +130,44 @@ public void ReadModelsCanHaveExtraAttributes() archiveReadModel.Path.Should().Be("C:\\test.zip"); archiveReadModel.Hash.Should().Be(0xFEEDBEEF); archiveReadModel.Index.Should().Be(77); + } + + [Fact] + public void CanGetCommitUpdates() + { + List updates = new(); + + Connection.Commits.Subscribe(update => + { + updates.Add(update.Datoms.ToArray()); + }); + + var tx = Connection.BeginTransaction(); + var file = new File(tx) + { + Path = "C:\\test.txt", + Hash = 0xDEADBEEF, + Index = 77 + }; + var result = tx.Commit(); + + var realId = result[file.Id]; + + updates.Should().HaveCount(1); + + for (var idx = 0; idx < 10; idx++) + { + tx = Connection.BeginTransaction(); + ModFileAttributes.Index.Add(tx, realId, (ulong)idx); + result = tx.Commit(); + + result.Datoms.Should().BeEquivalentTo(updates[idx + 1]); + + updates.Should().HaveCount(idx + 2); + var updateDatom = updates[idx + 1].OfType>() + .First(); + updateDatom.V.Should().Be((ulong)idx); + } } From 257dc3d6e7f60b66a0c1ec257e7ee4e5a577fb0d Mon Sep 17 00:00:00 2001 From: halgari Date: Mon, 19 Feb 2024 21:03:40 -0700 Subject: [PATCH 8/8] Can join into child entities and link entities via reverse lookups --- .../Datom.cs | 16 ++- .../IDatomStore.cs | 5 + .../IDb.cs | 19 ++- .../ITransaction.cs | 5 + .../Models/AReadModel.cs | 37 ++++- .../Models/ReverseLookupAttribute.cs | 12 ++ .../BuiltInSerializers/EntityIdSerializer.cs | 31 ++++ .../BuiltInSerializers/TxIdSerializer.cs | 30 ++++ .../Indexes/AVTEIndex.cs | 135 ++++++++++++++++++ .../RocksDBDatomStore.cs | 14 ++ .../Services.cs | 4 +- src/NexusMods.EventSourcing/Connection.cs | 35 +++-- src/NexusMods.EventSourcing/Db.cs | 24 +++- src/NexusMods.EventSourcing/ModelReflector.cs | 7 +- src/NexusMods.EventSourcing/Transaction.cs | 5 +- .../Model/Attributes/LoadoutAttributes.cs | 16 +++ .../Model/Attributes/ModAttributes.cs | 21 +++ .../Model/Loadout.cs | 50 +++++++ .../Model/Mod.cs | 37 +++++ .../Services.cs | 7 +- .../NexusMods.EventSourcing.Tests/DbTests.cs | 26 ++++ 21 files changed, 503 insertions(+), 33 deletions(-) create mode 100644 src/NexusMods.EventSourcing.Abstractions/Models/ReverseLookupAttribute.cs create mode 100644 src/NexusMods.EventSourcing.DatomStore/BuiltInSerializers/EntityIdSerializer.cs create mode 100644 src/NexusMods.EventSourcing.DatomStore/BuiltInSerializers/TxIdSerializer.cs create mode 100644 src/NexusMods.EventSourcing.DatomStore/Indexes/AVTEIndex.cs create mode 100644 tests/NexusMods.EventSourcing.TestModel/Model/Attributes/LoadoutAttributes.cs create mode 100644 tests/NexusMods.EventSourcing.TestModel/Model/Attributes/ModAttributes.cs create mode 100644 tests/NexusMods.EventSourcing.TestModel/Model/Loadout.cs create mode 100644 tests/NexusMods.EventSourcing.TestModel/Model/Mod.cs diff --git a/src/NexusMods.EventSourcing.Abstractions/Datom.cs b/src/NexusMods.EventSourcing.Abstractions/Datom.cs index 4772508d..2fbc915a 100644 --- a/src/NexusMods.EventSourcing.Abstractions/Datom.cs +++ b/src/NexusMods.EventSourcing.Abstractions/Datom.cs @@ -10,11 +10,11 @@ public interface IDatom void Emit(ref TSink sink) where TSink : IDatomSink; /// - /// Duplicates the datom with a new entity id + /// The datom should call the remap function on each entity id it contains + /// to remap the entity ids to actual ids /// - /// - /// - IDatom RemapEntityId(ulong newId); + /// + void Remap(Func remapFn); } public interface IDatomWithTx : IDatom @@ -36,9 +36,13 @@ public void Emit(ref TSink sink) where TSink : IDatomSink } /// - public IDatom RemapEntityId(ulong newId) + public void Remap(Func remapFn) { - return new AssertDatom(newId, v); + e = remapFn(EntityId.From(e)).Value; + if (v is EntityId entityId) + { + v = (TVal) (object) EntityId.From(remapFn(entityId).Value); + } } } diff --git a/src/NexusMods.EventSourcing.Abstractions/IDatomStore.cs b/src/NexusMods.EventSourcing.Abstractions/IDatomStore.cs index 2fac6cd8..cf489c33 100644 --- a/src/NexusMods.EventSourcing.Abstractions/IDatomStore.cs +++ b/src/NexusMods.EventSourcing.Abstractions/IDatomStore.cs @@ -42,4 +42,9 @@ public interface IDatomStore : IDisposable /// /// Expression GetValueReadExpression(Type attribute, Expression valueSpan, out ulong attributeId); + + /// + /// Gets all the entities that reference the given entity id with the given attribute. + /// + IEnumerable ReverseLookup(TxId txId) where TAttribute : IAttribute; } diff --git a/src/NexusMods.EventSourcing.Abstractions/IDb.cs b/src/NexusMods.EventSourcing.Abstractions/IDb.cs index ab0f0d03..546d12c4 100644 --- a/src/NexusMods.EventSourcing.Abstractions/IDb.cs +++ b/src/NexusMods.EventSourcing.Abstractions/IDb.cs @@ -21,9 +21,24 @@ public IIterator Where() /// /// Returns a read model for each of the given entity ids. /// - /// + public IEnumerable Get(IEnumerable ids) + where TModel : IReadModel; + + + /// + /// Gets a read model for the given entity id. + /// + /// /// /// - public IEnumerable Get(IEnumerable ids) + public TModel Get(EntityId id) where TModel : IReadModel; + + /// + /// Gets a read model for every enitity that references the given entity id + /// with the given attribute. + /// + public IEnumerable GetReverse(EntityId id) + where TModel : IReadModel + where TAttribute : IAttribute; } diff --git a/src/NexusMods.EventSourcing.Abstractions/ITransaction.cs b/src/NexusMods.EventSourcing.Abstractions/ITransaction.cs index efa2de60..8906b293 100644 --- a/src/NexusMods.EventSourcing.Abstractions/ITransaction.cs +++ b/src/NexusMods.EventSourcing.Abstractions/ITransaction.cs @@ -40,4 +40,9 @@ void Add(EntityId entityId, TVal val, bool isAssert = true) /// Commits the transaction /// ICommitResult Commit(); + + /// + /// Gets the temporary id for the transaction + /// + public TxId ThisTxId { get; } } diff --git a/src/NexusMods.EventSourcing.Abstractions/Models/AReadModel.cs b/src/NexusMods.EventSourcing.Abstractions/Models/AReadModel.cs index 3bc59f1b..659adf54 100644 --- a/src/NexusMods.EventSourcing.Abstractions/Models/AReadModel.cs +++ b/src/NexusMods.EventSourcing.Abstractions/Models/AReadModel.cs @@ -1,4 +1,7 @@ -namespace NexusMods.EventSourcing.Abstractions.Models; +using System; +using System.Collections.Generic; + +namespace NexusMods.EventSourcing.Abstractions.Models; /// /// Base class for all read models. @@ -23,8 +26,40 @@ internal AReadModel(EntityId id) Id = id; } + /// + /// Retrieves the read model from the database + /// + /// + /// + /// + /// + protected TReadModel Get(EntityId entityId) + where TReadModel : AReadModel, IReadModel + { + return Db.Get(entityId); + } + + /// + /// Retrieves the matching read models from the database via the specified reverse lookup attribute + /// + /// + /// + /// + /// + protected IEnumerable GetReverse() + where TReadModel : AReadModel, IReadModel + where TAttribute : ScalarAttribute + { + return Db.GetReverse(Id); + } + /// /// The base identifier for the entity. /// public EntityId Id { get; internal set; } + + /// + /// The database this read model is associated with. + /// + public IDb Db { get; internal set; } = null!; } diff --git a/src/NexusMods.EventSourcing.Abstractions/Models/ReverseLookupAttribute.cs b/src/NexusMods.EventSourcing.Abstractions/Models/ReverseLookupAttribute.cs new file mode 100644 index 00000000..6379306c --- /dev/null +++ b/src/NexusMods.EventSourcing.Abstractions/Models/ReverseLookupAttribute.cs @@ -0,0 +1,12 @@ +using System; + +namespace NexusMods.EventSourcing.Abstractions.Models; + +/// +/// Defines a backwards lookup attribute +/// +public class ReverseLookupAttribute : Attribute +where TAttribute : ScalarAttribute +{ + +} diff --git a/src/NexusMods.EventSourcing.DatomStore/BuiltInSerializers/EntityIdSerializer.cs b/src/NexusMods.EventSourcing.DatomStore/BuiltInSerializers/EntityIdSerializer.cs new file mode 100644 index 00000000..34db9379 --- /dev/null +++ b/src/NexusMods.EventSourcing.DatomStore/BuiltInSerializers/EntityIdSerializer.cs @@ -0,0 +1,31 @@ +using System; +using System.Buffers; +using System.Buffers.Binary; +using NexusMods.EventSourcing.Abstractions; + +namespace NexusMods.EventSourcing.DatomStore.BuiltInSerializers; + +public class EntityIdSerialzer : IValueSerializer +{ + public Type NativeType => typeof(EntityId); + + public static readonly UInt128 Id = "E2C3185E-C082-4641-B25E-7CEC803A2F48".ToUInt128Guid(); + public UInt128 UniqueId => Id; + public int Compare(ReadOnlySpan a, ReadOnlySpan b) + { + return BinaryPrimitives.ReadUInt64LittleEndian(a).CompareTo(BinaryPrimitives.ReadUInt64LittleEndian(b)); + } + + public void Write(EntityId value, TWriter buffer) where TWriter : IBufferWriter + { + var span = buffer.GetSpan(8); + BinaryPrimitives.WriteUInt64LittleEndian(span, value.Value); + buffer.Advance(8); + } + + public int Read(ReadOnlySpan buffer, out EntityId val) + { + val = EntityId.From(BinaryPrimitives.ReadUInt64LittleEndian(buffer)); + return 8; + } +} diff --git a/src/NexusMods.EventSourcing.DatomStore/BuiltInSerializers/TxIdSerializer.cs b/src/NexusMods.EventSourcing.DatomStore/BuiltInSerializers/TxIdSerializer.cs new file mode 100644 index 00000000..77451c31 --- /dev/null +++ b/src/NexusMods.EventSourcing.DatomStore/BuiltInSerializers/TxIdSerializer.cs @@ -0,0 +1,30 @@ +using System; +using System.Buffers; +using System.Buffers.Binary; +using NexusMods.EventSourcing.Abstractions; +namespace NexusMods.EventSourcing.DatomStore.BuiltInSerializers; + +public class TxIdSerializer : IValueSerializer +{ + public Type NativeType => typeof(TxId); + + public static readonly UInt128 Id = "BB2B2BAF-9AA8-4DB0-8BFC-A0A853ED9BA0".ToUInt128Guid(); + public UInt128 UniqueId => Id; + public int Compare(ReadOnlySpan a, ReadOnlySpan b) + { + return BinaryPrimitives.ReadUInt64LittleEndian(a).CompareTo(BinaryPrimitives.ReadUInt64LittleEndian(b)); + } + + public void Write(TxId value, TWriter buffer) where TWriter : IBufferWriter + { + var span = buffer.GetSpan(8); + BinaryPrimitives.WriteUInt64LittleEndian(span, value.Value); + buffer.Advance(8); + } + + public int Read(ReadOnlySpan buffer, out TxId val) + { + val = TxId.From(BinaryPrimitives.ReadUInt64LittleEndian(buffer)); + return 8; + } +} diff --git a/src/NexusMods.EventSourcing.DatomStore/Indexes/AVTEIndex.cs b/src/NexusMods.EventSourcing.DatomStore/Indexes/AVTEIndex.cs new file mode 100644 index 00000000..1f7fc447 --- /dev/null +++ b/src/NexusMods.EventSourcing.DatomStore/Indexes/AVTEIndex.cs @@ -0,0 +1,135 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Runtime.InteropServices; +using NexusMods.EventSourcing.Abstractions; +using Reloaded.Memory.Extensions; +using RocksDbSharp; + +namespace NexusMods.EventSourcing.DatomStore.Indexes; + +public class AVTEIndex(AttributeRegistry registry) : + AIndexDefinition(registry, "avte"), IComparatorIndex +{ + public static unsafe int Compare(AIndexDefinition idx, KeyHeader* a, uint aLength, KeyHeader* b, uint bLength) + { + // Attribute, Value, TX, Entity + var cmp = KeyHeader.CompareAttribute(a, b); + if (cmp != 0) return cmp; + cmp = KeyHeader.CompareValues(idx.Registry, a, aLength, b, bLength); + if (cmp != 0) return cmp; + cmp = KeyHeader.CompareTx(a, b); + if (cmp != 0) return cmp; + cmp = KeyHeader.CompareEntity(a, b); + if (cmp != 0) return cmp; + return KeyHeader.CompareIsAssert(a, b); + } + + public unsafe struct AVTEIterator : IDisposable + { + private readonly KeyHeader* _key; + private KeyHeader* _current; + private UIntPtr _currentLength; + private readonly Iterator _iterator; + private readonly AttributeRegistry _registry; + private bool _needsSeek; + + public AVTEIterator(ulong txId, AttributeRegistry registry, AVTEIndex idx) + { + _registry = registry; + _iterator = idx.Db.NewIterator(idx.ColumnFamilyHandle); + _key = (KeyHeader*)Marshal.AllocHGlobal(KeyHeader.Size); + _key->Entity = ulong.MaxValue; + _key->AttributeId = ulong.MaxValue; + _key->Tx = txId; + _key->IsAssert = true; + _needsSeek = true; + } + + + public void Set() where TAttribute : IAttribute + { + _key->Entity = ulong.MaxValue; + _key->AttributeId = _registry.GetAttributeId(); + _needsSeek = true; + } + + public IDatom Current + { + get + { + Debug.Assert(!_needsSeek, "Must call Next() before accessing Current"); + var currentValue = new ReadOnlySpan((byte*)_current + KeyHeader.Size, (int)_currentLength - KeyHeader.Size); + return _registry.ReadDatom(ref *_current, currentValue); + } + } + + public EntityId EntityId => EntityId.From(_current->Entity); + + public TValue GetValue() + where TAttribute : IAttribute + { + Debug.Assert(!_needsSeek, "Must call Next() before accessing GetValue"); + var currentValue = new ReadOnlySpan((byte*)_current + KeyHeader.Size, (int)_currentLength - KeyHeader.Size); + return _registry.ReadValue(ref *_current, currentValue); + + } + + public ulong AttributeId + { + get + { + Debug.Assert(!_needsSeek, "Must call Next() before accessing AttributeId"); + return _current->AttributeId; + } + } + + public ReadOnlySpan ValueSpan => _iterator.GetKeySpan().SliceFast(KeyHeader.Size); + + public bool Next() + { + if (_needsSeek) + { + _iterator.SeekForPrev((byte*)_key, KeyHeader.Size); + _needsSeek = false; + } + else + { + _key->Entity = _current->Entity - 1; + _iterator.Prev(); + } + + while (true) + { + + if (!_iterator.Valid()) return false; + + _current = (KeyHeader*)Native.Instance.rocksdb_iter_key(_iterator.Handle, out _currentLength); + + Debug.Assert(_currentLength < KeyHeader.Size, "Key length is less than KeyHeader.Size"); + + if (_current->AttributeId != _key->AttributeId) + return false; + + if (_current->Tx > _key->Tx) + { + _iterator.Prev(); + continue; + } + + if (_current->Entity > _key->Entity) + { + _iterator.Prev(); + continue; + } + + return true; + } + } + public void Dispose() + { + _iterator.Dispose(); + Marshal.FreeHGlobal((IntPtr)_key); + } + } +} diff --git a/src/NexusMods.EventSourcing.DatomStore/RocksDBDatomStore.cs b/src/NexusMods.EventSourcing.DatomStore/RocksDBDatomStore.cs index 036d66ee..847dcbb0 100644 --- a/src/NexusMods.EventSourcing.DatomStore/RocksDBDatomStore.cs +++ b/src/NexusMods.EventSourcing.DatomStore/RocksDBDatomStore.cs @@ -23,6 +23,7 @@ public class RocksDBDatomStore : IDatomStore private ulong _tx; private readonly AETVIndex _avetIndex; private readonly EATVIndex _eatvIndex; + private readonly AVTEIndex _avteIndex; public RocksDBDatomStore(ILogger logger, AttributeRegistry registry, DatomStoreSettings settings) { @@ -42,6 +43,8 @@ public RocksDBDatomStore(ILogger logger, AttributeRegistry re _eatvIndex.Init(_db); _avetIndex = new AETVIndex(_registry); _avetIndex.Init(_db); + _avteIndex = new AVTEIndex(_registry); + _avteIndex.Init(_db); _pooledWriter = new PooledMemoryBufferWriter(128); @@ -68,6 +71,7 @@ private void Serialize(WriteBatch batch, ulong e, TVal val, ulong t var span = _pooledWriter.GetWrittenSpan(); _eatvIndex.Put(batch, span); _avetIndex.Put(batch, span); + _avteIndex.Put(batch, span); } private struct TransactSink(RocksDBDatomStore store, WriteBatch batch, ulong tx) : IDatomSink @@ -130,6 +134,16 @@ public Expression GetValueReadExpression(Type attribute, Expression valueSpan, o return _registry.GetReadExpression(attribute, valueSpan, out attributeId); } + public IEnumerable ReverseLookup(TxId txId) where TAttribute : IAttribute + { + using var iterator = new AVTEIndex.AVTEIterator(txId.Value, _registry, _avteIndex); + iterator.Set(); + while (iterator.Next()) + { + yield return iterator.EntityId; + } + } + public void Dispose() { _db.Dispose(); diff --git a/src/NexusMods.EventSourcing.DatomStore/Services.cs b/src/NexusMods.EventSourcing.DatomStore/Services.cs index 90f133cc..dab1e318 100644 --- a/src/NexusMods.EventSourcing.DatomStore/Services.cs +++ b/src/NexusMods.EventSourcing.DatomStore/Services.cs @@ -27,7 +27,9 @@ public static IServiceCollection AddDatomStore(this IServiceCollection services) .AddValueSerializer() .AddValueSerializer() .AddValueSerializer() - .AddValueSerializer(); + .AddValueSerializer() + .AddValueSerializer() + .AddValueSerializer(); return services; } diff --git a/src/NexusMods.EventSourcing/Connection.cs b/src/NexusMods.EventSourcing/Connection.cs index ddd24259..0aea6152 100644 --- a/src/NexusMods.EventSourcing/Connection.cs +++ b/src/NexusMods.EventSourcing/Connection.cs @@ -109,29 +109,28 @@ public ICommitResult Transact(IEnumerable datoms) var remaps = new Dictionary(); var datomsArray = datoms.ToArray(); + EntityId RemapFn(EntityId input) + { + if (Ids.GetPartition(input) == Ids.Partition.Tmp) + { + if (!remaps.TryGetValue(input.Value, out var id)) + { + var newId = _nextEntityId++; + remaps[input.Value] = newId; + return EntityId.From(newId); + } + return EntityId.From(id); + } + return input; + } + lock (_lock) { var newDatoms = new List(); foreach (var datom in datomsArray) { - var eid = datom.E; - if (Ids.GetPartition(eid) == Ids.Partition.Tmp) - { - if (!remaps.TryGetValue(eid, out var id)) - { - var newId = _nextEntityId++; - remaps[eid] = newId; - newDatoms.Add(datom.RemapEntityId(newId)); - } - else - { - newDatoms.Add(datom.RemapEntityId(id)); - } - } - else - { - newDatoms.Add(datom); - } + datom.Remap(RemapFn); + newDatoms.Add(datom); } var newTx = _store.Transact(newDatoms); TxId = newTx; diff --git a/src/NexusMods.EventSourcing/Db.cs b/src/NexusMods.EventSourcing/Db.cs index 68b9a53a..a858431c 100644 --- a/src/NexusMods.EventSourcing/Db.cs +++ b/src/NexusMods.EventSourcing/Db.cs @@ -28,7 +28,29 @@ public IEnumerable Get(IEnumerable ids) where TModel : foreach (var id in ids) { iterator.Set(id); - var model = reader(id, iterator); + var model = reader(id, iterator, this); + yield return model; + } + } + + public TModel Get(EntityId id) where TModel : IReadModel + { + using var iterator = store.EntityIterator(txId); + iterator.Set(id); + var reader = connection.ModelReflector.GetReader(); + return reader(id, iterator, this); + } + + /// + public IEnumerable GetReverse(EntityId id) where TAttribute : IAttribute where TModel : IReadModel + { + var iterator = store.ReverseLookup(txId); + using var entityIterator = store.EntityIterator(txId); + var reader = connection.ModelReflector.GetReader(); + foreach (var entityId in iterator) + { + entityIterator.Set(entityId); + var model = reader(entityId, entityIterator, this); yield return model; } } diff --git a/src/NexusMods.EventSourcing/ModelReflector.cs b/src/NexusMods.EventSourcing/ModelReflector.cs index ab5754b1..34066aaf 100644 --- a/src/NexusMods.EventSourcing/ModelReflector.cs +++ b/src/NexusMods.EventSourcing/ModelReflector.cs @@ -21,7 +21,7 @@ internal class ModelReflector(IDatomStore store) private delegate void EmitterFn(TTransaction tx, TReadModel model) where TReadModel : IReadModel; - internal delegate TReadModel ReaderFn(EntityId id, IEntityIterator iterator) + internal delegate TReadModel ReaderFn(EntityId id, IEntityIterator iterator, IDb db) where TReadModel : IReadModel; public void Add(TTransaction tx, IReadModel model) @@ -107,6 +107,8 @@ public ReaderFn MakeReader() where TModel : IReadModel var entityIdParameter = Expression.Parameter(typeof(EntityId), "entityId"); var iteratorParameter = Expression.Parameter(typeof(IEntityIterator), "iterator"); + var dbParameter = Expression.Parameter(typeof(IDb), "db"); + var newModelExpr = Expression.Variable(typeof(TModel), "newModel"); var spanExpr = Expression.Property(iteratorParameter, "ValueSpan"); @@ -115,6 +117,7 @@ public ReaderFn MakeReader() where TModel : IReadModel exprs.Add(Expression.Assign(newModelExpr, Expression.New(ctor, Expression.Constant(null, typeof(ITransaction))))); exprs.Add(Expression.Assign(Expression.Property(newModelExpr, "Id"), entityIdParameter)); + exprs.Add(Expression.Assign(Expression.Property(newModelExpr, "Db"), dbParameter)); exprs.Add(Expression.Label(whileTopLabel)); exprs.Add(Expression.IfThen( @@ -141,7 +144,7 @@ public ReaderFn MakeReader() where TModel : IReadModel var block = Expression.Block(new[] {newModelExpr}, exprs); - var lambda = Expression.Lambda>(block, entityIdParameter, iteratorParameter); + var lambda = Expression.Lambda>(block, entityIdParameter, iteratorParameter, dbParameter); return lambda.Compile(); } } diff --git a/src/NexusMods.EventSourcing/Transaction.cs b/src/NexusMods.EventSourcing/Transaction.cs index 6def27bb..b4a1214a 100644 --- a/src/NexusMods.EventSourcing/Transaction.cs +++ b/src/NexusMods.EventSourcing/Transaction.cs @@ -8,7 +8,7 @@ namespace NexusMods.EventSourcing; public class Transaction(Connection connection) : ITransaction { - private ulong _tempId = Ids.MinId(Ids.Partition.Tmp); + private ulong _tempId = Ids.MinId(Ids.Partition.Tmp) + 1; private ConcurrentBag _datoms = new(); private ConcurrentBag _models = new(); @@ -44,4 +44,7 @@ public ICommitResult Commit() } return connection.Transact(_datoms); } + + /// + public TxId ThisTxId => TxId.From(Ids.MinId(Ids.Partition.Tmp)); } diff --git a/tests/NexusMods.EventSourcing.TestModel/Model/Attributes/LoadoutAttributes.cs b/tests/NexusMods.EventSourcing.TestModel/Model/Attributes/LoadoutAttributes.cs new file mode 100644 index 00000000..084cdb4f --- /dev/null +++ b/tests/NexusMods.EventSourcing.TestModel/Model/Attributes/LoadoutAttributes.cs @@ -0,0 +1,16 @@ +using NexusMods.EventSourcing.Abstractions; + +namespace NexusMods.EventSourcing.TestModel.Model.Attributes; + +public class LoadoutAttributes +{ + /// + /// Name of the loadout + /// + public class Name : ScalarAttribute; + + /// + /// The last transaction that updated the loadout + /// + public class UpdatedTx : ScalarAttribute; +} diff --git a/tests/NexusMods.EventSourcing.TestModel/Model/Attributes/ModAttributes.cs b/tests/NexusMods.EventSourcing.TestModel/Model/Attributes/ModAttributes.cs new file mode 100644 index 00000000..cd9ed563 --- /dev/null +++ b/tests/NexusMods.EventSourcing.TestModel/Model/Attributes/ModAttributes.cs @@ -0,0 +1,21 @@ +using NexusMods.EventSourcing.Abstractions; + +namespace NexusMods.EventSourcing.TestModel.Model.Attributes; + +public class ModAttributes +{ + /// + /// Name of the loadout + /// + public class Name : ScalarAttribute; + + /// + /// The last transaction that updated the loadout + /// + public class UpdatedTx : ScalarAttribute; + + /// + /// The id of the loadout this mod belongs to + /// + public class LoadoutId : ScalarAttribute; +} diff --git a/tests/NexusMods.EventSourcing.TestModel/Model/Loadout.cs b/tests/NexusMods.EventSourcing.TestModel/Model/Loadout.cs new file mode 100644 index 00000000..b86ce9ce --- /dev/null +++ b/tests/NexusMods.EventSourcing.TestModel/Model/Loadout.cs @@ -0,0 +1,50 @@ +using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Abstractions.Models; +using NexusMods.EventSourcing.TestModel.Model.Attributes; + +namespace NexusMods.EventSourcing.TestModel.Model; + +public class Loadout(ITransaction? tx) : AReadModel(tx) +{ + /// + /// The name of the loadout. + /// + [From] + public required string Name { get; init; } + + /// + /// The last tx that updated the loadout. + /// + [From] + public required TxId Invalidator { get; init; } + + /// + /// The mods in the loadout. + /// + public IEnumerable Mods => GetReverse(); + + + /// + /// Create a new loadout with the given name. + /// + /// + /// + /// + public static Loadout Create(ITransaction tx, string name) + { + return new Loadout(tx) + { + Name = name, + Invalidator = tx.ThisTxId + }; + } + + /// + /// Updates this loadout marking it as touched by the given transaction. + /// + /// + public void Touch(ITransaction tx) + { + LoadoutAttributes.UpdatedTx.Add(tx, Id, tx.ThisTxId); + } +} diff --git a/tests/NexusMods.EventSourcing.TestModel/Model/Mod.cs b/tests/NexusMods.EventSourcing.TestModel/Model/Mod.cs new file mode 100644 index 00000000..39304255 --- /dev/null +++ b/tests/NexusMods.EventSourcing.TestModel/Model/Mod.cs @@ -0,0 +1,37 @@ +using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Abstractions.Models; +using NexusMods.EventSourcing.TestModel.Model.Attributes; + +namespace NexusMods.EventSourcing.TestModel.Model; + +public class Mod(ITransaction? tx) : AReadModel(tx) +{ + + [From] + public required string Name { get; init; } + + + [From] + public required EntityId LoadoutId { get; init; } + + /// + /// The loadout for this mod. + /// + public Loadout Loadout => Get(LoadoutId); + + + public static Mod Create(ITransaction tx, string name, EntityId loadoutId) + { + var mod = new Mod(tx) + { + Name = name, + LoadoutId = loadoutId + }; + return mod; + } + + public void Touch(ITransaction tx) + { + Loadout.Touch(tx); + } +} diff --git a/tests/NexusMods.EventSourcing.TestModel/Services.cs b/tests/NexusMods.EventSourcing.TestModel/Services.cs index 6be3d6db..72cf7667 100644 --- a/tests/NexusMods.EventSourcing.TestModel/Services.cs +++ b/tests/NexusMods.EventSourcing.TestModel/Services.cs @@ -10,5 +10,10 @@ public static IServiceCollection AddTestModel(this IServiceCollection services) .AddAttribute() .AddAttribute() .AddAttribute() - .AddAttribute(); + .AddAttribute() + .AddAttribute() + .AddAttribute() + .AddAttribute() + .AddAttribute() + .AddAttribute(); } diff --git a/tests/NexusMods.EventSourcing.Tests/DbTests.cs b/tests/NexusMods.EventSourcing.Tests/DbTests.cs index 759a43ce..fbc36c50 100644 --- a/tests/NexusMods.EventSourcing.Tests/DbTests.cs +++ b/tests/NexusMods.EventSourcing.Tests/DbTests.cs @@ -171,5 +171,31 @@ public void CanGetCommitUpdates() } + [Fact] + public void CanGetChildEntities() + { + var tx = Connection.BeginTransaction(); + var loadout = Loadout.Create(tx, "Test Loadout"); + var mod1 = Mod.Create(tx, "Test Mod 1", loadout.Id); + var mod2 = Mod.Create(tx, "Test Mod 2", loadout.Id); + var result = tx.Commit(); + + var newDb = Connection.Db; + + loadout = newDb.Get([result[loadout.Id]]).First(); + + loadout.Mods.Count().Should().Be(2); + loadout.Mods.Select(m => m.Name).Should().BeEquivalentTo(["Test Mod 1", "Test Mod 2"]); + + var firstMod = loadout.Mods.First(); + Ids.IsPartition(firstMod.Loadout.Id.Value, Ids.Partition.Entity) + .Should() + .Be(true, "the temp id should be replaced with a real id"); + firstMod.Loadout.Id.Should().Be(loadout.Id); + firstMod.Db.Should().Be(newDb); + loadout.Name.Should().Be("Test Loadout"); + firstMod.Loadout.Name.Should().Be("Test Loadout"); + } + }