From a48dff3bfd6c6063548487c87eda7993ff90f849 Mon Sep 17 00:00:00 2001 From: halgari Date: Wed, 10 Jan 2024 10:50:31 -0700 Subject: [PATCH] Fix several serialization bugs --- .../EntityId.cs | 20 +++++++- src/NexusMods.EventSourcing/EventFormatter.cs | 50 ------------------- .../EventSerializer.cs | 38 -------------- .../Serialization/ArraySerializer.cs | 6 ++- .../Serialization/EntityIdSerializer.cs | 2 +- .../Serialization/StringSerializer.cs | 5 +- src/NexusMods.EventSourcing/Services.cs | 2 +- .../Events/AddCollection.cs | 3 +- .../BasicFunctionalityTests.cs | 7 +-- .../EventSerializerTests.cs | 3 +- .../InMemoryEventStoreTests.cs | 5 +- .../EventStoreTests/RocksDBEventStoreTests.cs | 5 +- .../SerializationTests.cs | 23 +++++++++ .../NexusMods.EventSourcing.Tests/Startup.cs | 3 +- 14 files changed, 64 insertions(+), 108 deletions(-) delete mode 100644 src/NexusMods.EventSourcing/EventFormatter.cs delete mode 100644 src/NexusMods.EventSourcing/EventSerializer.cs diff --git a/src/NexusMods.EventSourcing.Abstractions/EntityId.cs b/src/NexusMods.EventSourcing.Abstractions/EntityId.cs index 8b696074..5757a33f 100644 --- a/src/NexusMods.EventSourcing.Abstractions/EntityId.cs +++ b/src/NexusMods.EventSourcing.Abstractions/EntityId.cs @@ -18,7 +18,7 @@ public static EntityId NewId() var value = BinaryPrimitives.ReadUInt128BigEndian(bytes); return From(value); } - + public static EntityId From(ReadOnlySpan data) => new(BinaryPrimitives.ReadUInt128BigEndian(data)); public void TryWriteBytes(Span span) @@ -48,6 +48,13 @@ public void TryWriteBytes(Span span) /// public static EntityId From(UInt128 id) => new(EntityId.From(id)); + /// + /// Reads the from the specified . + /// + /// + /// + public static EntityId From(ReadOnlySpan data) => new(EntityId.From(data)); + /// @@ -55,7 +62,16 @@ public void TryWriteBytes(Span span) /// /// /// - public static EntityId From(string id) => From(UInt128.Parse(id, NumberStyles.HexNumber)); + public static EntityId From(string id) + { + if (Guid.TryParse(id, out var guid)) + { + Span bytes = stackalloc byte[16]; + guid.TryWriteBytes(bytes); + return From(BinaryPrimitives.ReadUInt128BigEndian(bytes)); + } + return From(UInt128.Parse(id, NumberStyles.HexNumber)); + } /// diff --git a/src/NexusMods.EventSourcing/EventFormatter.cs b/src/NexusMods.EventSourcing/EventFormatter.cs deleted file mode 100644 index fe00f3b0..00000000 --- a/src/NexusMods.EventSourcing/EventFormatter.cs +++ /dev/null @@ -1,50 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using MemoryPack; -using MemoryPack.Formatters; -using NexusMods.EventSourcing.Abstractions; - -namespace NexusMods.EventSourcing; - -internal class EventFormatter : MemoryPackFormatter -{ - private static Guid _zeroGuid = Guid.Empty; - private readonly Dictionary _eventByGuid; - private readonly Dictionary _eventsByType; - - public EventFormatter(IEnumerable events) - { - var eventsArray = events.ToArray(); - _eventByGuid = eventsArray.ToDictionary(e => e.Id, e => e.Type); - _eventsByType = eventsArray.ToDictionary(e => e.Type, e => e.Id); - } - - public override void Serialize(ref MemoryPackWriter writer, scoped ref IEvent? value) - { - if (value == null) - { - writer.WriteValue(_zeroGuid); - return; - } - - var type = value.GetType(); - writer.WriteValue(_eventsByType[type]); - writer.WriteValue(type, value); - } - - public override void Deserialize(ref MemoryPackReader reader, scoped ref IEvent? value) - { - throw new NotImplementedException(); - /* - var readValue = reader.ReadValue(); - if (readValue == _zeroGuid) - { - value = null; - return; - } - var mappedType = _eventByGuid[readValue]; - value = (IEvent)reader.ReadValue(mappedType)!; - */ - } -} diff --git a/src/NexusMods.EventSourcing/EventSerializer.cs b/src/NexusMods.EventSourcing/EventSerializer.cs deleted file mode 100644 index 814d2a83..00000000 --- a/src/NexusMods.EventSourcing/EventSerializer.cs +++ /dev/null @@ -1,38 +0,0 @@ -using System; -using System.Buffers; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using MemoryPack; -using MemoryPack.Formatters; -using NexusMods.EventSourcing.Abstractions; - -namespace NexusMods.EventSourcing; - -public class EventSerializer : IEventSerializer -{ - private readonly PooledMemoryBufferWriter _writer; - - public EventSerializer(IEnumerable events) - { - var formatter = new EventFormatter(events); - if (!MemoryPackFormatterProvider.IsRegistered()) - MemoryPackFormatterProvider.Register(formatter); - _writer = new PooledMemoryBufferWriter(1024 * 10); // 10kb - } - - public ReadOnlySpan Serialize(IEvent @event) - { - _writer.Reset(); - MemoryPackSerializer.Serialize(_writer, @event); - return _writer.GetWrittenSpan(); - } - - public IEvent Deserialize(ReadOnlySpan data) - { - return MemoryPackSerializer.Deserialize(data)!; - } - - - -} diff --git a/src/NexusMods.EventSourcing/Serialization/ArraySerializer.cs b/src/NexusMods.EventSourcing/Serialization/ArraySerializer.cs index 10d44ca6..5d28cdf5 100644 --- a/src/NexusMods.EventSourcing/Serialization/ArraySerializer.cs +++ b/src/NexusMods.EventSourcing/Serialization/ArraySerializer.cs @@ -70,11 +70,13 @@ public void Serialize(TItem[] value, TWriter output) where TWriter : IB { var totalSize = sizeof(ushort) + (itemSize * value.Length); var span = output.GetSpan(totalSize); - BinaryPrimitives.WriteUInt32BigEndian(span, (ushort)value.Length); + BinaryPrimitives.WriteUInt16BigEndian(span, (ushort)value.Length); + var offset = sizeof(ushort); foreach (var item in value) { - itemSerializer.Serialize(item, span.SliceFast(itemSize, itemSize)); + itemSerializer.Serialize(item, span.SliceFast(offset, itemSize)); + offset += itemSize; } output.Advance(totalSize); } diff --git a/src/NexusMods.EventSourcing/Serialization/EntityIdSerializer.cs b/src/NexusMods.EventSourcing/Serialization/EntityIdSerializer.cs index 31babdd6..d444e771 100644 --- a/src/NexusMods.EventSourcing/Serialization/EntityIdSerializer.cs +++ b/src/NexusMods.EventSourcing/Serialization/EntityIdSerializer.cs @@ -77,6 +77,6 @@ public void Serialize(EntityId value, Span output) public EntityId Deserialize(ReadOnlySpan from) { - return EntityId.From(BinaryPrimitives.ReadUInt64BigEndian(from)); + return EntityId.From(from); } } diff --git a/src/NexusMods.EventSourcing/Serialization/StringSerializer.cs b/src/NexusMods.EventSourcing/Serialization/StringSerializer.cs index 9798fe4a..e10533f3 100644 --- a/src/NexusMods.EventSourcing/Serialization/StringSerializer.cs +++ b/src/NexusMods.EventSourcing/Serialization/StringSerializer.cs @@ -2,6 +2,7 @@ using System.Buffers; using System.Buffers.Binary; using NexusMods.EventSourcing.Abstractions.Serialization; +using Reloaded.Memory.Extensions; namespace NexusMods.EventSourcing.Serialization; @@ -23,14 +24,14 @@ public void Serialize(string value, TWriter output) where TWriter : IBu var size = System.Text.Encoding.UTF8.GetByteCount(value); var span = output.GetSpan(size + 2); BinaryPrimitives.WriteUInt16LittleEndian(span, (ushort)size); - System.Text.Encoding.UTF8.GetBytes(value, span[2..]); + System.Text.Encoding.UTF8.GetBytes(value, span.SliceFast(2)); output.Advance(size + 2); } public int Deserialize(ReadOnlySpan from, out string value) { var size = BinaryPrimitives.ReadUInt16LittleEndian(from); - value = System.Text.Encoding.UTF8.GetString(from[2..(2 + size)]); + value = System.Text.Encoding.UTF8.GetString(from.SliceFast(2, size)); return size + 2; } } diff --git a/src/NexusMods.EventSourcing/Services.cs b/src/NexusMods.EventSourcing/Services.cs index 053fb8bf..0ef4a32d 100644 --- a/src/NexusMods.EventSourcing/Services.cs +++ b/src/NexusMods.EventSourcing/Services.cs @@ -15,7 +15,7 @@ public static IServiceCollection AddEventSourcing(this IServiceCollection servic .AddSingleton() .AddSingleton() .AddSingleton() - .AddSingleton() + .AddSingleton() .AddSingleton() .AddSingleton() .AddSingleton() diff --git a/tests/NexusMods.EventSourcing.TestModel/Events/AddCollection.cs b/tests/NexusMods.EventSourcing.TestModel/Events/AddCollection.cs index 221002e5..fd4a2d4b 100644 --- a/tests/NexusMods.EventSourcing.TestModel/Events/AddCollection.cs +++ b/tests/NexusMods.EventSourcing.TestModel/Events/AddCollection.cs @@ -5,8 +5,7 @@ namespace NexusMods.EventSourcing.TestModel.Events; [EventId("9C6CF87E-9469-4C9E-87AB-6FE7EF331358")] -[MemoryPackable] -public partial record AddCollection(EntityId CollectionId, string Name, EntityId LoadoutId, EntityId[] Mods) : IEvent +public record AddCollection(EntityId CollectionId, string Name, EntityId LoadoutId, EntityId[] Mods) : IEvent { public void Apply(T context) where T : IEventContext { diff --git a/tests/NexusMods.EventSourcing.Tests/BasicFunctionalityTests.cs b/tests/NexusMods.EventSourcing.Tests/BasicFunctionalityTests.cs index f62d8601..ca5a0148 100644 --- a/tests/NexusMods.EventSourcing.Tests/BasicFunctionalityTests.cs +++ b/tests/NexusMods.EventSourcing.Tests/BasicFunctionalityTests.cs @@ -1,5 +1,6 @@ using System.Collections.Specialized; using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Serialization; using NexusMods.EventSourcing.TestModel; using NexusMods.EventSourcing.TestModel.Events; using NexusMods.EventSourcing.TestModel.Model; @@ -9,9 +10,9 @@ namespace NexusMods.EventSourcing.Tests; public class BasicFunctionalityTests { private readonly IEntityContext _ctx; - public BasicFunctionalityTests(EventSerializer serializer) + public BasicFunctionalityTests(BinaryEventSerializer serializer) { - var store = new InMemoryEventStore(serializer); + var store = new InMemoryEventStore(serializer); _ctx = new EntityContext(store); } @@ -57,7 +58,7 @@ public void CanLinkEntities() tx.Commit(); var loadout = _ctx.Get(loadoutId); - loadout.Mods.Count().Should().Be(1); + loadout.Mods.Count.Should().Be(1); loadout.Mods.First().Name.Should().Be("First Mod"); var mod = _ctx.Get(modId); diff --git a/tests/NexusMods.EventSourcing.Tests/EventSerializerTests.cs b/tests/NexusMods.EventSourcing.Tests/EventSerializerTests.cs index 3698990a..d99f1afe 100644 --- a/tests/NexusMods.EventSourcing.Tests/EventSerializerTests.cs +++ b/tests/NexusMods.EventSourcing.Tests/EventSerializerTests.cs @@ -1,10 +1,11 @@ using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Serialization; using NexusMods.EventSourcing.TestModel.Events; using NexusMods.EventSourcing.TestModel.Model; namespace NexusMods.EventSourcing.Tests; -public class EventSerializerTests(EventSerializer serializer) +public class EventSerializerTests(BinaryEventSerializer serializer) { [Fact] diff --git a/tests/NexusMods.EventSourcing.Tests/EventStoreTests/InMemoryEventStoreTests.cs b/tests/NexusMods.EventSourcing.Tests/EventStoreTests/InMemoryEventStoreTests.cs index 7b2443e2..5e1f28f0 100644 --- a/tests/NexusMods.EventSourcing.Tests/EventStoreTests/InMemoryEventStoreTests.cs +++ b/tests/NexusMods.EventSourcing.Tests/EventStoreTests/InMemoryEventStoreTests.cs @@ -1,8 +1,9 @@ +using NexusMods.EventSourcing.Serialization; using NexusMods.EventSourcing.TestModel; namespace NexusMods.EventSourcing.Tests.EventStoreTests; -public class InMemoryEventStoreTests : AEventStoreTest> +public class InMemoryEventStoreTests : AEventStoreTest> { - public InMemoryEventStoreTests(EventSerializer serializer) : base(new InMemoryEventStore(serializer)) { } + public InMemoryEventStoreTests(BinaryEventSerializer serializer) : base(new InMemoryEventStore(serializer)) { } } diff --git a/tests/NexusMods.EventSourcing.Tests/EventStoreTests/RocksDBEventStoreTests.cs b/tests/NexusMods.EventSourcing.Tests/EventStoreTests/RocksDBEventStoreTests.cs index 8b09e5c2..7ef3f857 100644 --- a/tests/NexusMods.EventSourcing.Tests/EventStoreTests/RocksDBEventStoreTests.cs +++ b/tests/NexusMods.EventSourcing.Tests/EventStoreTests/RocksDBEventStoreTests.cs @@ -1,10 +1,11 @@ using NexusMods.EventSourcing.RocksDB; +using NexusMods.EventSourcing.Serialization; using NexusMods.Paths; namespace NexusMods.EventSourcing.Tests.EventStoreTests; -public class RocksDBEventStoreTests(EventSerializer serializer) : AEventStoreTest>( - new RocksDBEventStore(serializer, new Settings +public class RocksDBEventStoreTests(BinaryEventSerializer serializer) : AEventStoreTest>( + new RocksDBEventStore(serializer, new Settings { StorageLocation = FileSystem.Shared.GetKnownPath(KnownPath.EntryDirectory) .Combine("FasterKV.EventStore" + Guid.NewGuid()) diff --git a/tests/NexusMods.EventSourcing.Tests/SerializationTests.cs b/tests/NexusMods.EventSourcing.Tests/SerializationTests.cs index 3533406b..4f0fb7a6 100644 --- a/tests/NexusMods.EventSourcing.Tests/SerializationTests.cs +++ b/tests/NexusMods.EventSourcing.Tests/SerializationTests.cs @@ -19,7 +19,30 @@ public void CanSerializeEvents() var deserialized = serializer.Deserialize(serialized); deserialized.Should().Be(evnt); + } + + [Theory] + [MemberData(nameof(ExampleEvents))] + public void CanSerializeAllEvents(IEvent @event) + { + var serialized = serializer.Serialize(@event); + var deserialized = serializer.Deserialize(serialized); + deserialized.Should() + .BeEquivalentTo(@event, opts => opts.RespectingRuntimeTypes()); + } + public static IEnumerable ExampleEvents() + { + var events = new List + { + new object[]{new CreateLoadout(EntityId.NewId(), "Test")}, + new object[]{new RenameLoadout(EntityId.NewId(), "Test")}, + new object[]{new AddMod("New Mod", true, EntityId.NewId(), new EntityId())}, + new object[]{new AddCollection(EntityId.NewId(), "NewCollection", EntityId.NewId(), + [EntityId.NewId()]) + }, + }; + return events; } diff --git a/tests/NexusMods.EventSourcing.Tests/Startup.cs b/tests/NexusMods.EventSourcing.Tests/Startup.cs index a851b35d..75ddddf9 100644 --- a/tests/NexusMods.EventSourcing.Tests/Startup.cs +++ b/tests/NexusMods.EventSourcing.Tests/Startup.cs @@ -11,8 +11,7 @@ public class Startup public void ConfigureServices(IServiceCollection container) { container - .AddSingleton() - .AddSingleton>() + .AddSingleton>() .AddEvents() .AddEvent() .AddEventSourcing()