Skip to content

Commit b883f9d

Browse files
halgarihalgari
authored
Merge pull request #27 from Nexus-Mods/support-multi-value-attributes
Added support for multi-value attributes
2 parents 1b54eb4 + 377f651 commit b883f9d

File tree

33 files changed

+284
-30
lines changed

33 files changed

+284
-30
lines changed

‎src/NexusMods.MneumonicDB.Abstractions/IDb.cs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,4 +64,11 @@ public IEnumerable<IReadDatom> Datoms<TAttribute>()
6464
/// Create a new iterator for the given index type.
6565
/// </summary>
6666
IDatomSource Iterate(IndexType index);
67+
68+
/// <summary>
69+
/// Gets all values for the given attribute on the given entity. There's no reason to use this
70+
/// on attributes that are not multi-valued.
71+
/// </summary>
72+
IEnumerable<TValueType> GetAll<TAttribute, TValueType>(ref ModelHeader model, EntityId modelId)
73+
where TAttribute : IAttribute<TValueType>;
6774
}

‎src/NexusMods.MneumonicDB.Abstractions/ScalarAttribute.cs

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
using System;
22
using System.Buffers;
3+
using System.Collections.Generic;
34
using NexusMods.MneumonicDB.Abstractions.Internals;
45
using NexusMods.MneumonicDB.Abstractions.Models;
56

@@ -21,11 +22,11 @@ public class ScalarAttribute<TAttribute, TValueType> : IAttribute<TValueType>
2122
protected ScalarAttribute(string uniqueName = "",
2223
bool isIndexed = false,
2324
bool keepHistory = true,
24-
bool multiArity = false)
25+
bool multiValued = false)
2526
{
2627
IsIndexed = isIndexed;
2728
KeepHistory = keepHistory;
28-
MultiArity = multiArity;
29+
Multivalued = multiValued;
2930
Id = uniqueName == "" ? Symbol.Intern(typeof(TAttribute).FullName!) : Symbol.InternPreSanitized(uniqueName);
3031
}
3132

@@ -37,7 +38,7 @@ protected ScalarAttribute(Symbol symbol)
3738
Id = symbol;
3839
}
3940

40-
public bool MultiArity { get; }
41+
public bool Multivalued { get; }
4142

4243
public bool KeepHistory { get; }
4344

@@ -128,6 +129,11 @@ public static TValueType Get(ref ModelHeader model)
128129
return model.Db.Get<TAttribute, TValueType>(ref model, model.Id);
129130
}
130131

132+
public static IEnumerable<TValueType> GetAll(ref ModelHeader model)
133+
{
134+
return model.Db.GetAll<TAttribute, TValueType>(ref model, model.Id);
135+
}
136+
131137
/// <inheritdoc />
132138
public static void Add(ref ModelHeader model, TValueType value)
133139
{

‎src/NexusMods.MneumonicDB.Storage/DatomStore.cs

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -330,10 +330,7 @@ private unsafe void Log(PendingTransaction pendingTransaction, out StoreResult r
330330
foreach (var datom in pendingTransaction.Data)
331331
{
332332
_writer.Reset();
333-
unsafe
334-
{
335-
_writer.Advance(sizeof(KeyPrefix));
336-
}
333+
_writer.Advance(sizeof(KeyPrefix));
337334

338335
var isRemapped = Ids.IsPartition(datom.E.Value, Ids.Partition.Tmp);
339336
datom.Explode(_registry, remapFn, out var e, out var a, _writer, out var isRetract);

‎src/NexusMods.MneumonicDB/Db.cs

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ internal class Db : IDb
1616
private readonly Connection _connection;
1717
private readonly AttributeRegistry _registry;
1818
private readonly EntityCache _cache;
19-
private readonly ConcurrentDictionary<EntityId, EntityId[]> _reverseCache = new();
19+
private readonly ConcurrentDictionary<(EntityId, Type), EntityId[]> _reverseCache = new();
2020
internal readonly ISnapshot Snapshot;
2121

2222
public Db(ISnapshot snapshot, Connection connection, TxId txId, AttributeRegistry registry)
@@ -39,7 +39,6 @@ public IEnumerable<TModel> Get<TModel>(IEnumerable<EntityId> ids)
3939
{
4040
yield return Get<TModel>(id);
4141
}
42-
4342
}
4443

4544
public TValue Get<TAttribute, TValue>(ref ModelHeader header, EntityId id)
@@ -65,7 +64,7 @@ public TModel[] GetReverse<TAttribute, TModel>(EntityId id)
6564
where TAttribute : IAttribute<EntityId>
6665
where TModel : struct, IEntity
6766
{
68-
if (!_reverseCache.TryGetValue(id, out var eIds))
67+
if (!_reverseCache.TryGetValue((id, typeof(TAttribute)), out var eIds))
6968
{
7069
using var attrSource = Snapshot.GetIterator(IndexType.VAETCurrent);
7170
var attrId = _registry.GetAttributeId<TAttribute>();
@@ -76,7 +75,7 @@ public TModel[] GetReverse<TAttribute, TModel>(EntityId id)
7675
.Select(c => c.CurrentKeyPrefix().E)
7776
.ToArray();
7877

79-
_reverseCache[id] = eIds;
78+
_reverseCache[(id, typeof(TAttribute))] = eIds;
8079
}
8180

8281
var results = GC.AllocateUninitializedArray<TModel>(eIds.Length);
@@ -121,5 +120,11 @@ public IDatomSource Iterate(IndexType index)
121120
return Snapshot.GetIterator(index);
122121
}
123122

123+
public IEnumerable<TValueType> GetAll<TAttribute, TValueType>(ref ModelHeader model, EntityId modelId)
124+
where TAttribute : IAttribute<TValueType>
125+
{
126+
return _cache.GetAll<TAttribute, TValueType>(ref model);
127+
}
128+
124129
public void Dispose() { }
125130
}

‎src/NexusMods.MneumonicDB/EntityCache.cs

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,4 +114,44 @@ private class Entry
114114
public ushort[] Offsets = null!;
115115
public byte[] Data = null!;
116116
}
117+
118+
public IEnumerable<TValue> GetAll<TAttribute, TValue>(ref ModelHeader header)
119+
where TAttribute : IAttribute<TValue>
120+
{
121+
Entry entry;
122+
if (header.InlineCache is Entry inline)
123+
entry = inline;
124+
else if (!cache.TryGetValue(header.Id, out entry!))
125+
entry = Cache(header.Id);
126+
127+
header.InlineCache = entry;
128+
129+
return GetAllInner<TAttribute, TValue>(entry);
130+
}
131+
132+
private IEnumerable<TValue> GetAllInner<TAttribute, TValue>(Entry entry) where TAttribute : IAttribute<TValue>
133+
{
134+
var localCache = InlineCache<TAttribute, TValue>.Instance;
135+
if (localCache.Registry != registry)
136+
{
137+
localCache = Recache<TAttribute, TValue>();
138+
}
139+
140+
var attributeIndex = Array.IndexOf(entry.Attributes, localCache.Id);
141+
if (attributeIndex == -1)
142+
yield break;
143+
144+
while (attributeIndex < entry.Attributes.Length)
145+
{
146+
if (entry.Attributes[attributeIndex] != localCache.Id)
147+
break;
148+
149+
var offset = entry.Offsets[attributeIndex];
150+
var length = entry.Offsets[attributeIndex + 1] - offset;
151+
152+
localCache.Serializer.Read(entry.Data.AsSpan().SliceFast(offset, length), out var value);
153+
yield return value;
154+
attributeIndex++;
155+
}
156+
}
117157
}

‎tests/NexusMods.MneumonicDB.Storage.Tests/ABackendTest.cs

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ public async Task InsertedDatomsShowUpInTheIndex(IndexType type)
3333

3434
var modId1 = NextTempId();
3535
var modId2 = NextTempId();
36+
var loadoutId = NextTempId();
37+
var collectionId = NextTempId();
3638

3739
var tx = await DatomStore.Transact([
3840
FileAttributes.Path.Assert(id1, "/foo/bar"),
@@ -44,16 +46,26 @@ public async Task InsertedDatomsShowUpInTheIndex(IndexType type)
4446
FileAttributes.ModId.Assert(id1, modId1),
4547
FileAttributes.ModId.Assert(id2, modId1),
4648
ModAttributes.Name.Assert(modId1, "Test Mod 1"),
47-
ModAttributes.Name.Assert(modId2, "Test Mod 2")
49+
ModAttributes.LoadoutId.Assert(modId1, loadoutId),
50+
ModAttributes.Name.Assert(modId2, "Test Mod 2"),
51+
ModAttributes.LoadoutId.Assert(modId2, loadoutId),
52+
LoadoutAttributes.Name.Assert(loadoutId, "Test Loadout 1"),
53+
CollectionAttributes.Name.Assert(collectionId, "Test Collection 1"),
54+
CollectionAttributes.LoadoutId.Assert(collectionId, loadoutId),
55+
CollectionAttributes.Mods.Assert(collectionId, modId1),
56+
CollectionAttributes.Mods.Assert(collectionId, modId2)
4857
]);
4958

5059
id1 = tx.Remaps[id1];
5160
id2 = tx.Remaps[id2];
61+
collectionId = tx.Remaps[collectionId];
5262

5363
tx = await DatomStore.Transact([
5464
// Rename file 1 and move file 1 to mod 2
5565
FileAttributes.Path.Assert(id2, "/foo/qux"),
56-
FileAttributes.ModId.Assert(id1, modId2)
66+
FileAttributes.ModId.Assert(id1, modId2),
67+
// Remove mod2 from collection
68+
CollectionAttributes.Mods.Retract(collectionId, modId2),
5769
]);
5870

5971
using var iterator = tx.Snapshot.GetIterator(type);

‎tests/NexusMods.MneumonicDB.Storage.Tests/AStorageTest.cs

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,17 @@ protected AStorageTest(IServiceProvider provider, Func<AttributeRegistry, IStore
3939
new DbAttribute(Symbol.Intern<FileAttributes.ModId>(), AttributeId.From(23),
4040
Symbol.Intern<EntityIdSerializer>()),
4141
new DbAttribute(Symbol.Intern<ModAttributes.Name>(), AttributeId.From(24),
42-
Symbol.Intern<StringSerializer>())
42+
Symbol.Intern<StringSerializer>()),
43+
new DbAttribute(Symbol.Intern<ModAttributes.LoadoutId>(), AttributeId.From(25),
44+
Symbol.Intern<EntityIdSerializer>()),
45+
new DbAttribute(Symbol.Intern<LoadoutAttributes.Name>(), AttributeId.From(26),
46+
Symbol.Intern<StringSerializer>()),
47+
new DbAttribute(Symbol.Intern<CollectionAttributes.Name>(), AttributeId.From(27),
48+
Symbol.Intern<StringSerializer>()),
49+
new DbAttribute(Symbol.Intern<CollectionAttributes.LoadoutId>(), AttributeId.From(28),
50+
Symbol.Intern<EntityIdSerializer>()),
51+
new DbAttribute(Symbol.Intern<CollectionAttributes.Mods>(), AttributeId.From(29),
52+
Symbol.Intern<EntityIdSerializer>())
4353
]);
4454
_path = FileSystem.Shared.GetKnownPath(KnownPath.EntryDirectory).Combine("tests_datomstore" + Guid.NewGuid());
4555

‎tests/NexusMods.MneumonicDB.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=AEVTCurrent.verified.txt

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,13 @@
88
+ | 0200000000000002 | (0015) Hash | 0x00000000DEADBEAF | 0100000000000002
99
+ | 0200000000000001 | (0016) Size | 42 B | 0100000000000002
1010
+ | 0200000000000002 | (0016) Size | 77 B | 0100000000000002
11-
+ | 0200000000000001 | (0017) ModId | 0200000000000005 | 0100000000000003
11+
+ | 0200000000000001 | (0017) ModId | 0200000000000007 | 0100000000000003
1212
+ | 0200000000000002 | (0017) ModId | 0200000000000003 | 0100000000000002
1313
+ | 0200000000000003 | (0018) Name | Test Mod 1 | 0100000000000002
14-
+ | 0200000000000004 | (0018) Name | Test Mod 2 | 0100000000000002
14+
+ | 0200000000000005 | (0018) Name | Test Mod 2 | 0100000000000002
15+
+ | 0200000000000003 | (0019) LoadoutId | 0200000000000004 | 0100000000000002
16+
+ | 0200000000000005 | (0019) LoadoutId | 0200000000000004 | 0100000000000002
17+
+ | 0200000000000004 | (001A) Name | Test Loadout 1 | 0100000000000002
18+
+ | 0200000000000006 | (001B) Name | Test Collection 1 | 0100000000000002
19+
+ | 0200000000000006 | (001C) LoadoutId | 0200000000000004 | 0100000000000002
20+
+ | 0200000000000006 | (001D) Mods | 0200000000000005 | 0100000000000002

‎tests/NexusMods.MneumonicDB.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=AEVTHistory.verified.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,4 @@
11
+ | 0200000000000002 | (0014) Path | /qix/bar | 0100000000000002
22
+ | 0200000000000001 | (0017) ModId | 0200000000000003 | 0100000000000002
3+
+ | 0200000000000006 | (001D) Mods | 0200000000000003 | 0100000000000002
4+
- | 0200000000000006 | (001D) Mods | 0200000000000007 | 0100000000000003

‎tests/NexusMods.MneumonicDB.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=EAVTCurrent.verified.txt

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,16 @@
55
+ | 0200000000000001 | (0014) Path | /foo/bar | 0100000000000002
66
+ | 0200000000000001 | (0015) Hash | 0x00000000DEADBEEF | 0100000000000002
77
+ | 0200000000000001 | (0016) Size | 42 B | 0100000000000002
8-
+ | 0200000000000001 | (0017) ModId | 0200000000000005 | 0100000000000003
8+
+ | 0200000000000001 | (0017) ModId | 0200000000000007 | 0100000000000003
99
+ | 0200000000000002 | (0014) Path | /foo/qux | 0100000000000003
1010
+ | 0200000000000002 | (0015) Hash | 0x00000000DEADBEAF | 0100000000000002
1111
+ | 0200000000000002 | (0016) Size | 77 B | 0100000000000002
1212
+ | 0200000000000002 | (0017) ModId | 0200000000000003 | 0100000000000002
1313
+ | 0200000000000003 | (0018) Name | Test Mod 1 | 0100000000000002
14-
+ | 0200000000000004 | (0018) Name | Test Mod 2 | 0100000000000002
14+
+ | 0200000000000003 | (0019) LoadoutId | 0200000000000004 | 0100000000000002
15+
+ | 0200000000000004 | (001A) Name | Test Loadout 1 | 0100000000000002
16+
+ | 0200000000000005 | (0018) Name | Test Mod 2 | 0100000000000002
17+
+ | 0200000000000005 | (0019) LoadoutId | 0200000000000004 | 0100000000000002
18+
+ | 0200000000000006 | (001B) Name | Test Collection 1 | 0100000000000002
19+
+ | 0200000000000006 | (001C) LoadoutId | 0200000000000004 | 0100000000000002
20+
+ | 0200000000000006 | (001D) Mods | 0200000000000005 | 0100000000000002

‎tests/NexusMods.MneumonicDB.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=EAVTHistory.verified.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,4 @@
11
+ | 0200000000000001 | (0017) ModId | 0200000000000003 | 0100000000000002
22
+ | 0200000000000002 | (0014) Path | /qix/bar | 0100000000000002
3+
+ | 0200000000000006 | (001D) Mods | 0200000000000003 | 0100000000000002
4+
- | 0200000000000006 | (001D) Mods | 0200000000000007 | 0100000000000003

‎tests/NexusMods.MneumonicDB.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=TxLog.verified.txt

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,14 @@
1111
+ | 0200000000000002 | (0016) Size | 77 B | 0100000000000002
1212
+ | 0200000000000002 | (0017) ModId | 0200000000000003 | 0100000000000002
1313
+ | 0200000000000003 | (0018) Name | Test Mod 1 | 0100000000000002
14-
+ | 0200000000000004 | (0018) Name | Test Mod 2 | 0100000000000002
15-
+ | 0200000000000001 | (0017) ModId | 0200000000000005 | 0100000000000003
14+
+ | 0200000000000003 | (0019) LoadoutId | 0200000000000004 | 0100000000000002
15+
+ | 0200000000000004 | (001A) Name | Test Loadout 1 | 0100000000000002
16+
+ | 0200000000000005 | (0018) Name | Test Mod 2 | 0100000000000002
17+
+ | 0200000000000005 | (0019) LoadoutId | 0200000000000004 | 0100000000000002
18+
+ | 0200000000000006 | (001B) Name | Test Collection 1 | 0100000000000002
19+
+ | 0200000000000006 | (001C) LoadoutId | 0200000000000004 | 0100000000000002
20+
+ | 0200000000000006 | (001D) Mods | 0200000000000003 | 0100000000000002
21+
+ | 0200000000000006 | (001D) Mods | 0200000000000005 | 0100000000000002
22+
+ | 0200000000000001 | (0017) ModId | 0200000000000007 | 0100000000000003
1623
+ | 0200000000000002 | (0014) Path | /foo/qux | 0100000000000003
24+
- | 0200000000000006 | (001D) Mods | 0200000000000007 | 0100000000000003

‎tests/NexusMods.MneumonicDB.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=VAETCurrent.verified.txt

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,6 @@
11
+ | 0200000000000002 | (0017) ModId | 0200000000000003 | 0100000000000002
2-
+ | 0200000000000001 | (0017) ModId | 0200000000000005 | 0100000000000003
2+
+ | 0200000000000003 | (0019) LoadoutId | 0200000000000004 | 0100000000000002
3+
+ | 0200000000000005 | (0019) LoadoutId | 0200000000000004 | 0100000000000002
4+
+ | 0200000000000006 | (001C) LoadoutId | 0200000000000004 | 0100000000000002
5+
+ | 0200000000000006 | (001D) Mods | 0200000000000005 | 0100000000000002
6+
+ | 0200000000000001 | (0017) ModId | 0200000000000007 | 0100000000000003

‎tests/NexusMods.MneumonicDB.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=VAETHistory.verified.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,3 @@
11
+ | 0200000000000001 | (0017) ModId | 0200000000000003 | 0100000000000002
2+
+ | 0200000000000006 | (001D) Mods | 0200000000000003 | 0100000000000002
3+
- | 0200000000000006 | (001D) Mods | 0200000000000007 | 0100000000000003

‎tests/NexusMods.MneumonicDB.Storage.Tests/BackendTestVerifyData/RocksDB.InsertedDatomsShowUpInTheIndex_type=AEVTCurrent.verified.txt

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,13 @@
88
+ | 0200000000000002 | (0015) Hash | 0x00000000DEADBEAF | 0100000000000002
99
+ | 0200000000000001 | (0016) Size | 42 B | 0100000000000002
1010
+ | 0200000000000002 | (0016) Size | 77 B | 0100000000000002
11-
+ | 0200000000000001 | (0017) ModId | 0200000000000005 | 0100000000000003
11+
+ | 0200000000000001 | (0017) ModId | 0200000000000007 | 0100000000000003
1212
+ | 0200000000000002 | (0017) ModId | 0200000000000003 | 0100000000000002
1313
+ | 0200000000000003 | (0018) Name | Test Mod 1 | 0100000000000002
14-
+ | 0200000000000004 | (0018) Name | Test Mod 2 | 0100000000000002
14+
+ | 0200000000000005 | (0018) Name | Test Mod 2 | 0100000000000002
15+
+ | 0200000000000003 | (0019) LoadoutId | 0200000000000004 | 0100000000000002
16+
+ | 0200000000000005 | (0019) LoadoutId | 0200000000000004 | 0100000000000002
17+
+ | 0200000000000004 | (001A) Name | Test Loadout 1 | 0100000000000002
18+
+ | 0200000000000006 | (001B) Name | Test Collection 1 | 0100000000000002
19+
+ | 0200000000000006 | (001C) LoadoutId | 0200000000000004 | 0100000000000002
20+
+ | 0200000000000006 | (001D) Mods | 0200000000000005 | 0100000000000002

‎tests/NexusMods.MneumonicDB.Storage.Tests/BackendTestVerifyData/RocksDB.InsertedDatomsShowUpInTheIndex_type=AEVTHistory.verified.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,4 @@
11
+ | 0200000000000002 | (0014) Path | /qix/bar | 0100000000000002
22
+ | 0200000000000001 | (0017) ModId | 0200000000000003 | 0100000000000002
3+
+ | 0200000000000006 | (001D) Mods | 0200000000000003 | 0100000000000002
4+
- | 0200000000000006 | (001D) Mods | 0200000000000007 | 0100000000000003

0 commit comments

Comments
 (0)