Merge branch 'meta_rework'

Author: Ottermandias

OtterGui

@@ -1,23 +1,43 @@
+using Newtonsoft.Json.Linq;
 using OtterGui;
 using OtterGui.Services;
+using Penumbra.Collections;
+using Penumbra.GameData.Structs;
 using Penumbra.Interop.PathResolving;
 using Penumbra.Meta.Manipulations;
 
 namespace Penumbra.Api.Api;
 
 public class MetaApi(CollectionResolver collectionResolver, ApiHelpers helpers) : IPenumbraApiMeta, IApiService
 {
+    public const int CurrentVersion = 0;
+
     public string GetPlayerMetaManipulations()
     {
         var collection = collectionResolver.PlayerCollection();
-        var set        = collection.MetaCache?.Manipulations.ToArray() ?? [];
-        return Functions.ToCompressedBase64(set, MetaManipulation.CurrentVersion);
+        return CompressMetaManipulations(collection);
     }
 
     public string GetMetaManipulations(int gameObjectIdx)
     {
         helpers.AssociatedCollection(gameObjectIdx, out var collection);
-        var set = collection.MetaCache?.Manipulations.ToArray() ?? [];
-        return Functions.ToCompressedBase64(set, MetaManipulation.CurrentVersion);
+        return CompressMetaManipulations(collection);
+    }
+
+    internal static string CompressMetaManipulations(ModCollection collection)
+    {
+        var array = new JArray();
+        if (collection.MetaCache is { } cache)
+        {
+            MetaDictionary.SerializeTo(array, cache.GlobalEqp.Select(kvp => kvp.Key));
+            MetaDictionary.SerializeTo(array, cache.Imc.Select(kvp => new KeyValuePair<ImcIdentifier, ImcEntry>(kvp.Key, kvp.Value.Entry)));
+            MetaDictionary.SerializeTo(array, cache.Eqp.Select(kvp => new KeyValuePair<EqpIdentifier, EqpEntry>(kvp.Key, kvp.Value.Entry)));
+            MetaDictionary.SerializeTo(array, cache.Eqdp.Select(kvp => new KeyValuePair<EqdpIdentifier, EqdpEntry>(kvp.Key, kvp.Value.Entry)));
+            MetaDictionary.SerializeTo(array, cache.Est.Select(kvp => new KeyValuePair<EstIdentifier, EstEntry>(kvp.Key, kvp.Value.Entry)));
+            MetaDictionary.SerializeTo(array, cache.Rsp.Select(kvp => new KeyValuePair<RspIdentifier, RspEntry>(kvp.Key, kvp.Value.Entry)));
+            MetaDictionary.SerializeTo(array, cache.Gmp.Select(kvp => new KeyValuePair<GmpIdentifier, GmpEntry>(kvp.Key, kvp.Value.Entry)));
+        }
+
+        return Functions.ToCompressedBase64(array, CurrentVersion);
     }
 }

Penumbra.GameData

@@ -159,32 +159,18 @@ private static bool ConvertPaths(IReadOnlyDictionary<string, string> redirection
     /// The empty string is treated as an empty set.
     /// Only returns true if all conversions are successful and distinct. 
     /// </summary>
-    private static bool ConvertManips(string manipString,
-        [NotNullWhen(true)] out HashSet<MetaManipulation>? manips)
+    private static bool ConvertManips(string manipString, [NotNullWhen(true)] out MetaDictionary? manips)
     {
         if (manipString.Length == 0)
         {
-            manips = [];
+            manips = new MetaDictionary();
             return true;
         }
 
-        if (Functions.FromCompressedBase64<MetaManipulation[]>(manipString, out var manipArray) != MetaManipulation.CurrentVersion)
-        {
-            manips = null;
-            return false;
-        }
-
-        manips = new HashSet<MetaManipulation>(manipArray!.Length);
-        foreach (var manip in manipArray.Where(m => m.Validate()))
-        {
-            if (manips.Add(manip))
-                continue;
-
-            Penumbra.Log.Warning($"Manipulation {manip} {manip.EntryToString()} is invalid and was skipped.");
-            manips = null;
-            return false;
-        }
+        if (Functions.FromCompressedBase64(manipString, out manips!) == MetaApi.CurrentVersion)
+            return true;
 
-        return true;
+        manips = null;
+        return false;
     }
 }

Penumbra/Api/Api/MetaApi.cs

@@ -4,6 +4,8 @@
 using OtterGui;
 using OtterGui.Raii;
 using OtterGui.Services;
+using OtterGui.Text;
+using Penumbra.Api.Api;
 using Penumbra.Api.Enums;
 using Penumbra.Api.IpcSubscribers;
 using Penumbra.Collections.Manager;
@@ -49,7 +51,7 @@ public void Draw()
         ImGui.InputTextWithHint("##tempMod",   "Temporary Mod Name...",         ref _tempModName,      32);
         ImGui.InputTextWithHint("##tempGame",  "Game Path...",                  ref _tempGamePath,     256);
         ImGui.InputTextWithHint("##tempFile",  "File Path...",                  ref _tempFilePath,     256);
-        ImGui.InputTextWithHint("##tempManip", "Manipulation Base64 String...", ref _tempManipulation, 256);
+        ImUtf8.InputText("##tempManip"u8, ref _tempManipulation, "Manipulation Base64 String..."u8);
         ImGui.Checkbox("Force Character Collection Overwrite", ref _forceOverwrite);
 
         using var table = ImRaii.Table(string.Empty, 3, ImGuiTableFlags.SizingFixedFit);
@@ -101,8 +103,7 @@ public void Draw()
          && copyCollection is { HasCache: true })
         {
             var files = copyCollection.ResolvedFiles.ToDictionary(kvp => kvp.Key.ToString(), kvp => kvp.Value.Path.ToString());
-            var manips = Functions.ToCompressedBase64(copyCollection.MetaCache?.Manipulations.ToArray() ?? Array.Empty<MetaManipulation>(),
-                MetaManipulation.CurrentVersion);
+            var manips = MetaApi.CompressMetaManipulations(copyCollection);
             _lastTempError = new AddTemporaryMod(pi).Invoke(_tempModName, guid, files, manips, 999);
         }
 
@@ -187,8 +188,8 @@ void PrintList(string collectionName, IReadOnlyList<TemporaryMod> list)
                 if (ImGui.IsItemHovered())
                 {
                     using var tt = ImRaii.Tooltip();
-                    foreach (var manip in mod.Default.Manipulations)
-                        ImGui.TextUnformatted(manip.ToString());
+                    foreach (var identifier in mod.Default.Manipulations.Identifiers)
+                        ImGui.TextUnformatted(identifier.ToString());
                 }
             }
         }

Penumbra/Api/Api/TemporaryApi.cs

@@ -1,3 +1,4 @@
+using OtterGui.Services;
 using Penumbra.Api.Enums;
 using Penumbra.Collections;
 using Penumbra.Meta.Manipulations;
@@ -18,7 +19,7 @@ public enum RedirectResult
     FilteredGamePath        = 3,
 }
 
-public class TempModManager : IDisposable
+public class TempModManager : IDisposable, IService
 {
     private readonly CommunicatorService _communicator;
 
@@ -43,7 +44,7 @@ public IReadOnlyList<TemporaryMod> ModsForAllCollections
         => _modsForAllCollections;
 
     public RedirectResult Register(string tag, ModCollection? collection, Dictionary<Utf8GamePath, FullPath> dict,
-        HashSet<MetaManipulation> manips, ModPriority priority)
+        MetaDictionary manips, ModPriority priority)
     {
         var mod = GetOrCreateMod(tag, collection, priority, out var created);
         Penumbra.Log.Verbose($"{(created ? "Created" : "Changed")} temporary Mod {mod.Name}.");

Penumbra/Api/IpcTester/TemporaryIpcTester.cs

@@ -125,12 +125,6 @@ public HashSet<Utf8GamePath>[] ReverseResolvePaths(IReadOnlyCollection<string> f
         return ret;
     }
 
-    public void ForceFile(Utf8GamePath path, FullPath fullPath)
-        => _manager.AddChange(ChangeData.ForcedFile(this, path, fullPath));
-
-    public void RemovePath(Utf8GamePath path)
-        => _manager.AddChange(ChangeData.ForcedFile(this, path, FullPath.Empty));
-
     public void ReloadMod(IMod mod, bool addMetaChanges)
         => _manager.AddChange(ChangeData.ModReload(this, mod, addMetaChanges));
 
@@ -233,15 +227,24 @@ internal void AddModSync(IMod mod, bool addMetaChanges)
         foreach (var (path, file) in files.FileRedirections)
             AddFile(path, file, mod);
 
-        foreach (var manip in files.Manipulations)
-            AddManipulation(manip, mod);
+        foreach (var (identifier, entry) in files.Manipulations.Eqp)
+            AddManipulation(mod, identifier, entry);
+        foreach (var (identifier, entry) in files.Manipulations.Eqdp)
+            AddManipulation(mod, identifier, entry);
+        foreach (var (identifier, entry) in files.Manipulations.Est)
+            AddManipulation(mod, identifier, entry);
+        foreach (var (identifier, entry) in files.Manipulations.Gmp)
+            AddManipulation(mod, identifier, entry);
+        foreach (var (identifier, entry) in files.Manipulations.Rsp)
+            AddManipulation(mod, identifier, entry);
+        foreach (var (identifier, entry) in files.Manipulations.Imc)
+            AddManipulation(mod, identifier, entry);
+        foreach (var identifier in files.Manipulations.GlobalEqp)
+            AddManipulation(mod, identifier, null!);
 
         if (addMetaChanges)
         {
             _collection.IncrementCounter();
-            if (mod.TotalManipulations > 0)
-                AddMetaFiles(false);
-
             _manager.MetaFileManager.ApplyDefaultFiles(_collection);
         }
     }
@@ -342,7 +345,7 @@ private bool AddConflict(object data, IMod addedMod, IMod existingMod)
             foreach (var conflict in tmpConflicts)
             {
                 if (data is Utf8GamePath path && conflict.Conflicts.RemoveAll(p => p is Utf8GamePath x && x.Equals(path)) > 0
-                 || data is MetaManipulation meta && conflict.Conflicts.RemoveAll(m => m is MetaManipulation x && x.Equals(meta)) > 0)
+                 || data is IMetaIdentifier meta && conflict.Conflicts.RemoveAll(m => m.Equals(meta)) > 0)
                     AddConflict(data, addedMod, conflict.Mod2);
             }
 
@@ -374,33 +377,28 @@ private bool AddConflict(object data, IMod addedMod, IMod existingMod)
     // For different mods, higher mod priority takes precedence before option group priority,
     // which takes precedence before option priority, which takes precedence before ordering.
     // Inside the same mod, conflicts are not recorded.
-    private void AddManipulation(MetaManipulation manip, IMod mod)
+    private void AddManipulation(IMod mod, IMetaIdentifier identifier, object entry)
     {
-        if (!Meta.TryGetValue(manip, out var existingMod))
+        if (!Meta.TryGetMod(identifier, out var existingMod))
         {
-            Meta.ApplyMod(manip, mod);
-            ModData.AddManip(mod, manip);
+            Meta.ApplyMod(mod, identifier, entry);
+            ModData.AddManip(mod, identifier);
             return;
         }
 
         // Lower prioritized option in the same mod.
         if (mod == existingMod)
             return;
 
-        if (AddConflict(manip, mod, existingMod))
+        if (AddConflict(identifier, mod, existingMod))
         {
-            ModData.RemoveManip(existingMod, manip);
-            Meta.ApplyMod(manip, mod);
-            ModData.AddManip(mod, manip);
+            ModData.RemoveManip(existingMod, identifier);
+            Meta.ApplyMod(mod, identifier, entry);
+            ModData.AddManip(mod, identifier);
         }
     }
 
 
-    // Add all necessary meta file redirects.
-    public void AddMetaFiles(bool fromFullCompute)
-        => Meta.SetImcFiles(fromFullCompute);
-
-
     // Identify and record all manipulated objects for this entire collection.
     private void SetChangedItems()
     {
@@ -437,9 +435,9 @@ void AddItems(IMod mod)
                 AddItems(modPath.Mod);
             }
 
-            foreach (var (manip, mod) in Meta)
+            foreach (var (manip, mod) in Meta.IdentifierSources)
             {
-                identifier.MetaChangedItems(items, manip);
+                manip.AddChangedItems(identifier, items);
                 AddItems(mod);
             }
 

Penumbra/Api/TempModManager.cs

@@ -1,5 +1,6 @@
 using Dalamud.Plugin.Services;
 using OtterGui.Classes;
+using OtterGui.Services;
 using Penumbra.Api;
 using Penumbra.Api.Enums;
 using Penumbra.Collections.Manager;
@@ -17,7 +18,7 @@
 
 namespace Penumbra.Collections.Cache;
 
-public class CollectionCacheManager : IDisposable
+public class CollectionCacheManager : IDisposable, IService
 {
     private readonly  FrameworkManager    _framework;
     private readonly  CommunicatorService _communicator;
@@ -180,8 +181,6 @@ private void FullRecalculation(ModCollection collection)
             foreach (var mod in _modStorage)
                 cache.AddModSync(mod, false);
 
-            cache.AddMetaFiles(true);
-
             collection.IncrementCounter();
 
             MetaFileManager.ApplyDefaultFiles(collection);

Penumbra/Collections/Cache/CmpCache.cs

@@ -9,12 +9,12 @@ namespace Penumbra.Collections.Cache;
 /// </summary>
 public class CollectionModData
 {
-    private readonly Dictionary<IMod, (HashSet<Utf8GamePath>, HashSet<MetaManipulation>)> _data = new();
+    private readonly Dictionary<IMod, (HashSet<Utf8GamePath>, HashSet<IMetaIdentifier>)> _data = new();
 
-    public IEnumerable<(IMod, IReadOnlySet<Utf8GamePath>, IReadOnlySet<MetaManipulation>)> Data
-        => _data.Select(kvp => (kvp.Key, (IReadOnlySet<Utf8GamePath>)kvp.Value.Item1, (IReadOnlySet<MetaManipulation>)kvp.Value.Item2));
+    public IEnumerable<(IMod, IReadOnlySet<Utf8GamePath>, IReadOnlySet<IMetaIdentifier>)> Data
+        => _data.Select(kvp => (kvp.Key, (IReadOnlySet<Utf8GamePath>)kvp.Value.Item1, (IReadOnlySet<IMetaIdentifier>)kvp.Value.Item2));
 
-    public (IReadOnlyCollection<Utf8GamePath> Paths, IReadOnlyCollection<MetaManipulation> Manipulations) RemoveMod(IMod mod)
+    public (IReadOnlyCollection<Utf8GamePath> Paths, IReadOnlyCollection<IMetaIdentifier> Manipulations) RemoveMod(IMod mod)
     {
         if (_data.Remove(mod, out var data))
             return data;
@@ -35,7 +35,7 @@ public void AddPath(IMod mod, Utf8GamePath path)
         }
     }
 
-    public void AddManip(IMod mod, MetaManipulation manipulation)
+    public void AddManip(IMod mod, IMetaIdentifier manipulation)
     {
         if (_data.TryGetValue(mod, out var data))
         {
@@ -54,7 +54,7 @@ public void RemovePath(IMod mod, Utf8GamePath path)
             _data.Remove(mod);
     }
 
-    public void RemoveManip(IMod mod, MetaManipulation manip)
+    public void RemoveManip(IMod mod, IMetaIdentifier manip)
     {
         if (_data.TryGetValue(mod, out var data) && data.Item2.Remove(manip) && data.Item1.Count == 0 && data.Item2.Count == 0)
             _data.Remove(mod);