[SchemaUpdater] WIP

EXDCommon.Tests/EXDCommon.Tests.csproj

@@ -0,0 +1,34 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net7.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+
+        <IsPackable>false</IsPackable>
+        <IsTestProject>true</IsTestProject>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.0"/>
+        <PackageReference Include="NUnit" Version="3.13.3"/>
+        <PackageReference Include="NUnit3TestAdapter" Version="4.4.2"/>
+        <PackageReference Include="NUnit.Analyzers" Version="3.6.1"/>
+        <PackageReference Include="coverlet.collector" Version="3.2.0"/>
+    </ItemGroup>
+
+    <ItemGroup>
+      <Folder Include="Data\" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\EXDCommon\EXDCommon.csproj" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <Content Include="Data\Schemas\*.yml">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      </Content>
+    </ItemGroup>
+
+</Project>

EXDCommon.Tests/GlobalUsings.cs

@@ -0,0 +1 @@
+global using NUnit.Framework;

EXDCommon.Tests/SerializeUtilTests.cs

@@ -0,0 +1,66 @@
+using EXDCommon.SchemaModel.EXDSchema;
+using EXDCommon.Utility;
+
+namespace EXDCommon.Tests;
+
+public class Tests
+{
+	[Test]
+	public void TestDeserializeSheet()
+	{
+		var content = File.ReadAllText("Data/Schemas/Aetheryte.yml");
+		var sheet = SerializeUtil.Deserialize<Sheet>(content);
+        Assert.That(sheet, Is.Not.Null);
+	}
+
+	// [Test]
+	// public void TestReserializeObjectBased()
+	// {
+	// 	var content = File.ReadAllText("Data/Schemas/Aetheryte.yml");
+	// 	var sheet = SerializeUtil.Deserialize<Sheet>(content);
+	// 	var content2 = SerializeUtil.Serialize(sheet);
+	// 	var sheet2 = SerializeUtil.Deserialize<Sheet>(content2);
+	// 	Assert.That(sheet, Is.EqualTo(sheet2));
+	// }
+
+	[Test]
+	public void TestReserializeTextBased()
+	{
+		var content = File.ReadAllText("Data/Schemas/Aetheryte.yml");
+		var sheet = SerializeUtil.Deserialize<Sheet>(content);
+		var content2 = SerializeUtil.Serialize(sheet);
+		Assert.That(content2, Is.EqualTo(content));
+	}
+
+	[Test]
+	public void TestFlatten()
+	{
+		var content = File.ReadAllText("Data/Schemas/Aetheryte.yml");
+		var sheet = SerializeUtil.Deserialize<Sheet>(content);
+		var flattenedSheet = SchemaUtil.Flatten(sheet);
+		var count = SchemaUtil.GetColumnCount(sheet);
+		Assert.That(flattenedSheet.Fields, Has.Count.EqualTo(count));
+	}
+
+	// [Test]
+	// public void TestUnflattenObjectBased()
+	// {
+	// 	var content = File.ReadAllText("Data/Schemas/AnimationLOD.yml");
+	// 	var sheet = SerializeUtil.Deserialize<Sheet>(content);
+	// 	var flattenedSheet = SchemaUtil.Flatten(sheet);
+	// 	var unflattenedSheet = SchemaUtil.Unflatten(flattenedSheet);
+	// 	Assert.That(sheet, Is.EqualTo(unflattenedSheet));
+	// }
+
+	// [TestCase("AnimationLOD")]
+	[TestCase("Aetheryte")]
+	public void TestUnflattenTextBased(string sheetName)
+	{
+		var content = File.ReadAllText($"Data/Schemas/{sheetName}.yml");
+		var sheet = SerializeUtil.Deserialize<Sheet>(content);
+		var flattenedSheet = SchemaUtil.Flatten(sheet);
+		var unflattenedSheet = SchemaUtil.Unflatten(flattenedSheet);
+		var content2 = SerializeUtil.Serialize(unflattenedSheet);
+		Assert.That(content2, Is.EqualTo(content));
+	}
+}

EXDCommon/SchemaModel/NewSheetDefinition.cs

@@ -80,7 +80,7 @@ public class Field
 	/// </summary>
 	[YamlIgnore]
 	public string PathWithArrayIndices;
-	
+
 	public override string ToString()
 	{
 		return $"{Name} ({Type})";

EXDCommon/Utility/SchemaUtil.cs

@@ -18,14 +18,16 @@ public static int GetColumnCount(Sheet sheet)
 
 	public static Sheet Flatten(Sheet sheet)
 	{
+		var newSheet = new Sheet { Name = sheet.Name, DisplayField = sheet.DisplayField };
 		var fields = new List<Field>();
+
 		foreach (var field in sheet.Fields)
 			Emit(fields, field);
 
-		sheet.Fields = fields;
-		for (int i = 0; i < sheet.Fields.Count; i++)
-			sheet.Fields[i].OffsetBasedIndex = (uint)i;
-		return sheet;
+		newSheet.Fields = fields;
+		for (int i = 0; i < newSheet.Fields.Count; i++)
+			newSheet.Fields[i].OffsetBasedIndex = (uint)i;
+		return newSheet;
 	}
 
 	/// <summary>
@@ -59,12 +61,12 @@ public static List<DefinedColumn> Flatten(ExcelHeaderFile exh, Sheet sheet, bool
 		return definedColumns;
 	}
 
-	private static void Emit(List<Field> list, Field field, List<(string, string)> hierarchy = null, string nameOverride = "")
+	private static void Emit(List<Field> list, Field field, List<string> hierarchy = null, List<string> arrayHierarchy = null, string nameOverride = "")
 	{
 		if (field.Type != FieldType.Array)
 		{
 			// Single field
-			list.Add(CreateField(field, false, 0, hierarchy, nameOverride));
+			list.Add(CreateField(field, false, 0, hierarchy, arrayHierarchy, nameOverride));
 		}
 		else if (field.Type == FieldType.Array)
 		{
@@ -73,7 +75,7 @@ private static void Emit(List<Field> list, Field field, List<(string, string)> h
 			{
 				for (int i = 0; i < field.Count.Value; i++)
 				{
-					list.Add(CreateField(field, true, i, hierarchy, ""));
+					list.Add(CreateField(field, true, i, hierarchy, arrayHierarchy, ""));
 				}
 			}
 			else
@@ -82,18 +84,20 @@ private static void Emit(List<Field> list, Field field, List<(string, string)> h
 				{
 					foreach (var nestedField in field.Fields)
 					{
-						var usableHierarchy = hierarchy == null ? new List<(string, string)>() : new List<(string, string)>(hierarchy);
+						var usableHierarchy = hierarchy == null ? new List<string>() : new List<string>(hierarchy);
+						var usableArrayHierarchy = hierarchy == null ? new List<string>() : new List<string>(hierarchy);
 						var hierarchyName = $"{field.Name}";
-						var hierarchyName2 = $"{field.Name}[{i}]";
-						usableHierarchy.Add((hierarchyName, hierarchyName2));
-						Emit(list, nestedField, usableHierarchy, field.Name);
+						var arrayHierarchyName = $"{field.Name}[{i}]";
+						usableHierarchy.Add(hierarchyName);
+						usableArrayHierarchy.Add(arrayHierarchyName);
+						Emit(list, nestedField, usableHierarchy, usableArrayHierarchy, field.Name);
 					}	
 				}
 			}
 		}
 	}
 
-	private static Field CreateField(Field baseField, bool fieldIsArrayElement, int index, List<(string, string)>? hierarchy, string nameOverride)
+	private static Field CreateField(Field baseField, bool fieldIsArrayElement, int index, List<string>? hierarchy, List<string>? arrayHierarchy, string nameOverride)
 	{
 		var addedField = new Field
 		{
@@ -114,10 +118,10 @@ private static Field CreateField(Field baseField, bool fieldIsArrayElement, int
 			path2 += $"[{index}]";
 		}
 
-		if (hierarchy != null)
+		if (hierarchy != null && arrayHierarchy != null)
 		{
 			addedField.Path = string.Join(".", hierarchy);
-			addedField.PathWithArrayIndices = string.Join(".", hierarchy);
+			addedField.PathWithArrayIndices = string.Join(".", arrayHierarchy);
 			if (!string.IsNullOrEmpty(path)) addedField.Path += $".{path}";
 			if (!string.IsNullOrEmpty(path)) addedField.PathWithArrayIndices += $".{path2}";
 		}
@@ -153,4 +157,67 @@ public static int GetFieldCount(Field field)
 		}
 		return 1;
 	}
+
+	public static Sheet Unflatten(Sheet sheet)
+	{
+		var newSheet = new Sheet { Name = sheet.Name, DisplayField = sheet.DisplayField };
+		var fields = new List<Field>(sheet.Fields);
+
+		var result = false;
+		do
+		{
+			result = Unflatten(fields);
+		} while (!result);
+
+		newSheet.Fields = fields;
+		return newSheet;
+	}
+
+	private static bool Unflatten(List<Field> fields)
+	{
+		if (!fields.Any()) return false;
+		// if (!Unflatten(fields)) return false;
+
+		int start = 0, end = 0;
+		var found = false;
+
+		for (int i = 0; i < fields.Count - 1; i++)
+		{
+			var currentFieldPath = "";
+			var nextFieldPath = "";
+
+			int j = 0;
+			do
+			{
+				currentFieldPath = fields[i + j].Path;
+				nextFieldPath = fields[i + j + 1].Path;
+				j++;
+			} while (currentFieldPath == nextFieldPath && i + j < fields.Count - 1);
+
+			start = i;
+			end = i + j - 1;
+
+			// Array (defined by field path) is longer than 1 element
+			if (j > 1)
+			{
+				found = true;
+				break;
+			}
+		}
+
+		if (found)
+		{
+			Console.WriteLine($"Found array of {fields[start].Name} from {start} to {end} with length {end - start + 1}");
+			var fieldStart = fields[start];
+			fieldStart = fields[start];
+			fieldStart.Type = FieldType.Array;
+			fieldStart.Count = end - start + 1;
+			// fieldStart.Comment = 
+			fields.RemoveRange(start + 1, end - start);
+
+			return false;
+		}
+
+		return true;
+	}
 }

EXDCommon/Utility/Util.cs

@@ -59,4 +59,20 @@ public static int GetBitOffset(int offset, ExcelColumnDataType dataType)
 			_ => bitOffset,
 		};
 	}
+
+	public static bool DictionaryEqual<T, TU>(Dictionary<T, List<TU>> oldDict, Dictionary<T, List<TU>> newDict) where T : notnull
+	{
+		// Simple check, are the counts the same?
+		if (!oldDict.Count.Equals(newDict.Count)) return false;
+
+		// Verify the keys
+		if (!oldDict.Keys.SequenceEqual(newDict.Keys)) return false;
+
+		// Verify the values for each key
+		foreach (var key in oldDict.Keys)
+			if (!oldDict[key].SequenceEqual(newDict[key]))
+				return false;
+
+		return true;
+	}
 }

EXDTools.sln

@@ -16,6 +16,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ZiPatchLib", "lib\ZiTool\Zi
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SchemaUpdater", "SchemaUpdater\SchemaUpdater.csproj", "{31C1028A-CB73-428B-B2E6-3DE791564840}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EXDCommon.Tests", "EXDCommon.Tests\EXDCommon.Tests.csproj", "{E8AE5811-ECF6-401E-B01F-459F93A89121}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -50,6 +52,10 @@ Global
 		{31C1028A-CB73-428B-B2E6-3DE791564840}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{31C1028A-CB73-428B-B2E6-3DE791564840}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{31C1028A-CB73-428B-B2E6-3DE791564840}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E8AE5811-ECF6-401E-B01F-459F93A89121}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E8AE5811-ECF6-401E-B01F-459F93A89121}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E8AE5811-ECF6-401E-B01F-459F93A89121}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E8AE5811-ECF6-401E-B01F-459F93A89121}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{E974175E-D301-4F3F-A8BD-7AC42BD3D9FB} = {5C037CD8-C1A6-407C-AE18-5F09AE478977}

EXDTools.sln.DotSettings.user

@@ -1,4 +1,11 @@
 <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=31C1028A_002DCB73_002D428B_002DB2E6_002D3DE791564840_002Ff_003ASchemaUpdater_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/Environment/AssemblyExplorer/XmlDocument/@EntryValue">&lt;AssemblyExplorer&gt;&#xD;
   &lt;Assembly Path="C:\Users\Liam\.nuget\packages\lumina\3.15.2\lib\net7.0\Lumina.dll" /&gt;&#xD;
-&lt;/AssemblyExplorer&gt;</s:String></wpf:ResourceDictionary>
+&lt;/AssemblyExplorer&gt;</s:String>
+	<s:String x:Key="/Default/Environment/Highlighting/HighlightingSourceSnapshotLocation/@EntryValue">C:\Users\Liam\AppData\Local\JetBrains\Rider2023.3\resharper-host\temp\Rider\vAny\CoverageData\_EXDTools.641274775\Snapshot\snapshot.utdcvr</s:String>
+	<s:String x:Key="/Default/Environment/UnitTesting/UnitTestSessionStore/Sessions/=5928ebb5_002D1807_002D4275_002Daf73_002D50cf7f1d26e9/@EntryIndexedValue">&lt;SessionState ContinuousTestingMode="0" IsActive="True" Name="All tests from &amp;lt;EXDCommon.Tests&amp;gt; #3" xmlns="urn:schemas-jetbrains-com:jetbrains-ut-session"&gt;&#xD;
+  &lt;Project Location="C:\Users\Liam\Documents\repos\EXDTools\EXDCommon.Tests" Presentation="&amp;lt;EXDCommon.Tests&amp;gt;" /&gt;&#xD;
+&lt;/SessionState&gt;</s:String>
+
+	</wpf:ResourceDictionary>

SchemaUpdater/SchemaUpdater.cs

@@ -1,4 +1,5 @@
-using System.Numerics;
+using System.Diagnostics;
+using System.Numerics;
 using EXDCommon.FileAccess;
 using EXDCommon.SchemaModel.EXDSchema;
 using EXDCommon.Sheets;
@@ -307,19 +308,43 @@ private uint FindNewIndex(ExcelSheetImpl oldSheet, uint oldColumnIdx, ExcelSheet
 
         return "";
     }
+
+    public static void TestUnflatten()
+    {
+        // Changed stuff test cases
+        // Aetheryte: added column at 20 (between columns)
+        // GimmickJump: unrecognized column changed in the middle of the columns
+        // WorldDCGroupType: added column at 2 (end)
+
+        // Unflatten test cases
+        // No arrays: ActionCastTimeline
+        // var loadedSheet = SerializeUtil.Deserialize<Sheet>(File.ReadAllText(@"C:\Users\Liam\Documents\repos\EXDSchema\Schemas\2024.01.06.0000.0000\ActionCastTimeline.yml"));
+        // Simple array with no links: AnimationLOD
+        // var loadedSheet = SerializeUtil.Deserialize<Sheet>(File.ReadAllText(@"C:\Users\Liam\Documents\repos\EXDSchema\Schemas\2024.01.06.0000.0000\AnimationLOD.yml"));
+
+        var loadedSheet = SerializeUtil.Deserialize<Sheet>(File.ReadAllText(@"C:\Users\Liam\Documents\repos\EXDSchema\Schemas\2024.01.06.0000.0000\ActionCastTimeline.yml"));
+        var flattened = SchemaUtil.Flatten(loadedSheet);
+        // Debugger.Break();
+
+        loadedSheet = SerializeUtil.Deserialize<Sheet>(File.ReadAllText(@"C:\Users\Liam\Documents\repos\EXDSchema\Schemas\2024.01.06.0000.0000\AnimationLOD.yml"));
+        flattened = SchemaUtil.Flatten(loadedSheet);
+        var unflatten = SchemaUtil.Unflatten(flattened);
+        Debugger.Break();
+
+    }
 
     private Sheet GenerateNewSchema(Sheet oldSchema, Sheet newSchema, Dictionary<int, int> columnMap)
     {
         var flattenedOldSchema = SchemaUtil.Flatten(oldSchema);
-
+        return null;
     }
 
-    // private Dictionary<int, int> LoadColumnMap(string sheet, string mapFile)
-    // {
-    //     var json = File.ReadAllText(mapFile);
-    //     var data = JsonConvert.DeserializeObject<Dictionary<string, Dictionary<int, int>>>(json);
-    //     return data[sheet];
-    // }
+    private Dictionary<int, int> LoadColumnMap(string sheet, string mapFile)
+    {
+        var json = File.ReadAllText(mapFile);
+        var data = JsonConvert.DeserializeObject<Dictionary<string, Dictionary<int, int>>>(json);
+        return data[sheet];
+    }
 
     private Dictionary<uint, uint> GenerateColumnMap(RawExcelSheet oldSheet, RawExcelSheet newSheet) 
     {