diff --git a/EXDCommon.Tests/EXDCommon.Tests.csproj b/EXDCommon.Tests/EXDCommon.Tests.csproj new file mode 100644 index 0000000..fbd61fe --- /dev/null +++ b/EXDCommon.Tests/EXDCommon.Tests.csproj @@ -0,0 +1,34 @@ + + + + net7.0 + enable + enable + + false + true + + + + + + + + + + + + + + + + + + + + + PreserveNewest + + + + diff --git a/EXDCommon.Tests/GlobalUsings.cs b/EXDCommon.Tests/GlobalUsings.cs new file mode 100644 index 0000000..cefced4 --- /dev/null +++ b/EXDCommon.Tests/GlobalUsings.cs @@ -0,0 +1 @@ +global using NUnit.Framework; \ No newline at end of file diff --git a/EXDCommon.Tests/SerializeUtilTests.cs b/EXDCommon.Tests/SerializeUtilTests.cs new file mode 100644 index 0000000..e460024 --- /dev/null +++ b/EXDCommon.Tests/SerializeUtilTests.cs @@ -0,0 +1,66 @@ +using EXDCommon.SchemaModel.EXDSchema; +using EXDCommon.Utility; + +namespace EXDCommon.Tests; + +public class Tests +{ + [Test] + public void TestDeserializeSheet() + { + var content = File.ReadAllText("Data/Schemas/Aetheryte.yml"); + var sheet = SerializeUtil.Deserialize(content); + Assert.That(sheet, Is.Not.Null); + } + + // [Test] + // public void TestReserializeObjectBased() + // { + // var content = File.ReadAllText("Data/Schemas/Aetheryte.yml"); + // var sheet = SerializeUtil.Deserialize(content); + // var content2 = SerializeUtil.Serialize(sheet); + // var sheet2 = SerializeUtil.Deserialize(content2); + // Assert.That(sheet, Is.EqualTo(sheet2)); + // } + + [Test] + public void TestReserializeTextBased() + { + var content = File.ReadAllText("Data/Schemas/Aetheryte.yml"); + var sheet = SerializeUtil.Deserialize(content); + var content2 = SerializeUtil.Serialize(sheet); + Assert.That(content2, Is.EqualTo(content)); + } + + [Test] + public void TestFlatten() + { + var content = File.ReadAllText("Data/Schemas/Aetheryte.yml"); + var sheet = SerializeUtil.Deserialize(content); + var flattenedSheet = SchemaUtil.Flatten(sheet); + var count = SchemaUtil.GetColumnCount(sheet); + Assert.That(flattenedSheet.Fields, Has.Count.EqualTo(count)); + } + + // [Test] + // public void TestUnflattenObjectBased() + // { + // var content = File.ReadAllText("Data/Schemas/AnimationLOD.yml"); + // var sheet = SerializeUtil.Deserialize(content); + // var flattenedSheet = SchemaUtil.Flatten(sheet); + // var unflattenedSheet = SchemaUtil.Unflatten(flattenedSheet); + // Assert.That(sheet, Is.EqualTo(unflattenedSheet)); + // } + + // [TestCase("AnimationLOD")] + [TestCase("Aetheryte")] + public void TestUnflattenTextBased(string sheetName) + { + var content = File.ReadAllText($"Data/Schemas/{sheetName}.yml"); + var sheet = SerializeUtil.Deserialize(content); + var flattenedSheet = SchemaUtil.Flatten(sheet); + var unflattenedSheet = SchemaUtil.Unflatten(flattenedSheet); + var content2 = SerializeUtil.Serialize(unflattenedSheet); + Assert.That(content2, Is.EqualTo(content)); + } +} \ No newline at end of file diff --git a/EXDCommon/SchemaModel/NewSheetDefinition.cs b/EXDCommon/SchemaModel/NewSheetDefinition.cs index a0578df..622e130 100644 --- a/EXDCommon/SchemaModel/NewSheetDefinition.cs +++ b/EXDCommon/SchemaModel/NewSheetDefinition.cs @@ -80,7 +80,7 @@ public class Field /// [YamlIgnore] public string PathWithArrayIndices; - + public override string ToString() { return $"{Name} ({Type})"; diff --git a/EXDCommon/Utility/SchemaUtil.cs b/EXDCommon/Utility/SchemaUtil.cs index 36c7ae3..f86d421 100644 --- a/EXDCommon/Utility/SchemaUtil.cs +++ b/EXDCommon/Utility/SchemaUtil.cs @@ -18,14 +18,16 @@ public static int GetColumnCount(Sheet sheet) public static Sheet Flatten(Sheet sheet) { + var newSheet = new Sheet { Name = sheet.Name, DisplayField = sheet.DisplayField }; var fields = new List(); + foreach (var field in sheet.Fields) Emit(fields, field); - sheet.Fields = fields; - for (int i = 0; i < sheet.Fields.Count; i++) - sheet.Fields[i].OffsetBasedIndex = (uint)i; - return sheet; + newSheet.Fields = fields; + for (int i = 0; i < newSheet.Fields.Count; i++) + newSheet.Fields[i].OffsetBasedIndex = (uint)i; + return newSheet; } /// @@ -59,12 +61,12 @@ public static List Flatten(ExcelHeaderFile exh, Sheet sheet, bool return definedColumns; } - private static void Emit(List list, Field field, List<(string, string)> hierarchy = null, string nameOverride = "") + private static void Emit(List list, Field field, List hierarchy = null, List arrayHierarchy = null, string nameOverride = "") { if (field.Type != FieldType.Array) { // Single field - list.Add(CreateField(field, false, 0, hierarchy, nameOverride)); + list.Add(CreateField(field, false, 0, hierarchy, arrayHierarchy, nameOverride)); } else if (field.Type == FieldType.Array) { @@ -73,7 +75,7 @@ private static void Emit(List list, Field field, List<(string, string)> h { for (int i = 0; i < field.Count.Value; i++) { - list.Add(CreateField(field, true, i, hierarchy, "")); + list.Add(CreateField(field, true, i, hierarchy, arrayHierarchy, "")); } } else @@ -82,18 +84,20 @@ private static void Emit(List list, Field field, List<(string, string)> h { foreach (var nestedField in field.Fields) { - var usableHierarchy = hierarchy == null ? new List<(string, string)>() : new List<(string, string)>(hierarchy); + var usableHierarchy = hierarchy == null ? new List() : new List(hierarchy); + var usableArrayHierarchy = hierarchy == null ? new List() : new List(hierarchy); var hierarchyName = $"{field.Name}"; - var hierarchyName2 = $"{field.Name}[{i}]"; - usableHierarchy.Add((hierarchyName, hierarchyName2)); - Emit(list, nestedField, usableHierarchy, field.Name); + var arrayHierarchyName = $"{field.Name}[{i}]"; + usableHierarchy.Add(hierarchyName); + usableArrayHierarchy.Add(arrayHierarchyName); + Emit(list, nestedField, usableHierarchy, usableArrayHierarchy, field.Name); } } } } } - private static Field CreateField(Field baseField, bool fieldIsArrayElement, int index, List<(string, string)>? hierarchy, string nameOverride) + private static Field CreateField(Field baseField, bool fieldIsArrayElement, int index, List? hierarchy, List? arrayHierarchy, string nameOverride) { var addedField = new Field { @@ -114,10 +118,10 @@ private static Field CreateField(Field baseField, bool fieldIsArrayElement, int path2 += $"[{index}]"; } - if (hierarchy != null) + if (hierarchy != null && arrayHierarchy != null) { addedField.Path = string.Join(".", hierarchy); - addedField.PathWithArrayIndices = string.Join(".", hierarchy); + addedField.PathWithArrayIndices = string.Join(".", arrayHierarchy); if (!string.IsNullOrEmpty(path)) addedField.Path += $".{path}"; if (!string.IsNullOrEmpty(path)) addedField.PathWithArrayIndices += $".{path2}"; } @@ -153,4 +157,67 @@ public static int GetFieldCount(Field field) } return 1; } + + public static Sheet Unflatten(Sheet sheet) + { + var newSheet = new Sheet { Name = sheet.Name, DisplayField = sheet.DisplayField }; + var fields = new List(sheet.Fields); + + var result = false; + do + { + result = Unflatten(fields); + } while (!result); + + newSheet.Fields = fields; + return newSheet; + } + + private static bool Unflatten(List fields) + { + if (!fields.Any()) return false; + // if (!Unflatten(fields)) return false; + + int start = 0, end = 0; + var found = false; + + for (int i = 0; i < fields.Count - 1; i++) + { + var currentFieldPath = ""; + var nextFieldPath = ""; + + int j = 0; + do + { + currentFieldPath = fields[i + j].Path; + nextFieldPath = fields[i + j + 1].Path; + j++; + } while (currentFieldPath == nextFieldPath && i + j < fields.Count - 1); + + start = i; + end = i + j - 1; + + // Array (defined by field path) is longer than 1 element + if (j > 1) + { + found = true; + break; + } + } + + if (found) + { + Console.WriteLine($"Found array of {fields[start].Name} from {start} to {end} with length {end - start + 1}"); + var fieldStart = fields[start]; + fieldStart = fields[start]; + fieldStart.Type = FieldType.Array; + fieldStart.Count = end - start + 1; + // fieldStart.Comment = + fields.RemoveRange(start + 1, end - start); + + return false; + } + + return true; + } } \ No newline at end of file diff --git a/EXDCommon/Utility/Util.cs b/EXDCommon/Utility/Util.cs index 3f986b7..d8d36ee 100644 --- a/EXDCommon/Utility/Util.cs +++ b/EXDCommon/Utility/Util.cs @@ -59,4 +59,20 @@ public static int GetBitOffset(int offset, ExcelColumnDataType dataType) _ => bitOffset, }; } + + public static bool DictionaryEqual(Dictionary> oldDict, Dictionary> newDict) where T : notnull + { + // Simple check, are the counts the same? + if (!oldDict.Count.Equals(newDict.Count)) return false; + + // Verify the keys + if (!oldDict.Keys.SequenceEqual(newDict.Keys)) return false; + + // Verify the values for each key + foreach (var key in oldDict.Keys) + if (!oldDict[key].SequenceEqual(newDict[key])) + return false; + + return true; + } } \ No newline at end of file diff --git a/EXDTools.sln b/EXDTools.sln index 533a64d..700e338 100644 --- a/EXDTools.sln +++ b/EXDTools.sln @@ -16,6 +16,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ZiPatchLib", "lib\ZiTool\Zi EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SchemaUpdater", "SchemaUpdater\SchemaUpdater.csproj", "{31C1028A-CB73-428B-B2E6-3DE791564840}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EXDCommon.Tests", "EXDCommon.Tests\EXDCommon.Tests.csproj", "{E8AE5811-ECF6-401E-B01F-459F93A89121}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -50,6 +52,10 @@ Global {31C1028A-CB73-428B-B2E6-3DE791564840}.Debug|Any CPU.Build.0 = Debug|Any CPU {31C1028A-CB73-428B-B2E6-3DE791564840}.Release|Any CPU.ActiveCfg = Release|Any CPU {31C1028A-CB73-428B-B2E6-3DE791564840}.Release|Any CPU.Build.0 = Release|Any CPU + {E8AE5811-ECF6-401E-B01F-459F93A89121}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E8AE5811-ECF6-401E-B01F-459F93A89121}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E8AE5811-ECF6-401E-B01F-459F93A89121}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E8AE5811-ECF6-401E-B01F-459F93A89121}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(NestedProjects) = preSolution {E974175E-D301-4F3F-A8BD-7AC42BD3D9FB} = {5C037CD8-C1A6-407C-AE18-5F09AE478977} diff --git a/EXDTools.sln.DotSettings.user b/EXDTools.sln.DotSettings.user index 95814ef..fb6f53f 100644 --- a/EXDTools.sln.DotSettings.user +++ b/EXDTools.sln.DotSettings.user @@ -1,4 +1,11 @@  + ForceIncluded <AssemblyExplorer> <Assembly Path="C:\Users\Liam\.nuget\packages\lumina\3.15.2\lib\net7.0\Lumina.dll" /> -</AssemblyExplorer> \ No newline at end of file +</AssemblyExplorer> + C:\Users\Liam\AppData\Local\JetBrains\Rider2023.3\resharper-host\temp\Rider\vAny\CoverageData\_EXDTools.641274775\Snapshot\snapshot.utdcvr + <SessionState ContinuousTestingMode="0" IsActive="True" Name="All tests from &lt;EXDCommon.Tests&gt; #3" xmlns="urn:schemas-jetbrains-com:jetbrains-ut-session"> + <Project Location="C:\Users\Liam\Documents\repos\EXDTools\EXDCommon.Tests" Presentation="&lt;EXDCommon.Tests&gt;" /> +</SessionState> + + \ No newline at end of file diff --git a/SchemaUpdater/SchemaUpdater.cs b/SchemaUpdater/SchemaUpdater.cs index 4c5eff8..ee5ab42 100644 --- a/SchemaUpdater/SchemaUpdater.cs +++ b/SchemaUpdater/SchemaUpdater.cs @@ -1,4 +1,5 @@ -using System.Numerics; +using System.Diagnostics; +using System.Numerics; using EXDCommon.FileAccess; using EXDCommon.SchemaModel.EXDSchema; using EXDCommon.Sheets; @@ -307,19 +308,43 @@ private uint FindNewIndex(ExcelSheetImpl oldSheet, uint oldColumnIdx, ExcelSheet return ""; } + + public static void TestUnflatten() + { + // Changed stuff test cases + // Aetheryte: added column at 20 (between columns) + // GimmickJump: unrecognized column changed in the middle of the columns + // WorldDCGroupType: added column at 2 (end) + + // Unflatten test cases + // No arrays: ActionCastTimeline + // var loadedSheet = SerializeUtil.Deserialize(File.ReadAllText(@"C:\Users\Liam\Documents\repos\EXDSchema\Schemas\2024.01.06.0000.0000\ActionCastTimeline.yml")); + // Simple array with no links: AnimationLOD + // var loadedSheet = SerializeUtil.Deserialize(File.ReadAllText(@"C:\Users\Liam\Documents\repos\EXDSchema\Schemas\2024.01.06.0000.0000\AnimationLOD.yml")); + + var loadedSheet = SerializeUtil.Deserialize(File.ReadAllText(@"C:\Users\Liam\Documents\repos\EXDSchema\Schemas\2024.01.06.0000.0000\ActionCastTimeline.yml")); + var flattened = SchemaUtil.Flatten(loadedSheet); + // Debugger.Break(); + + loadedSheet = SerializeUtil.Deserialize(File.ReadAllText(@"C:\Users\Liam\Documents\repos\EXDSchema\Schemas\2024.01.06.0000.0000\AnimationLOD.yml")); + flattened = SchemaUtil.Flatten(loadedSheet); + var unflatten = SchemaUtil.Unflatten(flattened); + Debugger.Break(); + + } private Sheet GenerateNewSchema(Sheet oldSchema, Sheet newSchema, Dictionary columnMap) { var flattenedOldSchema = SchemaUtil.Flatten(oldSchema); - + return null; } - // private Dictionary LoadColumnMap(string sheet, string mapFile) - // { - // var json = File.ReadAllText(mapFile); - // var data = JsonConvert.DeserializeObject>>(json); - // return data[sheet]; - // } + private Dictionary LoadColumnMap(string sheet, string mapFile) + { + var json = File.ReadAllText(mapFile); + var data = JsonConvert.DeserializeObject>>(json); + return data[sheet]; + } private Dictionary GenerateColumnMap(RawExcelSheet oldSheet, RawExcelSheet newSheet) { diff --git a/SchemaValidator/SchemaValidator.cs b/SchemaValidator/SchemaValidator.cs index 8cb679f..b5c7a4c 100644 --- a/SchemaValidator/SchemaValidator.cs +++ b/SchemaValidator/SchemaValidator.cs @@ -15,6 +15,23 @@ namespace SchemaValidator; public class SchemaValidator { + public static void Thing() + { + var outputDir = @"C:\Users\Liam\Desktop\tmp\worker\output"; + var storageDir = @"C:\Users\Liam\Desktop\tmp\worker\storage"; + + var baseDir = @"C:\Users\Liam\Documents\repos\EXDSchema\Schemas"; + var schemaFileDir = @"C:\Users\Liam\Documents\repos\EXDTools\SchemaValidator\Schema.json"; + + foreach (var schemaDir in Directory.EnumerateDirectories(baseDir)) + { + var gameVer = Path.GetFileName(schemaDir); + var dir = Path.Combine(outputDir, $"{gameVer}.json"); + Console.WriteLine($"Gamever {gameVer}"); + Main(new[] { dir, storageDir, schemaFileDir, schemaDir }); + } + } + public static void Main(string[] args) { if (args.Length is not 3 and not 4 and not 5) @@ -127,23 +144,23 @@ public static void Main(string[] args) // --- - foreach (var result in results.Results.Where(r => r.Status == ValidationStatus.Warning)) - { - var msgfmt = result.Message == "" ? "" : $" - "; - Console.WriteLine($"{result.Status}: {result.SheetName} - {result.ValidatorName}{msgfmt}{result.Message}"); - } - - foreach (var result in results.Results.Where(r => r.Status == ValidationStatus.Failed)) - { - var msgfmt = result.Message == "" ? "" : $" - "; - Console.WriteLine($"{result.Status}: {result.SheetName} - {result.ValidatorName}{msgfmt}{result.Message}"); - } - - foreach (var result in results.Results.Where(r => r.Status == ValidationStatus.Error)) - { - var msgfmt = result.Message == "" ? "" : $" - "; - Console.WriteLine($"{result.Status}: {result.SheetName} - {result.ValidatorName}{msgfmt}{result.Message}"); - } + // foreach (var result in results.Results.Where(r => r.Status == ValidationStatus.Warning)) + // { + // var msgfmt = result.Message == "" ? "" : $" - "; + // Console.WriteLine($"{result.Status}: {result.SheetName} - {result.ValidatorName}{msgfmt}{result.Message}"); + // } + // + // foreach (var result in results.Results.Where(r => r.Status == ValidationStatus.Failed)) + // { + // var msgfmt = result.Message == "" ? "" : $" - "; + // Console.WriteLine($"{result.Status}: {result.SheetName} - {result.ValidatorName}{msgfmt}{result.Message}"); + // } + // + // foreach (var result in results.Results.Where(r => r.Status == ValidationStatus.Error)) + // { + // var msgfmt = result.Message == "" ? "" : $" - "; + // Console.WriteLine($"{result.Status}: {result.SheetName} - {result.ValidatorName}{msgfmt}{result.Message}"); + // } var successCount = results.Results.Count(r => r.Status == ValidationStatus.Success); var warningCount = results.Results.Count(r => r.Status == ValidationStatus.Warning);