Skip to content

Commit f80a9fb

Browse files
author
Sam Byass
committed
Pushing what I have locally (read commit body):
- New hook for output formats - Plugin loggers - Binary structure size tracking - ELF loading time optimizations - v29.1 support - v27 generic type fixes - Moved build report to a plugin
1 parent f2e894c commit f80a9fb

24 files changed

+368
-48
lines changed

‎.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@ SharpTreeView/obj/
3333
/StableNameDotNet/bin/
3434
/StableNameDotNet/obj/
3535

36+
/Cpp2IL.Plugin.BuildReport/bin/
37+
/Cpp2IL.Plugin.BuildReport/obj/
38+
3639
.vs
3740

3841
*.user

‎Cpp2IL.Core/Api/Cpp2IlOutputFormat.cs

+9
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,15 @@ public abstract class Cpp2IlOutputFormat
1313
/// The name of the output format displayed to the user (e.g. in logs or the GUI)
1414
/// </summary>
1515
public abstract string OutputFormatName { get; }
16+
17+
/// <summary>
18+
/// Called when this output format is selected by the user, before any binary is loaded.
19+
/// You do not have a context, but you could use this to, for example, configure the library if you need to enable a feature.
20+
/// </summary>
21+
public virtual void OnOutputFormatSelected()
22+
{
23+
24+
}
1625

1726
public abstract void DoOutput(ApplicationAnalysisContext context, string outputRoot);
1827
}

‎Cpp2IL.Core/Api/Cpp2IlPlugin.cs

+8
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,19 @@
11
using System;
22
using System.IO;
3+
using Cpp2IL.Core.Logging;
34
using LibCpp2IL;
45

56
namespace Cpp2IL.Core.Api;
67

78
public abstract class Cpp2IlPlugin
89
{
10+
protected readonly PluginLogger Logger;
11+
12+
protected Cpp2IlPlugin()
13+
{
14+
Logger = new(this);
15+
}
16+
917
public abstract string Name { get; }
1018
public abstract string Description { get; }
1119

‎Cpp2IL.Core/Api/PluginLogger.cs

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
using Cpp2IL.Core.Logging;
2+
3+
namespace Cpp2IL.Core.Api;
4+
5+
public sealed class PluginLogger
6+
{
7+
private readonly Cpp2IlPlugin _plugin;
8+
private readonly string _name;
9+
10+
internal PluginLogger(Cpp2IlPlugin plugin)
11+
{
12+
_plugin = plugin;
13+
_name = $"Plugin: {plugin.Name}";
14+
}
15+
16+
public void VerboseNewline(string message, string source = "Program") => Logger.VerboseNewline($"{message}", _name);
17+
18+
public void Verbose(string message, string source = "Program") => Logger.Verbose($"{message}", _name);
19+
20+
public void InfoNewline(string message, string source = "Program") => Logger.InfoNewline($"{message}", _name);
21+
22+
public void Info(string message, string source = "Program") => Logger.Info($"{message}", _name);
23+
24+
public void WarnNewline(string message, string source = "Program") => Logger.WarnNewline($"{message}", _name);
25+
26+
public void Warn(string message, string source = "Program") => Logger.Warn($"{message}", _name);
27+
28+
public void ErrorNewline(string message, string source = "Program") => Logger.ErrorNewline($"{message}", _name);
29+
30+
public void Error(string message, string source = "Program") => Logger.Error($"{message}", _name);
31+
}

‎Cpp2IL.Core/CorePlugin/Cpp2IlCorePlugin.cs

-1
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@ public override void OnLoad()
3737
OutputFormatRegistry.Register<AsmResolverDummyDllOutputFormat>();
3838
OutputFormatRegistry.Register<IsilDumpOutputFormat>();
3939
OutputFormatRegistry.Register<WasmMappingOutputFormat>();
40-
OutputFormatRegistry.Register<BuildReportOutputFormat>();
4140

4241
Logger.VerboseNewline("\tRegistering built-in processing layers", "Core Plugin");
4342

‎Cpp2IL.Core/Cpp2IlPluginManager.cs

+2-1
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,11 @@ public static class Cpp2IlPluginManager
1515

1616
internal static void LoadFromDirectory(string pluginsDir)
1717
{
18+
Logger.InfoNewline($"Loading plugins from {pluginsDir}...", "Plugins");
19+
1820
if(!Directory.Exists(pluginsDir))
1921
return;
2022

21-
Logger.InfoNewline($"Loading plugins from {pluginsDir}...", "Plugins");
2223
foreach (var file in Directory.EnumerateFiles(pluginsDir))
2324
{
2425
if (Path.GetExtension(file) == ".dll")

‎Cpp2IL.Core/Utils/V29AttributeUtils.cs

+9
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
using Cpp2IL.Core.Extensions;
77
using Cpp2IL.Core.Model.Contexts;
88
using Cpp2IL.Core.Model.CustomAttributes;
9+
using LibCpp2IL;
910
using LibCpp2IL.BinaryStructures;
1011
using LibCpp2IL.Metadata;
1112

@@ -20,6 +21,9 @@ public static Il2CppMethodDefinition[] ReadConstructors(Stream stream, uint coun
2021

2122
for (var i = 0; i < count; i++)
2223
indices[i] = reader.ReadUInt32();
24+
25+
if(ClassReadingBinaryReader.EnableReadableSizeInformation)
26+
context.Metadata.TrackRead<AnalyzedCustomAttribute>((int) (4 * count), trackIfFinishedReading: true);
2327

2428
return indices.Select(i => context.Metadata.methodDefs[i]).ToArray();
2529
}
@@ -28,6 +32,8 @@ public static AnalyzedCustomAttribute ReadAttribute(Stream stream, MethodAnalysi
2832
{
2933
var ret = new AnalyzedCustomAttribute(constructor);
3034

35+
var startPos = stream.Position;
36+
3137
var numCtorArgs = stream.ReadUnityCompressedUint();
3238
var numFields = stream.ReadUnityCompressedUint();
3339
var numProps = stream.ReadUnityCompressedUint();
@@ -60,6 +66,9 @@ public static AnalyzedCustomAttribute ReadAttribute(Stream stream, MethodAnalysi
6066

6167
ret.Properties.Add(new(property, value));
6268
}
69+
70+
if(ClassReadingBinaryReader.EnableReadableSizeInformation)
71+
context.Metadata.TrackRead<AnalyzedCustomAttribute>((int) (stream.Position - startPos), trackIfFinishedReading: true);
6372

6473
return ret;
6574
}

‎Cpp2IL.Core/CorePlugin/BuildReportOutputFormat.cs ‎Cpp2IL.Plugin.BuildReport/BuildReportOutputFormat.cs

+77-12
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,25 @@
1-
using System;
2-
using System.Collections.Generic;
3-
using System.IO;
4-
using System.Linq;
5-
using System.Text;
1+
using System.Text;
2+
using System.Text.Json;
63
using Cpp2IL.Core.Api;
74
using Cpp2IL.Core.Logging;
85
using Cpp2IL.Core.Model.Contexts;
6+
using Cpp2IL.Plugin.BuildReport.Model;
97
using LibCpp2IL;
108
using LibCpp2IL.Metadata;
119

12-
namespace Cpp2IL.Core.CorePlugin;
10+
namespace Cpp2IL.Plugin.BuildReport;
1311

1412
public class BuildReportOutputFormat : Cpp2IlOutputFormat
1513
{
1614
public override string OutputFormatId => "buildreport";
1715
public override string OutputFormatName => "IL2CPP Build Report";
18-
16+
17+
public override void OnOutputFormatSelected()
18+
{
19+
//We want readable information for metadata sizes in the build report
20+
ClassReadingBinaryReader.EnableReadableSizeInformation = true;
21+
}
22+
1923
public override void DoOutput(ApplicationAnalysisContext context, string outputRoot)
2024
{
2125
//This output format serves as a way to see what's taking up space (and thus, presumably, build time) in your game
@@ -53,15 +57,25 @@ public override void DoOutput(ApplicationAnalysisContext context, string outputR
5357
}
5458

5559
var numberOfGenericVariants = genericMethodDataByBaseMd.Values.Sum(gmd => gmd.NumVariants);
60+
var numberOfGenericPointers = genericMethodDataByBaseMd.Values.Sum(gmd => gmd.CountedVariantAddresses.Count);
5661
var sizeOfAllGenerics = genericMethodDataByBaseMd.Sum(m => (long) m.Value.TotalSizeInstructions);
5762

5863
var mostUsedGenericMethods = genericMethodDataByBaseMd.OrderByDescending(kvp => kvp.Value.NumVariants).Take(100).ToList();
5964
var mostSpaceConsumingGenericMethods = genericMethodDataByBaseMd.OrderByDescending(kvp => kvp.Value.TotalSizeInstructions).Take(100).ToList();
6065

61-
var methodsBySize = context.AllTypes.SelectMany(t => t.Methods).Where(m => m.UnderlyingPointer != 0).ToDictionary(m => m, m => m.RawBytes.Length);
66+
var methodsBySize = context.AllTypes.SelectMany(t => t.Methods).Where(m => m is not InjectedMethodAnalysisContext).Where(m => m.UnderlyingPointer != 0).ToDictionary(m => m, m => m.RawBytes.Length);
67+
var numberOfNonGenerics = methodsBySize.Count;
6268
var sizeOfAllNonGenerics = methodsBySize.Sum(m => m.Value);
6369
var largestMethods = methodsBySize.OrderByDescending(kvp => kvp.Value).Take(100).ToList();
6470

71+
var inBinaryMetadataObjectsBySize = context.Binary.BytesReadPerClass.OrderByDescending(kvp => kvp.Value).ToList();
72+
var inBinaryMetadataUnspecifiedSize = context.Binary.InBinaryMetadataSize - inBinaryMetadataObjectsBySize.Sum(kvp => kvp.Value);
73+
inBinaryMetadataObjectsBySize.Add(new(null, (int) inBinaryMetadataUnspecifiedSize));
74+
75+
var metadataFileObjectsBySize = context.Metadata.BytesReadPerClass.OrderByDescending(kvp => kvp.Value).ToList();
76+
var metadataFileUnspecifiedSize = context.Metadata.Length - metadataFileObjectsBySize.Sum(kvp => kvp.Value);
77+
metadataFileObjectsBySize.Add(new(null, (int) metadataFileUnspecifiedSize));
78+
6579
var attributeGeneratorsBySize = context.Assemblies
6680
.Cast<HasCustomAttributes>()
6781
.Concat(context.AllTypes)
@@ -106,13 +120,13 @@ public override void DoOutput(ApplicationAnalysisContext context, string outputR
106120
ret.AppendLine($"IL2CPP Binary report generated at {DateTime.Now:f}:");
107121
ret.AppendLine();
108122
ret.AppendLine($"Binary size: {binarySize} bytes ({binarySize / 1024f / 1024:f2}MB)");
109-
ret.AppendLine($" Of which generic method bodies: {sizeOfAllGenerics} bytes ({sizeOfAllGenerics / 1024f / 1024:f2}MB, {(double) sizeOfAllGenerics / binarySize:p})");
110-
ret.AppendLine($" Of which non-generic method bodies: {sizeOfAllNonGenerics} bytes ({sizeOfAllNonGenerics / 1024f / 1024:f2}MB, {(double) sizeOfAllNonGenerics / binarySize:p})");
123+
ret.AppendLine($" Of which {numberOfGenericPointers} generic method bodies: {sizeOfAllGenerics} bytes ({sizeOfAllGenerics / 1024f / 1024:f2}MB, {(double) sizeOfAllGenerics / binarySize:p})");
124+
ret.AppendLine($" Of which {numberOfNonGenerics} non-generic method bodies: {sizeOfAllNonGenerics} bytes ({sizeOfAllNonGenerics / 1024f / 1024:f2}MB, {(double) sizeOfAllNonGenerics / binarySize:p})");
111125
ret.AppendLine($" Of which Custom Attribute generator bodies: {sizeOfAllCaGenerators} bytes ({sizeOfAllCaGenerators / 1024f / 1024:f2}MB, {(double) sizeOfAllCaGenerators / binarySize:p})");
112126
ret.AppendLine($" Of which in-binary il2cpp metadata: {metadataInBinarySize} bytes ({metadataInBinarySize / 1024f / 1024:f2}MB, {(double) metadataInBinarySize / binarySize:p})");
113127
ret.AppendLine($" Of which C++ functions (il2cpp api functions, etc) and misc data: {unclassifiedSize} bytes ({(unclassifiedSize) / 1024f / 1024:f2}MB, {(double) unclassifiedSize / binarySize:p})");
114128
ret.AppendLine();
115-
ret.AppendLine($"Total number of generic variants: {numberOfGenericVariants}");
129+
ret.AppendLine($"Total number of generic variants: {numberOfGenericVariants} ({numberOfGenericPointers} after generic merging)");
116130
ret.AppendLine(" Top 100 most varied generic methods:");
117131
ret.AppendLine();
118132

@@ -145,11 +159,47 @@ public override void DoOutput(ApplicationAnalysisContext context, string outputR
145159
foreach (var kvp in sortedUsageByAssembly)
146160
ret.AppendLine($" {kvp.Key.Definition.AssemblyName.Name}: {kvp.Value / 1024f / 1024:f3}MB ({(double) kvp.Value / binarySize:P} of binary)");
147161

162+
ret.AppendLine();
163+
ret.AppendLine(" In-Binary Metadata breakdown:");
164+
165+
foreach (var kvp in inBinaryMetadataObjectsBySize)
166+
if(kvp.Value >= 0)
167+
ret.AppendLine($" {kvp.Key?.Name ?? "Unspecified"}: {kvp.Value} bytes ({kvp.Value / 1024f / 1024:f2}MB)");
168+
169+
ret.AppendLine();
170+
ret.AppendLine($" Metadata file (global-metadata.dat) breakdown (total size {context.Metadata.Length} bytes, {context.Metadata.Length / 1024f / 1024:f2}MB):");
171+
172+
foreach (var kvp in metadataFileObjectsBySize)
173+
if(kvp.Value >= 0)
174+
ret.AppendLine($" {kvp.Key?.Name ?? "Unspecified"}: {kvp.Value} bytes ({kvp.Value / 1024f / 1024:f2}MB)");
175+
148176
//Save output
149177
var outputPath = Path.Combine(outputRoot, "buildreport.txt");
150178
File.WriteAllText(outputPath, ret.ToString());
151179

152-
Logger.InfoNewline($"Wrote build report to {outputPath}", "BuildReportOutputFormat");
180+
Logger.InfoNewline($"Wrote human-readable build report to {outputPath}", "BuildReportOutputFormat");
181+
182+
//Save JSON output
183+
var jsonOutputPath = Path.Combine(outputRoot, "buildreport.json");
184+
185+
var outputData = new OutputData();
186+
187+
outputData.Binary.RawReadableClasses = inBinaryMetadataObjectsBySize.Select(kvp => new OutputReadableClassData
188+
{
189+
BytesUsed = kvp.Value,
190+
TypeName = kvp.Key?.Name ?? "Unspecified",
191+
}).ToArray();
192+
193+
outputData.GlobalMetadata.RawReadableClasses = metadataFileObjectsBySize.Select(kvp => new OutputReadableClassData
194+
{
195+
BytesUsed = kvp.Value,
196+
TypeName = kvp.Key?.Name ?? "Unspecified",
197+
}).ToArray();
198+
199+
var jsonText = JsonSerializer.Serialize(outputData, new JsonSerializerOptions() {WriteIndented = true});
200+
File.WriteAllText(jsonOutputPath, jsonText);
201+
202+
Logger.InfoNewline($"Wrote JSON build report to {jsonOutputPath}", "BuildReportOutputFormat");
153203
}
154204

155205
private static string GetMethodName(Il2CppMethodDefinition md)
@@ -161,7 +211,9 @@ private class GenericMethodData
161211
{
162212
public int NumVariants;
163213
public HashSet<ulong> CountedVariantAddresses = new();
214+
public HashSet<string> FullNames = new();
164215
public int TotalSizeInstructions;
216+
public List<OutputMethodData> OutputMethodData = new();
165217

166218
public GenericMethodData(ApplicationAnalysisContext context, Il2CppMethodDefinition md)
167219
{
@@ -172,11 +224,24 @@ public GenericMethodData(ApplicationAnalysisContext context, Il2CppMethodDefinit
172224
{
173225
if(CountedVariantAddresses.Contains(cpp2IlMethodRef.GenericVariantPtr))
174226
continue;
227+
228+
var outputMethod = new OutputMethodData()
229+
{
230+
FullName = cpp2IlMethodRef.ToString(),
231+
NonGenericVersionFullName = cpp2IlMethodRef.BaseMethod.HumanReadableSignature,
232+
IsRemovedByGenericSharing = true,
233+
};
175234

235+
OutputMethodData.Add(outputMethod);
236+
176237
if(!context.ConcreteGenericMethodsByRef.TryGetValue(cpp2IlMethodRef, out var concreteAnalysisContext))
177238
continue; //Eliminated by generic sharing
239+
240+
outputMethod.IsRemovedByGenericSharing = false;
241+
outputMethod.MachineCodeSizeBytes = concreteAnalysisContext.RawBytes.Length;
178242

179243
CountedVariantAddresses.Add(cpp2IlMethodRef.GenericVariantPtr);
244+
FullNames.Add(cpp2IlMethodRef.ToString());
180245
TotalSizeInstructions += concreteAnalysisContext.RawBytes.Length;
181246
}
182247
}

‎Cpp2IL.Plugin.BuildReport/BuildReportPlugin.cs

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
using Cpp2IL.Core.Api;
2+
using Cpp2IL.Core.Attributes;
3+
using Cpp2IL.Plugin.BuildReport;
4+
5+
[assembly: RegisterCpp2IlPlugin(typeof(BuildReportPlugin))]
6+
7+
namespace Cpp2IL.Plugin.BuildReport;
8+
9+
public class BuildReportPlugin : Cpp2IlPlugin
10+
{
11+
public override string Name => "Build Report Plugin";
12+
public override string Description => "Adds an output format which generates information useful to the developer about what is taking up space in the build process";
13+
public override void OnLoad()
14+
{
15+
OutputFormatRegistry.Register<BuildReportOutputFormat>();
16+
Logger.VerboseNewline("Build Report Plugin loaded and output format registered.");
17+
}
18+
}

‎Cpp2IL.Plugin.BuildReport/Cpp2IL.Plugin.BuildReport.csproj

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<TargetFramework>net6.0</TargetFramework>
5+
<ImplicitUsings>enable</ImplicitUsings>
6+
<Nullable>enable</Nullable>
7+
</PropertyGroup>
8+
9+
<ItemGroup>
10+
<ProjectReference Include="..\Cpp2IL.Core\Cpp2IL.Core.csproj">
11+
<PrivateAssets>all</PrivateAssets>
12+
<IncludeAssets>compile; build</IncludeAssets>
13+
<ExcludedAssets>runtime</ExcludedAssets>
14+
</ProjectReference>
15+
</ItemGroup>
16+
17+
<ItemGroup>
18+
<PackageReference Include="System.Text.Json" Version="6.0.5" />
19+
</ItemGroup>
20+
21+
</Project>

‎Cpp2IL.Plugin.BuildReport/Model/OutputBinaryData.cs

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
namespace Cpp2IL.Plugin.BuildReport.Model;
2+
3+
public class OutputBinaryData
4+
{
5+
public OutputMethodData[] Methods { get; set; }
6+
public OutputReadableClassData[] RawReadableClasses { get; set; }
7+
}

‎Cpp2IL.Plugin.BuildReport/Model/OutputData.cs

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
namespace Cpp2IL.Plugin.BuildReport.Model;
2+
3+
public class OutputData
4+
{
5+
public OutputBinaryData Binary { get; set; } = new();
6+
public OutputGlobalMetadataData GlobalMetadata { get; set; } = new();
7+
}

‎Cpp2IL.Plugin.BuildReport/Model/OutputGlobalMetadataData.cs

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
namespace Cpp2IL.Plugin.BuildReport.Model;
2+
3+
public class OutputGlobalMetadataData
4+
{
5+
public OutputReadableClassData[] RawReadableClasses { get; set; }
6+
}

‎Cpp2IL.Plugin.BuildReport/Model/OutputMethodData.cs

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
namespace Cpp2IL.Plugin.BuildReport.Model;
2+
3+
public class OutputMethodData
4+
{
5+
public string FullName { get; set; }
6+
public int MachineCodeSizeBytes { get; set; }
7+
public string? NonGenericVersionFullName { get; set; }
8+
public bool IsRemovedByGenericSharing { get; set; }
9+
}

‎Cpp2IL.Plugin.BuildReport/Model/OutputReadableClassData.cs

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
namespace Cpp2IL.Plugin.BuildReport.Model;
2+
3+
public class OutputReadableClassData
4+
{
5+
public string TypeName { get; set; }
6+
public int BytesUsed { get; set; }
7+
}

0 commit comments

Comments
 (0)