Skip to content

Commit 44c5c30

Browse files
jpobstjpobst
authored
[XABT] Make assembly compression incremental. (#9704)
If using `$(AndroidEnableAssemblyCompression)` today, we do not compress assemblies incrementally. That is, we always compress every assembly even if they haven't changed since the previous build. Update the `CompressAssemblies` task to check if the input is newer than the output, and skip recompressing if the input has not changed. This is most visible with the following test case: ``` dotnet new android dotnet build -p:EmbedAssembliesIntoApk=true -p:AndroidIncludeDebugSymbols=false ``` | Scenario (`CompressAssemblies` tasks) | main | This PR | | --------------- | -------- | -------- | | Full | 44.26 s | 44.2 s | | NoChanges | not run | 34 ms | | ChangeResource | 27.8 s | 9 ms | | AddResource | 25.81 s | 25 ms | | ChangeCSharp | 25.87 s | 30 ms | | ChangeCSharpJLO | 27.04 s | 23.36 s | `ChangeCSharpJLO` should be similarly reduced, however something earlier in the build process is causing `Mono.Android.dll` to get touched so it is getting recompressed. We'll leave that as an investigation for another day.
1 parent 70a4f93 commit 44c5c30

File tree

7 files changed

+317
-134
lines changed

7 files changed

+317
-134
lines changed

‎src/Xamarin.Android.Build.Tasks/Tasks/CollectAssemblyFilesForArchive.cs

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
#nullable enable
2+
3+
using System;
4+
using System.Collections.Generic;
5+
using System.Linq;
6+
using Microsoft.Android.Build.Tasks;
7+
using Microsoft.Build.Framework;
8+
using Microsoft.Build.Utilities;
9+
using Xamarin.Android.Tools;
10+
11+
namespace Xamarin.Android.Tasks;
12+
13+
/// <summary>
14+
/// This task figures out the compression/assembly store/wrapping operations that need to
15+
/// be performed on the assemblies before they are added to the APK. This is done "ahead of time"
16+
/// so that the actual work can be done in an incremental way.
17+
/// </summary>
18+
public class CollectAssemblyFilesToCompress : AndroidTask
19+
{
20+
public override string TaskPrefix => "CAF";
21+
22+
[Required]
23+
public string AssemblyCompressionDirectory { get; set; } = "";
24+
25+
public bool EmbedAssemblies { get; set; }
26+
27+
[Required]
28+
public bool EnableCompression { get; set; }
29+
30+
public bool IncludeDebugSymbols { get; set; }
31+
32+
[Required]
33+
public string ProjectFullPath { get; set; } = "";
34+
35+
[Required]
36+
public ITaskItem [] ResolvedFrameworkAssemblies { get; set; } = [];
37+
38+
[Required]
39+
public ITaskItem [] ResolvedUserAssemblies { get; set; } = [];
40+
41+
[Required]
42+
public string [] SupportedAbis { get; set; } = [];
43+
44+
[Output]
45+
public ITaskItem [] AssembliesToCompressOutput { get; set; } = [];
46+
47+
[Output]
48+
public ITaskItem [] ResolvedFrameworkAssembliesOutput { get; set; } = [];
49+
50+
[Output]
51+
public ITaskItem [] ResolvedUserAssembliesOutput { get; set; } = [];
52+
53+
public override bool RunTask ()
54+
{
55+
ResolvedFrameworkAssembliesOutput = ResolvedFrameworkAssemblies;
56+
ResolvedUserAssembliesOutput = ResolvedUserAssemblies;
57+
58+
// We aren't going to compress any assemblies
59+
if (IncludeDebugSymbols || !EnableCompression || !EmbedAssemblies)
60+
return true;
61+
62+
var assemblies_to_compress = new List<ITaskItem> ();
63+
var compressed_assemblies_info = GetCompressedAssemblyInfo ();
64+
65+
// Get all the user and framework assemblies we may need to compresss
66+
var assemblies = ResolvedFrameworkAssemblies.Concat (ResolvedUserAssemblies).Where (asm => !(ShouldSkipAssembly (asm))).ToArray ();
67+
var per_arch_assemblies = MonoAndroidHelper.GetPerArchAssemblies (assemblies, SupportedAbis, true);
68+
69+
foreach (var kvp in per_arch_assemblies) {
70+
Log.LogDebugMessage ($"Preparing assemblies for architecture '{kvp.Key}'");
71+
72+
foreach (var asm in kvp.Value.Values) {
73+
74+
if (bool.TryParse (asm.GetMetadata ("AndroidSkipCompression"), out bool value) && value) {
75+
Log.LogDebugMessage ($"Skipping compression of {asm.ItemSpec} due to 'AndroidSkipCompression' == 'true' ");
76+
continue;
77+
}
78+
79+
if (!AssemblyCompression.TryGetDescriptorIndex (Log, asm, compressed_assemblies_info, out var descriptor_index)) {
80+
Log.LogDebugMessage ($"Skipping compression of {asm.ItemSpec} due to missing descriptor index.");
81+
continue;
82+
}
83+
84+
var compressed_assembly = AssemblyCompression.GetCompressedAssemblyOutputPath (asm, AssemblyCompressionDirectory);
85+
86+
assemblies_to_compress.Add (CreateAssemblyToCompress (asm.ItemSpec, compressed_assembly, descriptor_index));
87+
88+
// Mark this assembly as "compressed", if the compression process fails we will remove this metadata later
89+
asm.SetMetadata ("CompressedAssembly", compressed_assembly);
90+
}
91+
}
92+
93+
AssembliesToCompressOutput = assemblies_to_compress.ToArray ();
94+
95+
return !Log.HasLoggedErrors;
96+
}
97+
98+
TaskItem CreateAssemblyToCompress (string sourceAssembly, string destinationAssembly, uint descriptorIndex)
99+
{
100+
var item = new TaskItem (sourceAssembly);
101+
item.SetMetadata ("DestinationPath", destinationAssembly);
102+
item.SetMetadata ("DescriptorIndex", descriptorIndex.ToString ());
103+
104+
return item;
105+
}
106+
107+
IDictionary<AndroidTargetArch, Dictionary<string, CompressedAssemblyInfo>> GetCompressedAssemblyInfo ()
108+
{
109+
var key = CompressedAssemblyInfo.GetKey (ProjectFullPath);
110+
Log.LogDebugMessage ($"Retrieving assembly compression info with key '{key}'");
111+
112+
var compressedAssembliesInfo = BuildEngine4.UnregisterTaskObjectAssemblyLocal<IDictionary<AndroidTargetArch, Dictionary<string, CompressedAssemblyInfo>>> (key, RegisteredTaskObjectLifetime.Build);
113+
114+
if (compressedAssembliesInfo is null)
115+
throw new InvalidOperationException ($"Assembly compression info not found for key '{key}'. Compression will not be performed.");
116+
BuildEngine4.RegisterTaskObjectAssemblyLocal (key, compressedAssembliesInfo, RegisteredTaskObjectLifetime.Build);
117+
118+
return compressedAssembliesInfo;
119+
}
120+
121+
bool ShouldSkipAssembly (ITaskItem asm)
122+
{
123+
var should_skip = asm.GetMetadataOrDefault ("AndroidSkipAddToPackage", false);
124+
125+
if (should_skip)
126+
Log.LogDebugMessage ($"Skipping {asm.ItemSpec} due to 'AndroidSkipAddToPackage' == 'true' ");
127+
128+
return should_skip;
129+
}
130+
}

‎src/Xamarin.Android.Build.Tasks/Tasks/CompressAssemblies.cs

Lines changed: 20 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,8 @@
22

33
using System;
44
using System.Collections.Generic;
5-
using System.IO;
6-
using System.Linq;
75
using Microsoft.Android.Build.Tasks;
86
using Microsoft.Build.Framework;
9-
using Xamarin.Android.Tools;
107

118
namespace Xamarin.Android.Tasks;
129

@@ -20,89 +17,39 @@ public class CompressAssemblies : AndroidTask
2017
public override string TaskPrefix => "CAS";
2118

2219
[Required]
23-
public string ApkOutputPath { get; set; } = "";
24-
25-
public bool EmbedAssemblies { get; set; }
26-
27-
[Required]
28-
public bool EnableCompression { get; set; }
29-
30-
public bool IncludeDebugSymbols { get; set; }
31-
32-
[Required]
33-
public string ProjectFullPath { get; set; } = "";
34-
35-
[Required]
36-
public ITaskItem [] ResolvedFrameworkAssemblies { get; set; } = [];
37-
38-
[Required]
39-
public ITaskItem [] ResolvedUserAssemblies { get; set; } = [];
40-
41-
[Required]
42-
public string [] SupportedAbis { get; set; } = [];
20+
public ITaskItem [] AssembliesToCompress { get; set; } = [];
4321

4422
[Output]
45-
public ITaskItem [] ResolvedFrameworkAssembliesOutput { get; set; } = [];
46-
47-
[Output]
48-
public ITaskItem [] ResolvedUserAssembliesOutput { get; set; } = [];
23+
public ITaskItem [] FailedToCompressAssembliesOutput { get; set; } = [];
4924

5025
public override bool RunTask ()
5126
{
52-
if (IncludeDebugSymbols || !EnableCompression || !EmbedAssemblies) {
53-
ResolvedFrameworkAssembliesOutput = ResolvedFrameworkAssemblies;
54-
ResolvedUserAssembliesOutput = ResolvedUserAssemblies;
55-
return true;
56-
}
27+
var failed_assemblies = new List<ITaskItem> ();
5728

58-
var compressed_assemblies_info = GetCompressedAssemblyInfo ();
29+
foreach (var assembly in AssembliesToCompress) {
30+
MonoAndroidHelper.LogIfReferenceAssembly (assembly, Log);
5931

60-
// Get all the user and framework assemblies we may need to compresss
61-
var assemblies = ResolvedFrameworkAssemblies.Concat (ResolvedUserAssemblies).Where (asm => !(ShouldSkipAssembly (asm))).ToArray ();
62-
var per_arch_assemblies = MonoAndroidHelper.GetPerArchAssemblies (assemblies, SupportedAbis, true);
63-
var compressed_output_dir = Path.GetFullPath (Path.Combine (Path.GetDirectoryName (ApkOutputPath), "..", "lz4"));
32+
if (!assembly.TryGetRequiredMetadata ("AssembliesToCompress", "DestinationPath", Log, out var destination_path))
33+
break;
6434

65-
foreach (var kvp in per_arch_assemblies) {
66-
Log.LogDebugMessage ($"Compressing assemblies for architecture '{kvp.Key}'");
35+
if (!assembly.TryGetRequiredMetadata ("AssembliesToCompress", "DescriptorIndex", Log, out var descriptor_index_string))
36+
break;
6737

68-
foreach (var asm in kvp.Value.Values) {
69-
MonoAndroidHelper.LogIfReferenceAssembly (asm, Log);
70-
71-
var compressed_assembly = AssemblyCompression.Compress (Log, asm, compressed_assemblies_info, compressed_output_dir);
72-
73-
if (compressed_assembly.HasValue ()) {
74-
Log.LogDebugMessage ($"Compressed '{asm.ItemSpec}' to '{compressed_assembly}'.");
75-
asm.SetMetadata ("CompressedAssembly", compressed_assembly);
76-
}
38+
if (!uint.TryParse (descriptor_index_string, out var descriptor_index)) {
39+
Log.LogError ($"Failed to parse 'DescriptorIndex' metadata value '{descriptor_index_string}' for assembly '{assembly.ItemSpec}'");
40+
break;
7741
}
42+
43+
if (!AssemblyCompression.TryCompress (Log, assembly.ItemSpec, destination_path, descriptor_index)) {
44+
failed_assemblies.Add (assembly);
45+
continue;
46+
}
47+
48+
Log.LogDebugMessage ($"Compressed '{assembly.ItemSpec}' to '{destination_path}'.");
7849
}
7950

80-
ResolvedFrameworkAssembliesOutput = ResolvedFrameworkAssemblies;
81-
ResolvedUserAssembliesOutput = ResolvedUserAssemblies;
51+
FailedToCompressAssembliesOutput = failed_assemblies.ToArray ();
8252

8353
return !Log.HasLoggedErrors;
8454
}
85-
86-
IDictionary<AndroidTargetArch, Dictionary<string, CompressedAssemblyInfo>> GetCompressedAssemblyInfo ()
87-
{
88-
var key = CompressedAssemblyInfo.GetKey (ProjectFullPath);
89-
Log.LogDebugMessage ($"Retrieving assembly compression info with key '{key}'");
90-
91-
var compressedAssembliesInfo = BuildEngine4.UnregisterTaskObjectAssemblyLocal<IDictionary<AndroidTargetArch, Dictionary<string, CompressedAssemblyInfo>>> (key, RegisteredTaskObjectLifetime.Build);
92-
93-
if (compressedAssembliesInfo is null)
94-
throw new InvalidOperationException ($"Assembly compression info not found for key '{key}'. Compression will not be performed.");
95-
96-
return compressedAssembliesInfo;
97-
}
98-
99-
bool ShouldSkipAssembly (ITaskItem asm)
100-
{
101-
var should_skip = asm.GetMetadataOrDefault ("AndroidSkipAddToPackage", false);
102-
103-
if (should_skip)
104-
Log.LogDebugMessage ($"Skipping {asm.ItemSpec} due to 'AndroidSkipAddToPackage' == 'true' ");
105-
106-
return should_skip;
107-
}
10855
}

‎src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblyCompressionFailures.cs

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#nullable enable
2+
3+
using System;
4+
using System.Linq;
5+
using Microsoft.Android.Build.Tasks;
6+
using Microsoft.Build.Framework;
7+
8+
namespace Xamarin.Android.Tasks;
9+
10+
/// <summary>
11+
/// Remove "CompressedAssembly" metadata from assemblies that failed to compress.
12+
/// </summary>
13+
public class ProcessAssemblyCompressionFailures : AndroidTask
14+
{
15+
public override string TaskPrefix => "PAC";
16+
17+
[Required]
18+
public ITaskItem [] FailedToCompressAssembliesOutput { get; set; } = [];
19+
20+
[Required]
21+
public ITaskItem [] ResolvedFrameworkAssemblies { get; set; } = [];
22+
23+
[Required]
24+
public ITaskItem [] ResolvedUserAssemblies { get; set; } = [];
25+
26+
[Output]
27+
public ITaskItem [] ResolvedFrameworkAssembliesOutput { get; set; } = [];
28+
29+
[Output]
30+
public ITaskItem [] ResolvedUserAssembliesOutput { get; set; } = [];
31+
32+
public override bool RunTask ()
33+
{
34+
// We always need to set the output properties so that future tasks can use them
35+
ResolvedFrameworkAssembliesOutput = ResolvedFrameworkAssemblies;
36+
ResolvedUserAssembliesOutput = ResolvedUserAssemblies;
37+
38+
foreach (var failure in FailedToCompressAssembliesOutput) {
39+
var assembly = ResolvedFrameworkAssemblies.Concat (ResolvedUserAssemblies).Single (a => a.ItemSpec == failure.ItemSpec);
40+
assembly.RemoveMetadata ("CompressedAssembly");
41+
42+
Log.LogDebugMessage ($"Removed 'CompressedAssembly' metadata from '{assembly.ItemSpec}'.");
43+
}
44+
45+
return !Log.HasLoggedErrors;
46+
}
47+
}

‎src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ public class Foo {
6767
Assert.IsTrue (b.Build (proj, doNotCleanupOnUpdate: true), "third build failed");
6868
b.Output.AssertTargetIsNotSkipped ("CoreCompile");
6969
b.Output.AssertTargetIsNotSkipped ("_Sign");
70+
b.Output.AssertTargetIsPartiallyBuilt ("_CompressAssemblies");
7071
}
7172
}
7273

0 commit comments

Comments
 (0)