Skip to content

Commit 928e7b9

Browse files
jpobstjpobst
authored
[ci] Parallelize and reduce overhead of MSBuild test stage. (#7850)
Bring the AzDO parallelization from #7804 and environment setup improvements from #7832 to `Xamarin.Android.Build.Tests.csproj` based test suites. This includes both the main `MSBuild` stage and the `Smoke Tests` jobs. Increases parallelization of all jobs as many were approaching ~90 minutes. As there is no longer a place in the `MSBuild` stage to run `Xamarin.Android.Tools.Aidl-Tests` tests, it was moved to the `macOS > Tests > APKs .NET` stage. This suite should be fine to only run on Mac and not Windows. (Note this test assembly was also updated to .NET 7. This required moving it from `Xamarin.Android-Tests.sln` which is currently built with Mono which cannot build .NET 7+ projects. It now is built via `Xamarin.Android.sln`.)
1 parent 03fff8c commit 928e7b9

File tree

60 files changed

+160
-342
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

60 files changed

+160
-342
lines changed

‎Xamarin.Android-Tests.sln

-14
Original file line numberDiff line numberDiff line change
@@ -86,10 +86,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Java.Interop-Tests", "tests
8686
EndProject
8787
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Java.Interop", "external\Java.Interop\src\Java.Interop\Java.Interop.csproj", "{94BD81F7-B06F-4295-9636-F8A3B6BDC762}"
8888
EndProject
89-
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Xamarin.Android.Tools.Aidl-Tests", "tests\Xamarin.Android.Tools.Aidl-Tests\Xamarin.Android.Tools.Aidl-Tests.csproj", "{883941C8-C4ED-428D-A7D2-26F2EEA23F92}"
90-
EndProject
91-
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Xamarin.Android.Tools.Aidl", "src\Xamarin.Android.Tools.Aidl\Xamarin.Android.Tools.Aidl.csproj", "{302D9D2E-1F98-4374-9B6B-922F78620C4B}"
92-
EndProject
9389
Global
9490
GlobalSection(SharedMSBuildProjectFiles) = preSolution
9591
tests\Mono.Android-Tests\Mono.Android-Test.Shared.projitems*{0ab4956e-6fb9-4da0-9d49-ab65a3ff403a}*SharedItemsImports = 13
@@ -237,14 +233,6 @@ Global
237233
{94BD81F7-B06F-4295-9636-F8A3B6BDC762}.Debug|Any CPU.Build.0 = Debug|Any CPU
238234
{94BD81F7-B06F-4295-9636-F8A3B6BDC762}.Release|Any CPU.ActiveCfg = Release|Any CPU
239235
{94BD81F7-B06F-4295-9636-F8A3B6BDC762}.Release|Any CPU.Build.0 = Release|Any CPU
240-
{883941C8-C4ED-428D-A7D2-26F2EEA23F92}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
241-
{883941C8-C4ED-428D-A7D2-26F2EEA23F92}.Debug|Any CPU.Build.0 = Debug|Any CPU
242-
{883941C8-C4ED-428D-A7D2-26F2EEA23F92}.Release|Any CPU.ActiveCfg = Release|Any CPU
243-
{883941C8-C4ED-428D-A7D2-26F2EEA23F92}.Release|Any CPU.Build.0 = Release|Any CPU
244-
{302D9D2E-1F98-4374-9B6B-922F78620C4B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
245-
{302D9D2E-1F98-4374-9B6B-922F78620C4B}.Debug|Any CPU.Build.0 = Debug|Any CPU
246-
{302D9D2E-1F98-4374-9B6B-922F78620C4B}.Release|Any CPU.ActiveCfg = Release|Any CPU
247-
{302D9D2E-1F98-4374-9B6B-922F78620C4B}.Release|Any CPU.Build.0 = Release|Any CPU
248236
EndGlobalSection
249237
GlobalSection(SolutionProperties) = preSolution
250238
HideSolutionNode = FALSE
@@ -281,8 +269,6 @@ Global
281269
{D1243BAB-23CA-4566-A2A3-3ADA2C2DC3AF} = {EFBC4DC0-DBFF-4DAA-B0B8-6D0CB02A25F5}
282270
{6CB00820-A66B-43E5-8785-ED456C6E9F39} = {EFBC4DC0-DBFF-4DAA-B0B8-6D0CB02A25F5}
283271
{94BD81F7-B06F-4295-9636-F8A3B6BDC762} = {EFBC4DC0-DBFF-4DAA-B0B8-6D0CB02A25F5}
284-
{883941C8-C4ED-428D-A7D2-26F2EEA23F92} = {EFBC4DC0-DBFF-4DAA-B0B8-6D0CB02A25F5}
285-
{302D9D2E-1F98-4374-9B6B-922F78620C4B} = {EFBC4DC0-DBFF-4DAA-B0B8-6D0CB02A25F5}
286272
EndGlobalSection
287273
GlobalSection(ExtensibilityGlobals) = postSolution
288274
SolutionGuid = {8643CD20-B195-4919-8135-27549488237E}

‎Xamarin.Android.sln

+7
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Java.Runtime.Environment",
156156
EndProject
157157
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "create-android-api", "build-tools\create-android-api\create-android-api.csproj", "{BA4D889D-066B-4C2C-A973-09E319CBC396}"
158158
EndProject
159+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Xamarin.Android.Tools.Aidl-Tests", "tests\Xamarin.Android.Tools.Aidl-Tests\Xamarin.Android.Tools.Aidl-Tests.csproj", "{A39B6D7C-6616-40D6-8AE4-C6CEE93D2708}"
160+
EndProject
159161
Global
160162
GlobalSection(SolutionConfigurationPlatforms) = preSolution
161163
Debug|AnyCPU = Debug|AnyCPU
@@ -424,6 +426,10 @@ Global
424426
{BA4D889D-066B-4C2C-A973-09E319CBC396}.Debug|AnyCPU.Build.0 = Debug|Any CPU
425427
{BA4D889D-066B-4C2C-A973-09E319CBC396}.Release|AnyCPU.ActiveCfg = Release|Any CPU
426428
{BA4D889D-066B-4C2C-A973-09E319CBC396}.Release|AnyCPU.Build.0 = Release|Any CPU
429+
{A39B6D7C-6616-40D6-8AE4-C6CEE93D2708}.Debug|AnyCPU.ActiveCfg = Debug|Any CPU
430+
{A39B6D7C-6616-40D6-8AE4-C6CEE93D2708}.Debug|AnyCPU.Build.0 = Debug|Any CPU
431+
{A39B6D7C-6616-40D6-8AE4-C6CEE93D2708}.Release|AnyCPU.ActiveCfg = Release|Any CPU
432+
{A39B6D7C-6616-40D6-8AE4-C6CEE93D2708}.Release|AnyCPU.Build.0 = Release|Any CPU
427433
EndGlobalSection
428434
GlobalSection(SolutionProperties) = preSolution
429435
HideSolutionNode = FALSE
@@ -495,6 +501,7 @@ Global
495501
{D8E14B43-E929-4C18-9FA6-2C3DC47EFC17} = {864062D3-A415-4A6F-9324-5820237BA058}
496502
{C0E44558-FEE3-4DD3-986A-3F46DD1BF41B} = {04E3E11E-B47D-4599-8AFC-50515A95E715}
497503
{BA4D889D-066B-4C2C-A973-09E319CBC396} = {E351F97D-EA4F-4E7F-AAA0-8EBB1F2A4A62}
504+
{A39B6D7C-6616-40D6-8AE4-C6CEE93D2708} = {CAB438D8-B0F5-4AF0-BEBD-9E2ADBD7B483}
498505
EndGlobalSection
499506
GlobalSection(ExtensibilityGlobals) = postSolution
500507
SolutionGuid = {53A1F287-EFB2-4D97-A4BB-4A5E145613F6}

‎build-tools/automation/azure-pipelines.yaml

+35-109
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,8 @@ variables:
5555
value: $[or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['System.PullRequest.TargetBranch'], 'release/'))]
5656
- name: DotNetNUnitCategories
5757
value: '& TestCategory != DotNetIgnore & TestCategory != HybridAOT & TestCategory != MkBundle & TestCategory != MonoSymbolicate & TestCategory != PackagesConfig & TestCategory != StaticProject & TestCategory != SystemApplication'
58+
- name: ExcludedNUnitCategories
59+
value: '& cat != DotNetIgnore & cat != HybridAOT & cat != MkBundle & cat != MonoSymbolicate & cat != PackagesConfig & cat != StaticProject & cat != SystemApplication'
5860
- ${{ if eq(variables['Build.DefinitionName'], 'Xamarin.Android-Private') }}:
5961
- group: AzureDevOps-Artifact-Feeds-Pats
6062
- group: DotNet-MSRC-Storage
@@ -315,6 +317,11 @@ stages:
315317
parameters:
316318
displayName: install emulator
317319
arguments: --s=EmulatorTestDependencies
320+
321+
- task: DownloadPipelineArtifact@2
322+
inputs:
323+
artifactName: $(TestAssembliesArtifactName)
324+
downloadPath: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)
318325

319326
# Set up dependencies to run tests in both debug and release configurations
320327
- task: DotNetCoreCLI@2
@@ -398,6 +405,12 @@ stages:
398405
artifactSource: bin/Test$(XA.Build.Configuration)/$(DotNetTargetFramework)-android/Mono.Android.NET_Tests-Signed.aab
399406
artifactFolder: $(DotNetTargetFramework)-AotLlvm
400407

408+
- template: yaml-templates/run-nunit-tests.yaml
409+
parameters:
410+
testRunTitle: Xamarin.Android.Tools.Aidl-Tests - macOS
411+
testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/$(DotNetStableTargetFramework)/Xamarin.Android.Tools.Aidl-Tests.dll
412+
testResultsFile: TestResult-Aidl-Tests-macOS-$(XA.Build.Configuration).xml
413+
401414
- task: MSBuild@1
402415
displayName: shut down emulator
403416
inputs:
@@ -440,23 +453,25 @@ stages:
440453

441454
- template: yaml-templates/fail-on-issue.yaml
442455

443-
# Xamarin.Android (Smoke Tests MSBuild - Mac-0)
444-
- template: yaml-templates/run-msbuild-mac-tests.yaml
456+
# MSBuild Smoke Tests - Mac
457+
- template: yaml-templates/run-msbuild-tests.yaml
445458
parameters:
446-
job_name: mac_dotnet_tests_0
447-
job_suffix: One .NET
448-
nunit_categories: '| (TestCategory = SmokeTests $(DotNetNUnitCategories))'
449-
target_framework: $(DotNetStableTargetFramework)
450-
provisionatorChannel: ${{ parameters.provisionatorChannel }}
451-
452-
# Xamarin.Android (Smoke Tests MSBuild - Win-0)
453-
- template: yaml-templates\run-msbuild-win-tests.yaml
459+
testOS: macOS
460+
jobName: mac_smoke_msbuild_tests
461+
jobDisplayName: macOS > Tests > MSBuild
462+
agentCount: 2
463+
testFilter: cat = SmokeTests
464+
installApkDiff: true
465+
466+
# MSBuild Smoke Tests - Win
467+
- template: yaml-templates/run-msbuild-tests.yaml
454468
parameters:
455-
job_name: win_dotnet_tests_0
456-
job_suffix: One .NET
457-
nunit_categories: '| (TestCategory = SmokeTests $(DotNetNUnitCategories))'
458-
target_framework: $(DotNetStableTargetFramework)
459-
provisionatorChannel: ${{ parameters.provisionatorChannel }}
469+
testOS: Windows
470+
jobName: win_smoke_msbuild_tests
471+
jobDisplayName: Windows > Tests > MSBuild
472+
agentCount: 2
473+
testFilter: cat = SmokeTests
474+
installApkDiff: true
460475

461476
# Check - "Xamarin.Android (macOS > Tests > MSBuild+Emulator)"
462477
- job: mac_msbuilddevice_tests
@@ -550,103 +565,14 @@ stages:
550565
artifactName: Test Results - MSBuild Smoke - Linux
551566

552567
- template: yaml-templates/fail-on-issue.yaml
553-
554-
- stage: msbuild_dotnet
555-
displayName: One .NET Tests
556-
dependsOn: mac_build
557-
condition: and(succeeded(), or(eq(variables['RunAllTests'], true), contains(dependencies.mac_build.outputs['mac_build_create_installers.TestConditions.TestAreas'], 'MSBuild')))
558-
jobs:
559-
# Xamarin.Android (Test MSBuild One .NET - macOS)
560-
- template: yaml-templates\run-msbuild-mac-tests.yaml
561-
parameters:
562-
node_id: 1
563-
job_name: mac_dotnet_tests_1
564-
job_suffix: One .NET
565-
nunit_categories: $(DotNetNUnitCategories) & TestCategory != SmokeTests
566-
target_framework: $(DotNetStableTargetFramework)
567-
provisionatorChannel: ${{ parameters.provisionatorChannel }}
568-
569-
- template: yaml-templates\run-msbuild-mac-tests.yaml
570-
parameters:
571-
node_id: 2
572-
job_name: mac_dotnet_tests_2
573-
job_suffix: One .NET
574-
nunit_categories: $(DotNetNUnitCategories) & TestCategory != SmokeTests
575-
target_framework: $(DotNetStableTargetFramework)
576-
provisionatorChannel: ${{ parameters.provisionatorChannel }}
577-
578-
- template: yaml-templates\run-msbuild-mac-tests.yaml
579-
parameters:
580-
node_id: 3
581-
job_name: mac_dotnet_tests_3
582-
job_suffix: One .NET
583-
nunit_categories: $(DotNetNUnitCategories) & TestCategory != SmokeTests
584-
target_framework: $(DotNetStableTargetFramework)
585-
provisionatorChannel: ${{ parameters.provisionatorChannel }}
586-
587-
- template: yaml-templates\run-msbuild-mac-tests.yaml
588-
parameters:
589-
node_id: 4
590-
job_name: mac_dotnet_tests_4
591-
job_suffix: One .NET
592-
nunit_categories: $(DotNetNUnitCategories) & TestCategory != SmokeTests
593-
target_framework: $(DotNetStableTargetFramework)
594-
provisionatorChannel: ${{ parameters.provisionatorChannel }}
595-
596-
- template: yaml-templates\run-msbuild-mac-tests.yaml
597-
parameters:
598-
node_id: 5
599-
job_name: mac_dotnet_tests_5
600-
job_suffix: One .NET
601-
nunit_categories: $(DotNetNUnitCategories) & TestCategory != SmokeTests
602-
target_framework: $(DotNetStableTargetFramework)
603-
provisionatorChannel: ${{ parameters.provisionatorChannel }}
604-
605-
- template: yaml-templates\run-msbuild-mac-tests.yaml
606-
parameters:
607-
node_id: 6
608-
job_name: mac_dotnet_tests_6
609-
job_suffix: One .NET
610-
nunit_categories: $(DotNetNUnitCategories) & TestCategory != SmokeTests
611-
target_framework: $(DotNetStableTargetFramework)
612-
provisionatorChannel: ${{ parameters.provisionatorChannel }}
613-
614-
# Xamarin.Android (Test MSBuild One .NET - Windows)
615-
- template: yaml-templates\run-msbuild-win-tests.yaml
616-
parameters:
617-
node_id: 1
618-
additional_node_id: 4
619-
job_name: win_dotnet_tests_1
620-
job_suffix: One .NET
621-
nunit_categories: $(DotNetNUnitCategories) & TestCategory != SmokeTests
622-
target_framework: $(DotNetStableTargetFramework)
623-
provisionatorChannel: ${{ parameters.provisionatorChannel }}
624-
625-
- template: yaml-templates\run-msbuild-win-tests.yaml
626-
parameters:
627-
node_id: 2
628-
additional_node_id: 5
629-
job_name: win_dotnet_tests_2
630-
job_suffix: One .NET
631-
nunit_categories: $(DotNetNUnitCategories) & TestCategory != SmokeTests
632-
target_framework: $(DotNetStableTargetFramework)
633-
provisionatorChannel: ${{ parameters.provisionatorChannel }}
634-
635-
- template: yaml-templates\run-msbuild-win-tests.yaml
636-
parameters:
637-
node_id: 3
638-
additional_node_id: 6
639-
job_name: win_dotnet_tests_3
640-
job_suffix: One .NET
641-
nunit_categories: $(DotNetNUnitCategories) & TestCategory != SmokeTests
642-
target_framework: $(DotNetStableTargetFramework)
643-
provisionatorChannel: ${{ parameters.provisionatorChannel }}
644-
568+
569+
- template: yaml-templates/stage-msbuild-tests.yaml
570+
parameters:
571+
stageCondition: and(succeeded(), or(eq(variables['RunAllTests'], true), contains(dependencies.mac_build.outputs['mac_build_create_installers.TestConditions.TestAreas'], 'MSBuild')))
572+
645573
- template: yaml-templates/stage-msbuild-emulator-tests.yaml
646574
parameters:
647-
provisionatorChannel: ${{ parameters.provisionatorChannel }}
648575
stageCondition: and(succeeded(), or(eq(variables['RunAllTests'], true), contains(dependencies.mac_build.outputs['mac_build_create_installers.TestConditions.TestAreas'], 'MSBuildDevice')))
649-
nunit_categories: '& cat != DotNetIgnore & cat != HybridAOT & cat != MkBundle & cat != MonoSymbolicate & cat != PackagesConfig & cat != StaticProject & cat != SystemApplication'
650576

651577
- stage: bcl_tests
652578
displayName: BCL Emulator Tests

‎build-tools/automation/yaml-templates/install-dotnet-test-slicer.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
parameters:
2-
version: '0.1.0-alpha1'
2+
version: '0.1.0-alpha2'
33
condition: succeeded()
44
continueOnError: true
55

‎build-tools/automation/yaml-templates/run-msbuild-mac-tests.yaml

-66
This file was deleted.

‎build-tools/automation/yaml-templates/run-msbuild-tests.yaml

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
parameters:
2+
testOS: # 'macOS' or 'Windows'
3+
jobName: # Name of the job
4+
jobDisplayName: # Display name of the job
5+
agentCount: # Number of build agents to run in parallel
6+
testFilter: # Filter used to select tests (NUnit test selection language, not dotnet test filter language)
7+
installApkDiff: false # Smoke tests use apkdiff, others do not
8+
9+
jobs:
10+
- job: ${{ parameters.jobName }}
11+
strategy:
12+
parallel: ${{ parameters.agentCount }}
13+
displayName: ${{ parameters.jobDisplayName }}
14+
${{ if eq(parameters.testOS, 'Windows') }}:
15+
pool: $(1ESWindowsPool)
16+
${{ if eq(parameters.testOS, 'macOS') }}:
17+
pool:
18+
vmImage: $(HostedMacImage)
19+
timeoutInMinutes: 180
20+
cancelTimeoutInMinutes: 5
21+
steps:
22+
- ${{ if eq(parameters.testOS, 'Windows') }}:
23+
- script: netsh int ipv4 set global sourceroutingbehavior=drop
24+
25+
- template: kill-processes.yaml
26+
27+
- template: clean.yaml
28+
29+
- template: setup-test-environment.yaml
30+
parameters:
31+
provisionClassic: false
32+
installTestSlicer: true
33+
installApkDiff: ${{ parameters.installApkDiff }}
34+
installLegacyDotNet: false
35+
restoreNUnitConsole: false
36+
updateMono: false
37+
38+
- task: DownloadPipelineArtifact@2
39+
inputs:
40+
artifactName: $(TestAssembliesArtifactName)
41+
downloadPath: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)
42+
43+
- template: run-sliced-nunit-tests.yaml
44+
parameters:
45+
testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/$(DotNetStableTargetFramework)/Xamarin.Android.Build.Tests.dll
46+
testFilter: ${{ parameters.testFilter }} $(ExcludedNUnitCategories)
47+
testRunTitle: Xamarin.Android.Build.Tests - ${{ parameters.testOS }}
48+
testResultsTitle: TestResult-MSBuildTests-${{ parameters.jobName }}
49+
50+
- template: upload-results.yaml
51+
parameters:
52+
artifactName: Test Results - MSBuild - ${{ parameters.testOS }}-$(System.JobPositionInPhase)
53+
54+
- template: fail-on-issue.yaml

0 commit comments

Comments
 (0)