Skip to content

Commit fa9b99f

Browse files
pjcollinspjcollins
authored
[ci] Improvements for emulator test jobs (#7731)
The following changes attempt to improve the reliability and performance of our MSBuild test jobs. All nightly tests have been updated to run against our .NET build/test artifacts rather than classic XA. The setup execution time for non-device tests has been improved by moving a handful of adb commands from BaseTest to DeviceTest setup. These would run for nearly every test run and result in a lot of waiting due to some RunProcess failures and timeouts. The output from the shell command we run to check if a device is online is now cached and only refreshed when required by certain tests/asserts. The `AssertHasDevices` check has been moved into `DeviceTest` setup, and removed from individual tests in most cases. Attempts to restart the emulator if a test determines that it is inaccessible have been fixed, and emulator data will not be reset in this case. I played around with a handful of emulator launch settings and found that removing the no-boot-anim and headless UI options improved the reliability of nightly launch tests, and resulted in faster boot times locally. These options are now disabled for those tests. The `DeploymentTest` class has been replaced by `TimeZoneInfoTests` and `LocalizationTests`, and the other tests in that class have been moved to `InstallAndRunTests`. `TimeZoneInfoTests` and `LocalizationTests` will now validate command line output rather than trying to press a button and read a UI element. This should make them more reliable and faster. The `TimeZoneInfoTests` and `LocalizationTests` suites have been moved into separate test stages in the nightly test job. An issue that caused LocalizationTests nodes 11 and 12 to not contain any tests has been fixed. An issue that caused LocalizationTests node 1 to also run tests from node 11 and node 12 has been fixed. Test result attachments have been fixed for `TimeZoneInfoTests` and `LocalizationTests`, ensuring that we capture the right logcat and build files for each test variant. An issue causing binlog files to be overwritten by tests that use multiple build targets has been fixed by naming the binlog file after the build log file. Console output verbosity has been set to normal for dotnet test invocations, which should help with debugging. Test NUnit packages have been updated to their latest versions.
1 parent fe00f48 commit fa9b99f

28 files changed

+1086
-900
lines changed

Diff for: ‎build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/StartAndroidEmulator.cs

+4-6
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,10 @@ public class StartAndroidEmulator : Task
3030
public string AvdManagerHome {get; set;}
3131
public string Port {get; set;}
3232
public string ImageName {get; set;} = "XamarinAndroidTestRunner64";
33-
public string Arguments {get; set;}
33+
public string Arguments {get; set;}
3434
public string ToolPath {get; set;}
3535
public string ToolExe {get; set;}
3636
public string LogcatFile {get; set;}
37-
public bool ShowWindow {get; set;} = true;
3837

3938
public override bool Execute ()
4039
{
@@ -69,10 +68,9 @@ void Run (string emulator)
6968
if (emulator == null)
7069
return;
7170

72-
var port = string.IsNullOrEmpty (Port) ? "" : $" -port {Port}";
73-
var showWindow = ShowWindow ? "" : " -no-window";
74-
var arguments = $"{Arguments ?? string.Empty} -verbose -detect-image-hang -logcat-output \"{LogcatFile}\" -no-boot-anim -no-audio -no-snapshot -cache-size 512 -change-locale en-US -timezone \"Etc/UTC\" {showWindow}{port} -avd {ImageName}";
75-
Log.LogMessage (MessageImportance.Low, $"Tool {emulator} execution started with arguments: {arguments}");
71+
var port = string.IsNullOrEmpty (Port) ? "" : $"-port {Port}";
72+
var arguments = $"{Arguments ?? string.Empty} -verbose -detect-image-hang -logcat-output \"{LogcatFile}\" -no-audio -no-snapshot -cache-size 512 -change-locale en-US -timezone \"Etc/UTC\" {port} -avd {ImageName}";
73+
Log.LogMessage ($"Tool {emulator} execution started with arguments: {arguments}");
7674
var psi = new ProcessStartInfo () {
7775
FileName = emulator,
7876
Arguments = arguments,

Diff for: ‎build-tools/automation/azure-pipelines-nightly.yaml

+316-69
Large diffs are not rendered by default.

Diff for: ‎build-tools/automation/yaml-templates/run-systemapp-tests.yaml renamed to ‎build-tools/automation/yaml-templates/run-emulator-tests.yaml

+14-14
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,17 @@
1-
# Runs TimeZoneInfo tests against an emulator running on macOS
1+
# Runs a test or set of tests on an emulator running on macOS
2+
3+
parameters:
4+
emulatorMSBuildArgs: ''
5+
jobName: CheckTimeZoneInfoIsCorrectNode1
6+
jobTimeout: 360
7+
testSteps: []
28

39
jobs:
4-
- job: mac_systemapp_tests
5-
displayName: System App Emulator Tests
10+
- job: mac_${{ parameters.jobName }}_tests
11+
displayName: ${{ parameters.jobName }} Emulator Tests
612
pool:
713
vmImage: $(HostedMacImage)
8-
timeoutInMinutes: 120
9-
cancelTimeoutInMinutes: 5
14+
timeoutInMinutes: ${{ parameters.jobTimeout }}
1015
workspace:
1116
clean: all
1217
steps:
@@ -27,14 +32,9 @@ jobs:
2732
solution: tests/Mono.Android-Tests/Mono.Android-Tests.csproj
2833
configuration: $(XA.Build.Configuration)
2934
msbuildArguments: >-
30-
/t:AcquireAndroidTarget /p:TestEmulatorArguments=-writable-system /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/start-emulator.binlog
35+
/t:AcquireAndroidTarget ${{ parameters.emulatorMSBuildArgs }} /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/start-emulator.binlog
3136
32-
- template: run-nunit-tests.yaml
33-
parameters:
34-
testRunTitle: System App On Device - macOS
35-
testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/net472/MSBuildDeviceIntegration.dll
36-
nunitConsoleExtraArgs: --where "cat == SystemApplication"
37-
testResultsFile: TestResult-SystemApp--$(XA.Build.Configuration).xml
37+
- ${{ parameters.testSteps }}
3838

3939
- task: MSBuild@1
4040
displayName: shut down emulator
@@ -48,6 +48,6 @@ jobs:
4848

4949
- template: upload-results.yaml
5050
parameters:
51-
artifactName: Test Results - System App With Emulator - macOS
51+
artifactName: Test Results - ${{ parameters.jobName }} With Emulator - macOS
5252

53-
- template: fail-on-issue.yaml
53+
- template: fail-on-issue.yaml

Diff for: ‎build-tools/automation/yaml-templates/run-timezoneinfo-tests.yaml

-57
This file was deleted.

Diff for: ‎build-tools/scripts/NUnitReferences.projitems

+1-5
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,6 @@
44
<PackageReference Include="NUnit" Version="3.13.3" />
55
<PackageReference Include="NUnit.ConsoleRunner" Version="$(NUnitConsoleVersion)" />
66
<PackageReference Include="NUnit3TestAdapter" Version="4.3.1" />
7-
</ItemGroup>
8-
<!-- Required packages for .NET Core -->
9-
<ItemGroup Condition=" '$(TargetFramework)' != 'net472' and '$(TargetFramework)' != 'netstandard2.0' ">
10-
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0-preview-20221003-04" />
11-
<PackageReference Include="System.Configuration.ConfigurationManager" Version="6.0.1" />
7+
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />
128
</ItemGroup>
139
</Project>

Diff for: ‎build-tools/scripts/TestApks.targets

+9-6
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,13 @@
2121
<TestAvdAbi Condition=" '$(TestAvdAbi)' == '' and '$(HostOS)' == 'Darwin' and '$(HostOSArchitecture)' == 'Arm64' ">arm64-v8a</TestAvdAbi>
2222
<TestAvdAbi Condition=" '$(TestAvdAbi)' == '' ">x86_64</TestAvdAbi>
2323
<TestAvdType Condition=" '$(TestAvdType)' == '' ">default</TestAvdType>
24+
<TestAvdForceCreation Condition=" '$(TestAvdForceCreation)' == '' ">true</TestAvdForceCreation>
25+
<TestAvdShowWindow Condition=" '$(TestAvdShowWindow)' == '' and '$(RunningOnCI)' == 'true' ">false</TestAvdShowWindow>
26+
<TestAvdExtraBootArgs Condition=" '$(TestAvdShowWindow)' == 'false' ">-no-window -no-boot-anim $(TestAvdExtraBootArgs)</TestAvdExtraBootArgs>
2427
<TestDeviceName Condition=" '$(TestDeviceName)' == '' ">pixel_4</TestDeviceName>
2528
<SdkManagerImageName Condition=" '$(SdkManagerImageName)' == '' ">system-images;android-$(TestAvdApiLevel);$(TestAvdType);$(TestAvdAbi)</SdkManagerImageName>
2629
<TestAvdName>XamarinAndroidTestRunner$(TestAvdApiLevel)-$(TestAvdAbi)</TestAvdName>
2730
<_AdbEmulatorPort>5570</_AdbEmulatorPort>
28-
<_AdbEmulatorShowWindow Condition=" '$(RunningOnCI)' == 'True' And '$(_AdbEmulatorShowWindow)' == '' ">False</_AdbEmulatorShowWindow>
29-
<_AdbEmulatorShowWindow Condition=" '$(_AdbEmulatorShowWindow)' == '' ">True</_AdbEmulatorShowWindow>
3031
<_ApkSizesReferenceDirectory>$(MSBuildThisFileDirectory)..\..\tests\apk-sizes-reference</_ApkSizesReferenceDirectory>
3132
<AvdLaunchTimeoutMinutes Condition=" '$(AvdLaunchTimeoutMinutes)' == '' ">10</AvdLaunchTimeoutMinutes>
3233
<AvdLaunchTimeoutSeconds>$([MSBuild]::Multiply($(AvdLaunchTimeoutMinutes), 60))</AvdLaunchTimeoutSeconds>
@@ -56,7 +57,7 @@
5657
<Output TaskParameter="IsValidTarget" PropertyName="_ValidAdbTarget" />
5758
</Xamarin.Android.Tools.BootstrapTasks.CheckAdbTarget>
5859
<CreateAndroidEmulator
59-
Condition=" '$(_ValidAdbTarget)' != 'True' "
60+
Condition=" '$(_ValidAdbTarget)' != 'True' and ( '$(TestAvdForceCreation)' == 'true' or !Exists('$(AvdManagerHome)\.android\avd\$(TestAvdName).avd') ) "
6061
AndroidAbi="$(TestAvdAbi)"
6162
AvdManagerHome="$(AvdManagerHome)"
6263
JavaSdkHome="$(JavaSdkDirectory)"
@@ -74,11 +75,10 @@
7475
Condition=" '$(_ValidAdbTarget)' != 'True' "
7576
AndroidSdkDirectory="$(AndroidSdkDirectory)"
7677
AvdManagerHome="$(AvdManagerHome)"
77-
Arguments="$(TestEmulatorArguments)"
78+
Arguments="$(TestAvdExtraBootArgs)"
7879
ImageName="$(TestAvdName)"
7980
LogcatFile="$(_LogcatFilenameBase)-full.txt"
8081
Port="$(_AdbEmulatorPort)"
81-
ShowWindow="$(_AdbEmulatorShowWindow)"
8282
ToolExe="$(EmulatorToolExe)"
8383
ToolPath="$(EmulatorToolPath)">
8484
<Output TaskParameter="AdbTarget" PropertyName="_AdbTarget" />
@@ -124,6 +124,9 @@
124124
</Target>
125125

126126
<Target Name="ReleaseAndroidTarget">
127+
<PropertyGroup>
128+
<_EmuTarget Condition=" '$(_EmuTarget)' == '' ">-s emulator-$(_AdbEmulatorPort)</_EmuTarget>
129+
</PropertyGroup>
127130
<Xamarin.Android.Tools.BootstrapTasks.Adb
128131
Condition="'@(_FailedComponent)' != ''"
129132
ContinueOnError="ErrorAndContinue"
@@ -141,7 +144,7 @@
141144
Timeout="60000"
142145
/>
143146
<KillProcess
144-
Condition=" '$(_EmuTarget)' != '' "
147+
Condition=" '$(_EmuPid)' != '' "
145148
ContinueOnError="ErrorAndContinue"
146149
ProcessId="$(_EmuPid)"
147150
/>

Diff for: ‎src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/BaseTest.cs

+3-48
Original file line numberDiff line numberDiff line change
@@ -25,16 +25,6 @@ public class BaseTest
2525
[SetUpFixture]
2626
public class SetUp
2727
{
28-
public static string DeviceAbi {
29-
get;
30-
private set;
31-
}
32-
33-
public static int DeviceSdkVersion {
34-
get;
35-
private set;
36-
}
37-
3828
public static string TestDirectoryRoot {
3929
get;
4030
private set;
@@ -44,41 +34,6 @@ public static string TestDirectoryRoot {
4434
public void BeforeAllTests ()
4535
{
4636
TestDirectoryRoot = XABuildPaths.TestOutputDirectory;
47-
48-
try {
49-
DeviceSdkVersion = GetSdkVersion ();
50-
if (DeviceSdkVersion != -1) {
51-
if (DeviceSdkVersion >= 21)
52-
DeviceAbi = RunAdbCommand ("shell getprop ro.product.cpu.abilist64").Trim ();
53-
54-
if (string.IsNullOrEmpty (DeviceAbi))
55-
DeviceAbi = RunAdbCommand ("shell getprop ro.product.cpu.abi") ?? RunAdbCommand ("shell getprop ro.product.cpu.abi2");
56-
57-
if (DeviceAbi.Contains (",")) {
58-
DeviceAbi = DeviceAbi.Split (',')[0];
59-
}
60-
}
61-
} catch (Exception ex) {
62-
Console.Error.WriteLine ("Failed to determine whether there is Android target emulator or not: " + ex);
63-
}
64-
}
65-
66-
int GetSdkVersion ()
67-
{
68-
var command = $"shell getprop ro.build.version.sdk";
69-
var result = RunAdbCommand (command);
70-
if (result.Contains ("*")) {
71-
// Run the command again, we likely got:
72-
// * daemon not running; starting now at tcp:5037
73-
// * daemon started successfully
74-
// adb.exe: device offline
75-
TestContext.WriteLine ($"Retrying:\n{command}\n{result}");
76-
result = RunAdbCommand (command);
77-
}
78-
if (!int.TryParse (result, out var sdkVersion)) {
79-
sdkVersion = -1;
80-
}
81-
return sdkVersion;
8237
}
8338

8439
[OneTimeTearDown]
@@ -208,7 +163,7 @@ protected static string RunAdbCommand (string command, bool ignoreErrors = true,
208163
string ext = Environment.OSVersion.Platform != PlatformID.Unix ? ".exe" : "";
209164
string adb = Path.Combine (AndroidSdkPath, "platform-tools", "adb" + ext);
210165
string adbTarget = Environment.GetEnvironmentVariable ("ADB_TARGET");
211-
return RunProcess (adb, $"{adbTarget} {command}");
166+
return RunProcess (adb, $"{adbTarget} {command}", timeout);
212167
}
213168

214169
protected static (int code, string stdOutput, string stdError) RunApkDiffCommand (string args)
@@ -225,9 +180,9 @@ protected static (int code, string stdOutput, string stdError) RunApkDiffCommand
225180
}
226181
}
227182

228-
protected static string RunProcess (string exe, string args)
183+
protected static string RunProcess (string exe, string args, int timeoutInSeconds = 30)
229184
{
230-
var (_, stdOutput, stdError) = RunProcessWithExitCode (exe, args);
185+
var (_, stdOutput, stdError) = RunProcessWithExitCode (exe, args, timeoutInSeconds);
231186

232187
return stdOutput + stdError;
233188
}

0 commit comments

Comments
 (0)