run.py

#!/usr/bin/env vpython
# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Run a test.

Sample usage:
  ./run.py \
  -a src/xcodebuild/Release-iphoneos/base_unittests.app \
  -o /tmp/out \
  -p iPhone 5s \
  -v 9.3 \
  -b 9b46

  Installs base_unittests.app in an iPhone 5s simulator running iOS 9.3 under
  Xcode build version 9b46, runs it, and captures all test data in /tmp/out.
  """

import argparse
import json
import logging
import os
import subprocess
import sys
import traceback

import shard_util
import test_runner
import wpr_runner
import xcodebuild_runner
import xcode_util as xcode


class Runner():
  """
  Object to encapsulate iOS test runner execution coordination. Parses
  arguments and invokes underlying test runners accordingly.
  """

  def __init__(self, args=None):
    """
    args = argparse Namespace object.
    test_args = string list of args.
    """
    self.args = argparse.Namespace()
    self.test_args = []

    if args:
      self.parse_args(args)

  def install_xcode(self):
    """Installs the requested Xcode build version.

    Returns:
      (bool, bool)
        First bool: True if installation was successful. False otherwise.
        Second bool: True if Xcode is legacy package. False if it's new.
    """
    try:
      if not self.args.mac_toolchain_cmd:
        raise test_runner.MacToolchainNotFoundError(self.args.mac_toolchain_cmd)
      # Guard against incorrect install paths. On swarming, this path
      # should be a requested named cache, and it must exist.
      if not os.path.exists(self.args.xcode_path):
        raise test_runner.XcodePathNotFoundError(self.args.xcode_path)

      runtime_cache_folder = None
      # Runner script only utilizes runtime cache when it's a simulator task.
      if self.args.version:
        runtime_cache_folder = xcode.construct_runtime_cache_folder(
            self.args.runtime_cache_prefix, self.args.version)
        if not os.path.exists(runtime_cache_folder):
          # Depending on infra project, runtime named cache might not be
          # deployed. Create the dir if it doesn't exist since xcode_util
          # assumes it exists.
          # TODO(crbug.com/1191260): Raise error instead of creating dirs after
          # runtime named cache is deployed everywhere.
          os.makedirs(runtime_cache_folder)
      # xcode.install() installs the Xcode & iOS runtime, and returns a bool
      # indicating if the Xcode version in CIPD is a legacy Xcode package (which
      # includes iOS runtimes).
      is_legacy_xcode = xcode.install(
          self.args.mac_toolchain_cmd,
          self.args.xcode_build_version,
          self.args.xcode_path,
          runtime_cache_folder=runtime_cache_folder,
          ios_version=self.args.version)
      xcode.select(self.args.xcode_path)
    except subprocess.CalledProcessError as e:
      # Flush buffers to ensure correct output ordering.
      sys.stdout.flush()
      sys.stderr.write('Xcode build version %s failed to install: %s\n' %
                       (self.args.xcode_build_version, e))
      sys.stderr.flush()
      return (False, False)
    else:
      return (True, is_legacy_xcode)

  def run(self, args):
    """
    Main coordinating function.
    """
    self.parse_args(args)

    # This logic is run by default before the otool command is invoked such that
    # otool has the correct Xcode selected for command line dev tools.
    install_success, is_legacy_xcode = self.install_xcode()
    if not install_success:
      raise test_runner.XcodeVersionNotFoundError(self.args.xcode_build_version)

    # GTEST_SHARD_INDEX and GTEST_TOTAL_SHARDS are additional test environment
    # variables, set by Swarming, that are only set for a swarming task
    # shard count is > 1.
    #
    # For a given test on a given run, otool should return the same total
    # counts and thus, should generate the same sublists. With the shard index,
    # each shard would then know the exact test case to run.
    gtest_shard_index = os.getenv('GTEST_SHARD_INDEX', 0)
    gtest_total_shards = os.getenv('GTEST_TOTAL_SHARDS', 0)
    if gtest_shard_index and gtest_total_shards:
      self.args.test_cases = shard_util.shard_test_cases(
          self.args, gtest_shard_index, gtest_total_shards)

    summary = {}
    tr = None

    if not os.path.exists(self.args.out_dir):
      os.makedirs(self.args.out_dir)

    try:
      if self.args.xcode_parallelization:
        tr = xcodebuild_runner.SimulatorParallelTestRunner(
            self.args.app,
            self.args.host_app,
            self.args.iossim,
            self.args.version,
            self.args.platform,
            out_dir=self.args.out_dir,
            release=self.args.release,
            retries=self.args.retries,
            shards=self.args.shards,
            test_cases=self.args.test_cases,
            test_args=self.test_args,
            use_clang_coverage=self.args.use_clang_coverage,
            env_vars=self.args.env_var)
      elif self.args.replay_path != 'NO_PATH':
        tr = wpr_runner.WprProxySimulatorTestRunner(
            self.args.app,
            self.args.host_app,
            self.args.iossim,
            self.args.replay_path,
            self.args.platform,
            self.args.version,
            self.args.wpr_tools_path,
            self.args.out_dir,
            env_vars=self.args.env_var,
            retries=self.args.retries,
            shards=self.args.shards,
            test_args=self.test_args,
            test_cases=self.args.test_cases,
            xctest=self.args.xctest,
        )
      elif self.args.iossim and self.args.platform and self.args.version:
        tr = test_runner.SimulatorTestRunner(
            self.args.app,
            self.args.iossim,
            self.args.platform,
            self.args.version,
            self.args.out_dir,
            env_vars=self.args.env_var,
            retries=self.args.retries,
            shards=self.args.shards,
            test_args=self.test_args,
            test_cases=self.args.test_cases,
            use_clang_coverage=self.args.use_clang_coverage,
            wpr_tools_path=self.args.wpr_tools_path,
            xctest=self.args.xctest,
        )
      elif self.args.xcodebuild_device_runner and self.args.xctest:
        tr = xcodebuild_runner.DeviceXcodeTestRunner(
            app_path=self.args.app,
            host_app_path=self.args.host_app,
            out_dir=self.args.out_dir,
            release=self.args.release,
            retries=self.args.retries,
            test_cases=self.args.test_cases,
            test_args=self.test_args,
            env_vars=self.args.env_var)
      else:
        tr = test_runner.DeviceTestRunner(
            self.args.app,
            self.args.out_dir,
            env_vars=self.args.env_var,
            restart=self.args.restart,
            retries=self.args.retries,
            test_args=self.test_args,
            test_cases=self.args.test_cases,
            xctest=self.args.xctest,
        )

      logging.info("Using test runner %s" % type(tr).__name__)
      return 0 if tr.launch() else 1
    except test_runner.DeviceError as e:
      sys.stderr.write(traceback.format_exc())
      summary['step_text'] = '%s%s' % (e.__class__.__name__,
                                       ': %s' % e.args[0] if e.args else '')

      # Swarming infra marks device status unavailable for any device related
      # issue using this return code.
      return 3
    except test_runner.TestRunnerError as e:
      sys.stderr.write(traceback.format_exc())
      summary['step_text'] = '%s%s' % (e.__class__.__name__,
                                       ': %s' % e.args[0] if e.args else '')

      # test_runner.Launch returns 0 on success, 1 on failure, so return 2
      # on exception to distinguish between a test failure, and a failure
      # to launch the test at all.
      return 2
    finally:
      if tr:
        summary['logs'] = tr.logs

      with open(os.path.join(self.args.out_dir, 'summary.json'), 'w') as f:
        json.dump(summary, f)
      if tr:
        with open(os.path.join(self.args.out_dir, 'full_results.json'),
                  'w') as f:
          json.dump(tr.test_results, f)

        # The value of test-launcher-summary-output is set by the recipe
        # and passed here via swarming.py. This argument defaults to
        # ${ISOLATED_OUTDIR}/output.json. out-dir is set to ${ISOLATED_OUTDIR}

        # TODO(crbug.com/1031338) - the content of this output.json will
        # work with Chromium recipe because we use the noop_merge merge script,
        # but will require structural changes to support the default gtest
        # merge script (ref: //testing/merge_scripts/standard_gtest_merge.py)
        output_json_path = (
            self.args.test_launcher_summary_output or
            os.path.join(self.args.out_dir, 'output.json'))
        with open(output_json_path, 'w') as f:
          json.dump(tr.test_results, f)

      # Move the iOS runtime back to cache dir if the Xcode package is not
      # legacy (i.e. Xcode program & runtimes are in different CIPD packages.)
      # and it's a simulator task.
      if not is_legacy_xcode and self.args.version:
        runtime_cache_folder = xcode.construct_runtime_cache_folder(
            self.args.runtime_cache_prefix, self.args.version)
        xcode.move_runtime(runtime_cache_folder, self.args.xcode_path, False)

      test_runner.defaults_delete('com.apple.CoreSimulator',
                                  'FramebufferServerRendererPolicy')

  def parse_args(self, args):
    """
    Parse the args into args and test_args.
    """
    parser = argparse.ArgumentParser()

    parser.add_argument(
        '-x',
        '--xcode-parallelization',
        help='Run tests using xcodebuild\'s parallelization.',
        action='store_true',
    )
    parser.add_argument(
        '-a',
        '--app',
        help='Compiled .app to run for EG1, Compiled -Runner.app for EG2',
        metavar='app',
    )
    parser.add_argument(
        '-b',
        '--xcode-build-version',
        help='Xcode build version to install.',
        required=True,
        metavar='build_id',
    )
    parser.add_argument(
        '-e',
        '--env-var',
        action='append',
        help='Environment variable to pass to the test itself.',
        metavar='ENV=val',
    )
    parser.add_argument(
        '--host-app',
        help='Compiled host .app to run.',
        default='NO_PATH',
        metavar='host_app',
    )
    parser.add_argument(
        '-i',
        '--iossim',
        help='Compiled iossim to run the app on.',
        metavar='iossim',
    )
    parser.add_argument(
        '-j',
        '--args-json',
        default='{}',
        help=
        'Specify "env_var": [...] and "test_args": [...] using a JSON dict.',
        metavar='{}',
    )
    parser.add_argument(
        '--mac-toolchain-cmd',
        help='Command to run mac_toolchain tool. Default: %(default)s.',
        default='mac_toolchain',
        metavar='mac_toolchain',
    )
    parser.add_argument(
        '-o',
        '--out-dir',
        help='Directory to store all test data in.',
        metavar='dir',
        required=True,
    )
    parser.add_argument(
        '-p',
        '--platform',
        help='Platform to simulate.',
        metavar='sim',
    )
    #TODO(crbug.com/1056887): Implement this arg in infra.
    parser.add_argument(
        '--release',
        help='Indicates if this is a release build.',
        action='store_true',
    )
    parser.add_argument(
        '--replay-path',
        help=('Path to a directory containing WPR replay and recipe files, for '
              'use with WprProxySimulatorTestRunner to replay a test suite '
              'against multiple saved website interactions. '
              'Default: %(default)s'),
        default='NO_PATH',
        metavar='replay-path',
    )
    parser.add_argument(
        '--restart',
        action='store_true',
        help=argparse.SUPPRESS,
    )
    parser.add_argument(
        '-r',
        '--retries',
        help='Number of times to retry failed test cases.',
        metavar='n',
        type=int,
    )
    parser.add_argument(
        '--runtime-cache-prefix',
        metavar='PATH',
        help=(
            'Path prefix for runtime cache folder. The prefix will be appended '
            'with iOS version to construct the path. iOS simulator will be '
            'installed to the path and further copied into Xcode. Default: '
            '%(default)s. WARNING: this folder will be overwritten! This '
            'folder is intended to be a cached CIPD installation.'),
        default='Runtime-ios-',
    )
    parser.add_argument(
        '-s',
        '--shards',
        help='Number of shards to split test cases.',
        metavar='n',
        type=int,
    )
    parser.add_argument(
        '-t',
        '--test-cases',
        action='append',
        help=('Tests that should be included in the test run. All other tests '
              'will be excluded from this run. If unspecified, run all tests.'),
        metavar='testcase',
    )
    parser.add_argument(
        '--use-clang-coverage',
        help='Enable code coverage related steps in test runner scripts.',
        action='store_true',
    )
    parser.add_argument(
        '--use-trusted-cert',
        action='store_true',
        help=('Whether to install a cert to the simulator to allow for local '
              'HTTPS testing.'),
    )
    parser.add_argument(
        '-v',
        '--version',
        help='Version of iOS the simulator should run.',
        metavar='ver',
    )
    parser.add_argument(
        '--wpr-tools-path',
        help=(
            'Location of WPR test tools (should be preinstalled, e.g. as part '
            'of a swarming task requirement). Default: %(default)s.'),
        default='NO_PATH',
        metavar='wpr-tools-path',
    )
    parser.add_argument(
        '--xcode-path',
        metavar='PATH',
        help=('Path to <Xcode>.app folder where contents of the app will be '
              'installed. Default: %(default)s. WARNING: this folder will be '
              'overwritten! This folder is intended to be a cached CIPD '
              'installation.'),
        default='Xcode.app',
    )
    parser.add_argument(
        '--xcodebuild-device-runner',
        help='Run tests using xcodebuild\'s on real device.',
        action='store_true',
    )
    parser.add_argument(
        '--xctest',
        action='store_true',
        help='Whether or not the given app should be run as an XCTest.',
    )
    parser.add_argument(
        '--test-launcher-summary-output',
        default=None,
        help='Full path to output.json file. output.json is consumed by both '
        'collect_task.py and merge scripts.')

    def load_from_json(args):
      """
      Load and set arguments from args_json
      """
      args_json = json.loads(args.args_json)
      args.env_var = args.env_var or []
      args.env_var.extend(args_json.get('env_var', []))
      args.restart = args_json.get('restart', args.restart)
      args.test_cases = args.test_cases or []
      args.test_cases.extend(args_json.get('test_cases', []))
      args.xctest = args_json.get('xctest', args.xctest)
      args.xcode_parallelization = args_json.get('xcode_parallelization',
                                                 args.xcode_parallelization)
      args.xcodebuild_device_runner = (
          args_json.get('xcodebuild_device_runner',
                        args.xcodebuild_device_runner))
      args.shards = args_json.get('shards', args.shards)
      test_args.extend(args_json.get('test_args', []))

    def validate(args):
      """
      Runs argument validation
      """
      if (not (args.xcode_parallelization or args.xcodebuild_device_runner) and
          (args.iossim or args.platform or args.version)):
        # If any of --iossim, --platform, or --version
        # are specified then they must all be specified.
        if not (args.iossim and args.platform and args.version):
          parser.error('must specify all or none of '
                       '-i/--iossim, -p/--platform, -v/--version')

      if args.xcode_parallelization and not (args.platform and args.version):
        parser.error('--xcode-parallelization also requires '
                     'both -p/--platform and -v/--version')

    args, test_args = parser.parse_known_args(args)
    load_from_json(args)
    validate(args)
    # TODO(crbug.com/1056820): |app| won't contain "Debug" or "Release" after
    # recipe migrations.
    args.release = args.release or (args.app and "Release" in args.app)
    self.args = args
    self.test_args = test_args


def main(args):
  logging.basicConfig(
      format='[%(asctime)s:%(levelname)s] %(message)s',
      level=logging.DEBUG,
      datefmt='%I:%M:%S')

  test_runner.defaults_delete('com.apple.CoreSimulator',
                              'FramebufferServerRendererPolicy')
  runner = Runner()
  logging.debug("Arg values passed for this run: %s" % args)
  return runner.run(args)


if __name__ == '__main__':
  sys.exit(main(sys.argv[1:]))