[#28] Added test_manager and test_runner

Adds two classes which manage context for running tests in containers.

The test_runner is an abstract base class that requires a single ethod
to be implemented by "concrete" test_runners which executes the test. In
this commit, the only test_runner implemented is the
`irods_python_suite`, which executes the tests from the iRODS python
test suite (i.e. /var/lib/irods/scripts/run_tests.py).

The test_manager manages a number of test_runners so that tests can be
distributed to the test_runners and then executed in parallel on the
available pool of zones.

Author: alanking

irods_testing_environment/context.py

@@ -119,6 +119,17 @@ def service_account_irods_env():
     return os.path.join(irods_home(), '.irods', 'irods_environment.json')
 
 
+def run_tests_script():
+    """Return the path to the script which runs the python tests."""
+    import os
+    return os.path.join(irods_home(), 'scripts', 'run_tests.py')
+
+
+def python():
+    """Return the appropriate python interpreter."""
+    return 'python3'
+
+
 def sanitize(repo_or_tag):
     """Sanitize the input from special characters rejected by docker-compose.
 

irods_testing_environment/test_manager.py

@@ -0,0 +1,104 @@
+# grown-up modules
+import logging
+
+# local modules
+from . import test_runner
+
+class test_manager:
+    """A class that manages a list of tests and `test_runners` for executing tests."""
+
+    def __init__(self, containers, tests, test_type='irods_python_suite'):
+        """Constructor for `test_manager`.
+
+        Arguments:
+        containers -- list of containers which will be used to construct `test_runner`s
+        tests -- list of tests which will run on the `test_runners`
+        test_type -- a string representing the name of the class implementing the test_runner
+        """
+        tr_name = '_'.join(['test_runner', test_type])
+        tr = eval('.'.join(['test_runner', tr_name]))
+        logging.info('[{}]'.format(tr))
+        self.test_runners = [tr(c) for c in containers]
+
+        logging.info('[{}]'.format(tests))
+        logging.info('[{}]'.format(str(self)))
+        logging.info('[{}]'.format(self.test_runners))
+
+        for i, t in enumerate(tests):
+            index = i % len(self.test_runners)
+
+            logging.info('index [{}], test [{}]'.format(index, t))
+            self.test_runners[index].add_test(t)
+
+        logging.info('[{}]'.format(str(self)))
+
+
+    def __str__(self):
+        """Return a string representation of the list of `test_runners`."""
+        return str([str(tr) for tr in self.test_runners])
+
+
+    def failed_tests(self):
+        """Return a list of failed tests across the managed `test_runners`."""
+        return [t for tr in self.test_runners for t in tr.failed_tests()]
+
+
+    def return_code(self):
+        """Return int representing the 'overall' return code from a test run.
+
+        Return 0 if all `test_runners` have a return code of 0. Otherwise, 1.
+        """
+        logging.info('[{}]'.format([tr.rc for tr in self.test_runners]))
+        return 0 if [tr.rc for tr in self.test_runners].count(0) == len(self.test_runners) else 1
+
+
+    def result_string(self):
+        """Return string showing tests that passed and failed from each `test_runner.`"""
+        r = '==== begin test run results ====\n'
+        for tr in self.test_runners:
+            r = r + tr.result_string()
+
+        if self.return_code() is not 0:
+            r = r + 'List of failed tests:\n\t{}\n'.format(' '.join(self.failed_tests()))
+            r = r + 'Return code:[{}]\n'.format(self.return_code())
+
+        else:
+            r = r + 'All tests passed! :)\n'
+
+        r = r + '==== end of test run results ====\n'
+
+        return r
+
+
+    def run(self, fail_fast=True, *args):
+        """Run managed `test_runners` in parallel.
+
+        Arguments:
+        fail_fast -- if True, the first test to fail ends the run
+        *args -- arguments to be passed along to the `test_runner`'s specific `run` method
+        """
+        import concurrent.futures
+
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            futures_to_test_runners = {
+                executor.submit(tr.run, fail_fast, *args): tr for tr in self.test_runners
+            }
+
+            for f in concurrent.futures.as_completed(futures_to_test_runners):
+                tr = futures_to_test_runners[f]
+
+                try:
+                    f.result()
+
+                    if tr.rc is 0 and len(tr.failed_tests()) is 0:
+                        logging.warning('tests completed successfully [{}]'.format(tr.name()))
+                    else:
+                        logging.warning('some tests failed [{}]'.format(tr.name()))
+
+                except Exception as e:
+                    logging.warning('[{}]: exception raised while running test'.format(tr.name()))
+                    logging.warning(e)
+
+                    tr.rc = 1
+
+                    if fail_fast: raise

irods_testing_environment/test_runner.py

@@ -0,0 +1,145 @@
+# grown-up modules
+import logging
+
+# local modules
+from . import context
+from . import execute
+
+class test_runner:
+    """A class that manages a list of tests and can execute them on a managed container."""
+
+    def __init__(self, executing_container, tests=None):
+        """Constructor for `test_runner`.
+
+        The list of tests is not required to be initialized here. `add_test` will append a
+        test to the end of the test list.
+
+        Arguments:
+        executing_container -- the container on which tests will be executed
+        tests -- optional list of tests to use with this runner
+        """
+        # TODO: each test runner is tied to a single container... might need to abstract this out later
+        self.executor = executing_container
+        self.tests = tests or list()
+        self.passed = list()
+        self.failed = list()
+        self.rc = 0
+
+
+    def __str__(self):
+        """Return a string representation of a map representing the data members."""
+        return str({
+            'executing_container': self.name(),
+            'return_code': self.rc,
+            'test_list': self.tests,
+            'passed_tests': self.passed_tests(),
+            'failed_tests': self.failed_tests()
+        })
+
+
+    def add_test(self, test):
+        """Append `test` to the test list and return self."""
+        logging.info('before [{}]'.format(self))
+        self.tests.append(test)
+        logging.info('after  [{}]'.format(self))
+        return self
+
+
+    def name(self):
+        """Return the name of the executing container."""
+        return self.executor.name
+
+
+    def test_list(self):
+        """Return the list of tests for which this runner is responsible."""
+        return self.tests
+
+
+    def passed_tests(self):
+        """Return the list of tests which have been executed and passed."""
+        return self.passed
+
+
+    def failed_tests(self):
+        """Return the list of tests which have been executed and failed."""
+        return self.failed
+
+
+    def skipped_tests(self):
+        """Return the list of tests which have not been executed."""
+        executed_tests = self.passed_tests() + self.failed_tests()
+        return list(filter(lambda t: t not in executed_tests, self.test_list()))
+
+
+    def result_string(self):
+        """Return a string representing the results of running the test list."""
+        r = '-----\nresults for [{}]\n'.format(self.name())
+
+        r = r + '\tpassed tests:\n'
+        for t in self.passed_tests():
+            r = r + '\t\t[{}]\n'.format(t)
+
+        r = r + '\tskipped tests:\n'
+        for t in self.skipped_tests():
+            r = r + '\t\t[{}]\n'.format(t)
+
+        r = r + '\tfailed tests:\n'
+        for t in self.failed_tests():
+            r = r + '\t\t[{}]\n'.format(t)
+
+        r = r + '\treturn code:[{}]\n-----\n'.format(self.rc)
+
+        return r
+
+
+    def run(self, fail_fast=True, *args):
+        """Execute test list sequentially on executing container.
+
+        Arguments:
+        fail_fast -- if True, the first test to fail ends the run
+        *args -- any additional arguments that the test execution can take
+        """
+        for t in self.tests:
+            cmd, ec = self.execute_test(t, *args)
+
+            if ec is 0:
+                self.passed_tests().append(t)
+                logging.info('[{}]: cmd succeeded [{}]'.format(self.name(), cmd))
+
+            else:
+                self.rc = ec
+                self.failed_tests().append(t)
+                logging.warning('[{}]: cmd failed [{}] [{}]'.format(self.name(), ec, cmd))
+
+                if fail_fast:
+                    raise RuntimeError('[{}]: command failed [{}]'.format(self.name(), cmd))
+
+        if self.rc is not 0:
+            logging.error('[{}]: tests that failed [{}]'.format(self.name(), self.failed_tests()))
+
+
+    def execute_test(self, test, *args):
+        """Execute `test` with return the command run and the return code."""
+        raise NotImplementedError('test_runner is a base class and should not be used directly')
+
+
+class test_runner_irods_python_suite(test_runner):
+    def __init__(self, executing_container, tests=None):
+        super(test_runner_irods_python_suite, self).__init__(executing_container, tests)
+
+
+    @staticmethod
+    def run_tests_command():
+        """Return a list of strings used as a space-delimited invocation of the test runner."""
+        return [context.python(), context.run_tests_script()]
+
+
+    def execute_test(self, test, options=None):
+        """Execute `test` with `options` and return the command run and the return code."""
+        cmd = self.run_tests_command() + ['--run_specific_test', test]
+        if options: cmd.append(options)
+        return cmd, execute.execute_command(self.executor,
+                                            ' '.join(cmd),
+                                            user='irods',
+                                            workdir=context.irods_home(),
+                                            stream_output=True)