update testing helpers

- `@matplotlib.testing.decorators.image_comparison` now prints the path
  to the diff image so it is easier to find, so that newcomers know it
  exists.

- update `tests.py` to give more information about how it should be run
  if matplotlib is not installed already.

- update `matplotlib.test` to return -1 instead of raising ImportErrors
  if there are preconditions not met. This allows the error messages to
  not be buried by the stack traces, is still a return value which would
  signify an error, and does not conflict with the range used by
  `pytest.ExitCode` (0-5) allowing callers to distinguish different
  pytest error codes from an error starting up pytest

- move the `import pytest` check out of `matplotlib._init_tests` because
  it is only called via `matplotlib.test` and via pytest's configure
  handler, and it is more appropriate and can be handled more gracefully
  if it is checked in `matplotlib.test`

Author: terencehonles

lib/matplotlib/__init__.py

@@ -1168,29 +1168,29 @@ def _init_tests():
             "Freetype build type is {}local".format(
                 "" if ft2font.__freetype_build_type__ == 'local' else "not "))
 
-    try:
-        import pytest
-    except ImportError:
-        print("matplotlib.test requires pytest to run.")
-        raise
-
 
 @cbook._delete_parameter("3.2", "switch_backend_warn")
 @cbook._delete_parameter("3.3", "recursionlimit")
 def test(verbosity=None, coverage=False, switch_backend_warn=True,
          recursionlimit=0, **kwargs):
     """Run the matplotlib test suite."""
-    _init_tests()
+
+    try:
+        import pytest
+    except ImportError:
+        print("matplotlib.test requires pytest to run.")
+        return -1
+
     if not os.path.isdir(os.path.join(os.path.dirname(__file__), 'tests')):
-        raise ImportError("Matplotlib test data is not installed")
+        print("Matplotlib test data is not installed")
+        return -1
 
     old_backend = get_backend()
     old_recursionlimit = sys.getrecursionlimit()
     try:
         use('agg')
         if recursionlimit:
             sys.setrecursionlimit(recursionlimit)
-        import pytest
 
         args = kwargs.pop('argv', [])
         provide_default_modules = True

lib/matplotlib/testing/decorators.py

@@ -130,11 +130,11 @@ def _raise_on_image_difference(expected, actual, tol):
 
     err = compare_images(expected, actual, tol, in_decorator=True)
     if err:
-        for key in ["actual", "expected"]:
+        for key in ["actual", "expected", "diff"]:
             err[key] = os.path.relpath(err[key])
         raise ImageComparisonFailure(
-            'images not close (RMS %(rms).3f):\n\t%(actual)s\n\t%(expected)s '
-             % err)
+            ('images not close (RMS %(rms).3f):'
+                '\n\t%(actual)s\n\t%(expected)s\n\t%(diff)s') % err)
 
 
 def _skip_if_format_is_uncomparable(extension):

tests.py

@@ -13,7 +13,12 @@
 
 
 if __name__ == '__main__':
-    from matplotlib import test
+    try:
+        from matplotlib import test
+    except ImportError:
+        print('matplotlib.test could not be imported.\n\n'
+              'Try a virtual env and `pip install -e .`')
+        sys.exit(-1)
 
     parser = argparse.ArgumentParser(add_help=False)
     parser.add_argument('--recursionlimit', type=int, default=None,