Another round of test updates.

Author: anntzer

HACKING.rst

@@ -11,3 +11,11 @@ The following vim settings may be useful::
 
    setlocal cinoptions+=(0,W2,l1,h0
    setlocal shiftwidth=2
+
+Reference style
+===============
+
+In comments:
+
+- refer to upstream issues as ``FIXME[upstream-name]: ... (#issue-id)``;
+- refer to specific Matplotlib tests as ``:mpltest:`test_foo.test_bar```.

ISSUES.rst

@@ -2,6 +2,7 @@ Crashes
 =======
 
 ::
+
    gca(); savefig("/tmp/test.cairoscript")
 
 Seems to be due to cairo trying to call ``write`` during shutdown when the
@@ -12,33 +13,42 @@ Fix needed
 ==========
 
 test_image
+   test_figimage0[pdf], test_figimage1[pdf], test_interp_nearest_vs_none[pdf,svg], test_rasterize_dpi[pdf,svg]
+      Invalid dpi manipulations in vector output.
+
    test_jpeg_alpha
       Not respecting savefig.facecolor.
 
-   test_interp_nearest_vs_none[pdf,svg], test_rasterize_dpi[pdf,svg]
-      Not respecting dpi in vector output.
-
 test_text
    test_multiline
       Bad multiline rendering.
 
    test_text_alignment
       Bad alignment.
 
+text_labels_and_annotations/usetex_baseline_test
+   Bad layout of mathtext.
+
 Upstream issues
 ===============
 
 Issues with Matplotlib
 ----------------------
 
-Matplotlib's software alpha compositor is incorrect (#8847). ::
-
-   test_image::test_image_composite_alpha[pdf,svg]
-
 Matplotlib's hatching is inconsistent across backends (#10034). ::
 
    test_artist::test_clipping
 
+Matplotlib fails to drop the alpha channel when saving comparison failure diffs
+(#11075) (and the baseline images are lacking an alpha channel). ::
+
+   test_axes::test_log_scales[png]
+   test_scale::test_logscale_mask[png]
+
+Matplotlib's SVG backend does not implement Gouraud shading. ::
+
+   test_axes::test_pcolormesh[svg]
+
 Matplotlib's partially transparent markers with edges are inconsistent across
 backends (#10035). ::
 
@@ -48,16 +58,17 @@ Matplotlib's PDFPages is coupled too tightly with the PDF backend (#9114). ::
 
    test_backend_pdf::test_composite_image, test_multipage_*
 
-Matplotlib's SVG backend does not implement Gouraud shading. ::
-
-   test_axes::test_pcolormesh[svg]
-
 Matplotlib does not write SVGs with ``image-rendering: pixelated`` (#10112). ::
 
    test_backend_svg::test_noscale[svg]
    test_image::test_rotate_image[svg]
    test_tightlayout::test_tight_layout5[svg]
 
+Matplotlib's software alpha compositor is incorrect (#8847). ::
+
+   test_image::test_image_composite_alpha[pdf,svg]
+
+
 Issues with cairo
 -----------------
 
@@ -115,7 +126,7 @@ Tight bboxes are different. ::
 
    test_bbox_tight::test_bbox_inches_tight_suptile_legend
 
-``--infinite-tolerance`` subverts Matplotlib's test interface. ::
+``--tolerance`` subverts Matplotlib's test interface. ::
 
    test_compare_image::*
 

README.rst

@@ -283,8 +283,9 @@ PyPI or from source.
 
 Nearly all image comparison tests "fail" as the renderers are fundamentally
 different; currently, the intent is to manually check the diff images.  Passing
-``--infinite-tolerance`` marks these tests as "passed" so that one can spot
-issues not related to rendering differences.
+``--tolerance=inf`` marks these tests as "passed" (while still textually
+reporting the image differences) so that one can spot issues not related to
+rendering differences.
 
 Some other (non-image-comparison) tests are also known to fail (they are listed
 in ``ISSUES.rst``, with the relevant explanations), and automatically skipped.

lib/mplcairo/base.py

@@ -118,10 +118,14 @@ def draw_tex(self, gc, x, y, s, prop, angle, ismath="TeX!", mtext=None):
             page = next(iter(dvi))
         mb = _mplcairo.MathtextBackendCairo()
         for x1, y1, dvifont, glyph, width in page.text:
+            texfont = _get_tex_font_map()[dvifont.texname]
+            if texfont.filename is None:
+                # Not TypeError:
+                # :mpltest:`test_backend_svg.test_missing_psfont`.
+                raise ValueError("No font file found for {} ({!a})"
+                                 .format(texfont.psname, texfont.texname))
             mb._render_usetex_glyph(
-                x1, -y1,
-                _get_tex_font_map()[dvifont.texname].filename, dvifont.size,
-                glyph)
+                x1, -y1, texfont.filename, dvifont.size, glyph)
         for x1, y1, h, w in page.boxes:
             mb.render_rect_filled(x1, -y1, x1 + w, -(y1 + h))
         mb._draw(self, x, y, angle)
@@ -212,10 +216,8 @@ def _get_cached_or_new_renderer(
     def get_renderer(self, *, _draw_if_new=False):
         return self._get_cached_or_new_renderer(
             GraphicsContextRendererCairo,
-            *(np.array([self.figure.bbox.width, self.figure.bbox.height])
-              * getattr(self, "_dpi_ratio", 1)),
-            # Py3.4 support: use kwarg for dpi.
-            dpi=self.figure.dpi, _draw_if_new=_draw_if_new)
+            self.figure.bbox.width, self.figure.bbox.height, self.figure.dpi,
+            _draw_if_new=_draw_if_new)
 
     renderer = property(get_renderer)  # NOTE: Needed for FigureCanvasAgg.
 

run-mpl-test-suite.py

@@ -8,7 +8,6 @@
 
 from argparse import ArgumentParser
 import os
-import inspect
 from pathlib import Path
 import shutil
 import subprocess
@@ -18,31 +17,41 @@
 
 import matplotlib as mpl
 import matplotlib.backends.backend_agg
-import matplotlib.testing.compare
+import matplotlib.testing.decorators
 import mplcairo.base
 
 import pytest
 
 
+_IGNORED_FAILURES = {}
+
+
 def main(argv=None):
     parser = ArgumentParser(
         description="""\
 Run the Matplotlib test suite, using the mplcairo backend to patch out
 Matplotlib's agg backend.
 """,
         epilog="Other arguments are forwarded to pytest.")
-    parser.add_argument("--infinite-tolerance", action="store_true",
-                        help="Set image comparison tolerance to infinity.")
+    parser.add_argument("--tolerance", type=float,
+                        help="Set image comparison tolerance.")
     args, rest = parser.parse_known_args(argv)
 
-    if args.infinite_tolerance:
-        sig = inspect.signature(mpl.testing.compare.compare_images)
-        def compare_images(*args, **kwargs):
-            ba = sig.bind(*args, **kwargs)
-            ba.arguments["tol"] = float("inf")
-            return compare_images.__wrapped__(*ba.args, **ba.kwargs)
-        compare_images.__wrapped__ = mpl.testing.compare.compare_images
-        mpl.testing.compare.compare_images = compare_images
+    if args.tolerance is not None:
+        def _raise_on_image_difference(expected, actual, tol):
+            cmp = mpl.testing.compare.compare_images(
+                expected, actual, tol, in_decorator=True)
+            if cmp:
+                if cmp["rms"] < args.tolerance:
+                    expected = Path(expected)
+                    expected = expected.relative_to(expected.parent.parent)
+                    _IGNORED_FAILURES[expected] = cmp["rms"]
+                else:
+                    __orig_raise_on_image_tolerance(expected, actual, tol)
+        __orig_raise_on_image_tolerance = \
+            mpl.testing.decorators._raise_on_image_difference
+        mpl.testing.decorators._raise_on_image_difference = \
+            _raise_on_image_difference
 
     mplcairo.base.get_hinting_flag = mpl.backends.backend_agg.get_hinting_flag
     mplcairo.base.FigureCanvasAgg = \
@@ -51,8 +60,20 @@ def compare_images(*args, **kwargs):
         mplcairo.base.GraphicsContextRendererCairo
     mpl.backends.backend_agg = \
         sys.modules["matplotlib.backends.backend_agg"] = mplcairo.base
+
     mpl.use("agg", warn=False, force=True)
-    import matplotlib.pyplot as plt
+    from matplotlib import pyplot as plt
+
+    __orig_switch_backend = plt.switch_backend
+    def switch_backend(backend):
+        __orig_switch_backend({
+            "gtk3agg": "module://mplcairo.gtk",
+            "qt5agg": "module://mplcairo.qt",
+            "tkagg": "module://mplcairo.tk",
+            "wxagg": "module://mplcairo.wx",
+        }.get(backend.lower(), backend))
+    plt.switch_backend = switch_backend
+
     plt.switch_backend("agg")
 
     return pytest.main([
@@ -68,13 +89,14 @@ def pytest_collection_modifyitems(session, config, items):
     if len(items) == 0:
         pytest.exit("No tests found; Matplotlib was likely installed without "
                     "test data.")
-    excluded_modules = {
+    xfail_modules = {
         "matplotlib.tests.test_compare_images",
     }
-    excluded_nodeids = {
+    xfail_nodeids = {
         "matplotlib/tests/" + name for name in [
             "test_agg.py::test_repeated_save_with_alpha",
             "test_artist.py::test_cull_markers",
+            "test_axes.py::test_log_scales[png]",
             "test_backend_pdf.py::test_composite_image",
             "test_backend_pdf.py::test_multipage_keep_empty",
             "test_backend_pdf.py::test_multipage_pagecount",
@@ -92,25 +114,34 @@ def pytest_collection_modifyitems(session, config, items):
             "test_bbox_tight.py::test_bbox_inches_tight_suptile_legend[svg]",
             "test_image.py::test_composite[True-1-ps- colorimage]",
             "test_image.py::test_composite[False-2-ps- colorimage]",
+            "test_scale.py::test_logscale_mask[png]",
             "test_simplification.py::test_throw_rendering_complexity_exceeded",
         ]
     }
-    selected = []
-    deselected = []
+    xfails = []
     for item in items:
-        if (item.module.__name__ in excluded_modules
-                or item.nodeid in excluded_nodeids):
-            deselected.append(item)
-        else:
-            selected.append(item)
-    items[:] = selected
-    config.hook.pytest_deselected(items=deselected)
-    invalid_exclusions = (
-        (excluded_modules - {item.module.__name__ for item in deselected})
-        | (excluded_nodeids - {item.nodeid for item in deselected}))
-    if invalid_exclusions:
-        warnings.warn("Unused exclusions:\n    {}"
-                      .format("\n    ".join(sorted(invalid_exclusions))))
+        if (item.module.__name__ in xfail_modules
+                or item.nodeid in xfail_nodeids):
+            xfails.append(item)
+            item.add_marker(
+                pytest.mark.xfail(reason="Test known to fail with mplcairo"))
+    invalid_xfails = (
+        (xfail_modules - {item.module.__name__ for item in xfails})
+        | (xfail_nodeids - {item.nodeid for item in xfails}))
+    if invalid_xfails:
+        warnings.warn("Unused xfails:\n    {}"
+                      .format("\n    ".join(sorted(invalid_xfails))))
+
+
+def pytest_terminal_summary(terminalreporter, exitstatus):
+    write = terminalreporter.write
+    if _IGNORED_FAILURES:
+        write("\n"
+              "Ignored the following image comparison failures:\n"
+              "RMS\texpected\n")
+        for rms, expected in sorted(
+                ((v, k) for k, v in _IGNORED_FAILURES.items()), reverse=True):
+            write("{:#.2f}\t{}\n".format(rms, expected))
 
 
 if __name__ == "__main__":

src/_pattern_cache.cpp

@@ -168,7 +168,7 @@ void PatternCache::mask(
   // matrix.  1x threshold will be added by the patterns_ cache.
   // If the entire object is within the threshold of the origin in either
   // direction, then draw it directly, as doing otherwise would be highly
-  // inaccurate (see e.g. test_mplot3d.test_quiver3d).
+  // inaccurate (see e.g. :mpltest:`test_mplot3d.test_quiver3d`).
   auto const& bbox = it_bboxes->second;
   // Here, gcc 7.2 fails to extend the lifetime of the temporaries whose
   // references are bound to constant *references* (various issues on bugzilla;

src/_util.cpp

@@ -157,9 +157,9 @@ AdditionalState& get_additional_state(cairo_t* cr)
 // if desired.
 // FIXME[cairo]: cairo requires coordinates to fit within a 24-bit signed
 // integer (https://bugs.freedesktop.org/show_bug.cgi?id=20091 and
-// test_simplification.test_overflow).  We simply clamp the values in the
-// general case (with codes) -- proper handling would involve clipping
-// of polygons and of Beziers -- and use a simple clippling algorithm
+// :mpltest:`test_simplification.test_overflow`).  We simply clamp the
+// values in the general case (with codes) -- proper handling would involve
+// clipping of polygons and of Beziers -- and use a simple clippling algorithm
 // (Cohen-Sutherland) in the simple (codeless) case as we expect most segments
 // to be within the clip rectangle -- cairo will run its own clipping later
 // anyways.
@@ -267,9 +267,9 @@ void load_path_exact(
         }
         break;
       // The semantics of nonfinite control points are tested in
-      // test_simplification.test_simplify_curve: if the last point is finite,
-      // it sets the current point for the next curve; otherwise, a new
-      // sub-path is created.
+      // :mpltest:`test_simplification.test_simplify_curve`: if the last point
+      // is finite, it sets the current point for the next curve; otherwise, a
+      // new sub-path is created.
       case PathCode::CURVE3: {
         auto x1 = vertices(i + 1, 0), y1 = vertices(i + 1, 1);
         cairo_matrix_transform_point(matrix, &x1, &y1);