[mypyc] Improve access to generated C on test failures and document this (#18476)

Now the generated C files for the first mypyc run test failure in a
pytest session will be copied to the `.mypyc_test_output` directory, and
this will be indicated in the test output. This is a convenience feature
to help in the common scenario where all test failures have the same
root cause, so any single output is sufficient for debugging.

Document this and `--mypyc-showc`, which allows showing generated C for
every test failure. The latter is too verbose to be enabled by default.

Author: JukkaL

.gitignore

@@ -17,6 +17,7 @@ venv/
 test-data/packages/.pip_lock
 dmypy.json
 .dmypy.json
+/.mypyc_test_output
 
 # Packages
 *.egg

mypy/test/config.py

@@ -18,6 +18,9 @@
 # It is also hard-coded in numerous places, so don't change it.
 test_temp_dir = "tmp"
 
+# Mypyc tests may write intermediate files (e.g. generated C) here on failure
+mypyc_output_dir = os.path.join(PREFIX, ".mypyc_test_output")
+
 # The PEP 561 tests do a bunch of pip installs which, even though they operate
 # on distinct temporary virtual environments, run into race conditions on shared
 # file-system state. To make this work reliably in parallel mode, we'll use a

mypy/test/data.py

@@ -20,7 +20,7 @@
 import pytest
 
 from mypy import defaults
-from mypy.test.config import PREFIX, test_data_prefix, test_temp_dir
+from mypy.test.config import PREFIX, mypyc_output_dir, test_data_prefix, test_temp_dir
 
 root_dir = os.path.normpath(PREFIX)
 
@@ -586,6 +586,13 @@ def fix_cobertura_filename(line: str) -> str:
 ##
 
 
+def pytest_sessionstart(session: Any) -> None:
+    # Clean up directory where mypyc tests write intermediate files on failure
+    # to avoid any confusion between test runs
+    if os.path.isdir(mypyc_output_dir):
+        shutil.rmtree(mypyc_output_dir)
+
+
 # This function name is special to pytest.  See
 # https://docs.pytest.org/en/latest/reference.html#initialization-hooks
 def pytest_addoption(parser: Any) -> None:

mypyc/doc/dev-intro.md

@@ -296,6 +296,22 @@ Compiled native functions have the prefix `CPyDef_`, while wrapper
 functions used for calling functions from interpreted Python code have
 the `CPyPy_` prefix.
 
+When running a test, the first test failure will copy generated C code
+into the `.mypyc_test_output` directory. You will see something like
+this in the test output:
+
+```
+...
+---------------------------- Captured stderr call -----------------------------
+
+Generated files: /Users/me/src/mypy/.mypyc_test_output (for first failure only)
+
+...
+```
+
+You can also run pytest with `--mypyc-showc` to display C code on every
+test failure.
+
 ## Other Important Limitations
 
 All of these limitations will likely be fixed in the future:

mypyc/test/test_run.py

@@ -17,7 +17,7 @@
 from mypy import build
 from mypy.errors import CompileError
 from mypy.options import Options
-from mypy.test.config import test_temp_dir
+from mypy.test.config import mypyc_output_dir, test_temp_dir
 from mypy.test.data import DataDrivenTestCase
 from mypy.test.helpers import assert_module_equivalence, perform_file_operations
 from mypyc.build import construct_groups
@@ -281,6 +281,7 @@ def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) ->
         if not run_setup(setup_file, ["build_ext", "--inplace"]):
             if testcase.config.getoption("--mypyc-showc"):
                 show_c(cfiles)
+            copy_output_files(mypyc_output_dir)
             assert False, "Compilation failed"
 
         # Assert that an output file got created
@@ -344,6 +345,7 @@ def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) ->
                 )
                 print("hint: You may need to build a debug version of Python first and use it")
                 print('hint: See also "Debuggging Segfaults" in mypyc/doc/dev-intro.md')
+            copy_output_files(mypyc_output_dir)
 
         # Verify output.
         if bench:
@@ -457,3 +459,17 @@ def fix_native_line_number(message: str, fnam: str, delta: int) -> str:
         message,
     )
     return message
+
+
+def copy_output_files(target_dir: str) -> None:
+    try:
+        os.mkdir(target_dir)
+    except OSError:
+        # Only copy data for the first failure, to avoid excessive output in case
+        # many tests fail
+        return
+
+    for fnam in glob.glob("build/*.[ch]"):
+        shutil.copy(fnam, target_dir)
+
+    sys.stderr.write(f"\nGenerated files: {target_dir} (for first failure only)\n\n")