tests: no more files written to the working directory.

Author: mr-c

.github/workflows/ci-tests.yml

@@ -114,6 +114,44 @@ jobs:
       - name: Test with tox
         run: tox
 
+  clean_working_dir:
+    name: No leftovers
+    runs-on: ubuntu-22.04
+    env:
+      py-semver: "3.11"
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Set up Singularity
+        run: |
+          wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-focal_amd64.deb
+          sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb
+
+      - name: Give the test runner user a name to make provenance happy.
+        run: sudo usermod -c 'CI Runner' $(whoami)
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.py-semver }}
+          cache: pip
+
+      - name: install with test dependencies
+        run: |
+          pip install -U pip setuptools wheel
+          pip install --no-build-isolation -rtest-requirements.txt .[deps]
+
+      - name: make working directory read-only
+        run: |
+          mkdir .pytest_cache
+          chmod a-w .
+
+      - name: run tests
+        run: make test
+
+
   conformance_tests:
     name: CWL conformance
     runs-on: ubuntu-22.04

tests/test_dependencies.py

@@ -1,11 +1,12 @@
 """Tests of satisfying SoftwareRequirement via dependencies."""
 import os
 import tempfile
+from getpass import getuser
 from pathlib import Path
 from shutil import which
 from types import ModuleType
-from typing import Optional, Tuple, Callable
-from getpass import getuser
+from typing import Optional, Tuple
+
 import pytest
 
 from cwltool.context import LoadingContext
@@ -63,6 +64,7 @@ def bioconda_setup(request: pytest.FixtureRequest) -> Tuple[Optional[int], str]:
     :py:method:`pytest.TempPathFactory.getbasetemp`.
     """
 
+    assert request.config.cache
     deps_dir = request.config.cache.get("bioconda_deps", None)
     if deps_dir is not None and not Path(deps_dir).exists():
         # cache value set, but cache is gone :( ... recreate
@@ -86,7 +88,7 @@ def bioconda_setup(request: pytest.FixtureRequest) -> Tuple[Optional[int], str]:
         request.config.cache.set("bioconda_deps", str(deps_dir))
 
     deps_dirpath = Path(deps_dir)
-    deps_dirpath.mkdir(exist_ok=True)
+    deps_dirpath.mkdir(parents=True, exist_ok=True)
 
     wflow = get_data("tests/seqtk_seq.cwl")
     job = get_data("tests/seqtk_seq_job.json")
@@ -106,7 +108,7 @@ def bioconda_setup(request: pytest.FixtureRequest) -> Tuple[Optional[int], str]:
 
 
 @pytest.mark.skipif(not deps, reason="galaxy-tool-util is not installed")
-def test_bioconda(bioconda_setup: Callable[[], Tuple[Optional[int], str]]) -> None:
+def test_bioconda(bioconda_setup: Tuple[Optional[int], str]) -> None:
     error_code, stderr = bioconda_setup
     assert error_code == 0, stderr
 

tests/test_examples.py

@@ -1291,6 +1291,8 @@ def test_cache_relative_paths(tmp_path: Path, factor: str) -> None:
     commands = factor.split()
     commands.extend(
         [
+            "--out",
+            str(tmp_path / "out"),
             "--cachedir",
             cache_dir,
             get_data(f"tests/{test_file}"),
@@ -1306,6 +1308,8 @@ def test_cache_relative_paths(tmp_path: Path, factor: str) -> None:
     commands = factor.split()
     commands.extend(
         [
+            "--out",
+            str(tmp_path / "out2"),
             "--cachedir",
             cache_dir,
             get_data(f"tests/{test_file}"),
@@ -1324,6 +1328,8 @@ def test_cache_relative_paths(tmp_path: Path, factor: str) -> None:
 def test_write_summary(tmp_path: Path) -> None:
     """Test --write-summary."""
     commands = [
+        "--out",
+        str(tmp_path / "out1"),
         get_data("tests/wf/no-parameters-echo.cwl"),
     ]
     error_code, stdout, stderr = get_main_output(commands)
@@ -1332,6 +1338,8 @@ def test_write_summary(tmp_path: Path) -> None:
 
     final_output_path = str(tmp_path / "final-output.json")
     commands_no = [
+        "--out",
+        str(tmp_path / "out2"),
         "--write-summary",
         final_output_path,
         get_data("tests/wf/no-parameters-echo.cwl"),
@@ -1347,10 +1355,11 @@ def test_write_summary(tmp_path: Path) -> None:
 
 
 @needs_docker
-def test_compute_checksum() -> None:
+def test_compute_checksum(tmp_path: Path) -> None:
     runtime_context = RuntimeContext()
     runtime_context.compute_checksum = True
     runtime_context.use_container = False
+    runtime_context.outdir = str(tmp_path)
     factory = cwltool.factory.Factory(runtime_context=runtime_context)
     echo = factory.make(get_data("tests/wf/cat-tool.cwl"))
     output = echo(
@@ -1413,10 +1422,12 @@ def test_bad_stdout_expr_error() -> None:
 
 
 @needs_docker
-def test_stdin_with_id_preset() -> None:
+def test_stdin_with_id_preset(tmp_path: Path) -> None:
     """Confirm that a type: stdin with a preset id does not give an error."""
     error_code, _, stderr = get_main_output(
         [
+            "--out",
+            str(tmp_path),
             get_data("tests/wf/1590.cwl"),
             "--file1",
             get_data("tests/wf/whale.txt"),
@@ -1590,7 +1601,7 @@ def test_v1_0_arg_empty_prefix_separate_false() -> None:
 
 def test_scatter_output_filenames(tmp_path: Path) -> None:
     """Confirm that the final output is renamed correctly from identically named scatter outputs."""
-    cwd = Path.cwd()
+    cwd = tmp_path
     with working_directory(tmp_path):
         rtc = RuntimeContext()
         rtc.outdir = str(cwd)
@@ -1726,10 +1737,10 @@ def test_command_line_tool_class() -> None:
     assert str(expression_tool) == f"CommandLineTool: file://{tool_path}"
 
 
-def test_record_default_with_long() -> None:
+def test_record_default_with_long(tmp_path: Path) -> None:
     """Confirm that record defaults are respected."""
     tool_path = get_data("tests/wf/paramref_arguments_roundtrip.cwl")
-    err_code, stdout, stderr = get_main_output([tool_path])
+    err_code, stdout, stderr = get_main_output(["--outdir", str(tmp_path), tool_path])
     assert err_code == 0
     result = json.loads(stdout)["same_record"]
     assert result["first"] == "y"

tests/test_iwdr.py

@@ -5,6 +5,8 @@
 from stat import S_IWGRP, S_IWOTH, S_IWRITE
 from typing import Any
 
+import pytest
+
 from cwltool.factory import Factory
 from cwltool.main import main
 
@@ -246,7 +248,9 @@ def test_iwdr_permutations_inplace(tmp_path_factory: Any) -> None:
 
 
 @needs_singularity
-def test_iwdr_permutations_singularity(tmp_path_factory: Any) -> None:
+def test_iwdr_permutations_singularity(
+    tmp_path_factory: pytest.TempPathFactory, monkeypatch: pytest.MonkeyPatch
+) -> None:
     misc = tmp_path_factory.mktemp("misc")
     fifth = misc / "fifth"
     fifth.mkdir()
@@ -269,6 +273,8 @@ def test_iwdr_permutations_singularity(tmp_path_factory: Any) -> None:
     twelfth = misc / "twelfth"
     twelfth.touch()
     outdir = str(tmp_path_factory.mktemp("outdir"))
+    singularity_dir = str(tmp_path_factory.mktemp("singularity"))
+    monkeypatch.setenv("CWL_SINGULARITY_CACHE", singularity_dir)
     err_code, stdout, _ = get_main_output(
         [
             "--outdir",
@@ -306,7 +312,9 @@ def test_iwdr_permutations_singularity(tmp_path_factory: Any) -> None:
 
 
 @needs_singularity
-def test_iwdr_permutations_singularity_inplace(tmp_path_factory: Any) -> None:
+def test_iwdr_permutations_singularity_inplace(
+    tmp_path_factory: pytest.TempPathFactory, monkeypatch: pytest.MonkeyPatch
+) -> None:
     """IWDR tests using --singularity and a forced InplaceUpdateRequirement."""
     misc = tmp_path_factory.mktemp("misc")
     fifth = misc / "fifth"
@@ -330,6 +338,8 @@ def test_iwdr_permutations_singularity_inplace(tmp_path_factory: Any) -> None:
     twelfth = misc / "twelfth"
     twelfth.touch()
     outdir = str(tmp_path_factory.mktemp("outdir"))
+    singularity_dir = str(tmp_path_factory.mktemp("singularity"))
+    monkeypatch.setenv("CWL_SINGULARITY_CACHE", singularity_dir)
     assert (
         main(
             [

tests/test_js_sandbox.py

@@ -63,6 +63,10 @@ def hide_nodejs(temp_dir: Path) -> str:
                 if entry not in ("nodejs", "node"):
                     os.symlink(os.path.join(dirname, entry), new_dir / entry)
             paths.append(str(new_dir))
+            dirname_path = Path(dirname)
+            for path in paths:
+                if Path(path).resolve() == dirname_path:
+                    paths.remove(path)
     return ":".join(paths)
 
 
@@ -94,12 +98,17 @@ def test_value_from_two_concatenated_expressions_singularity(
     js_engine = sandboxjs.get_js_engine()
     js_engine.have_node_slim = False  # type: ignore[attr-defined]
     js_engine.localdata = threading.local()  # type: ignore[attr-defined]
-    new_paths = hide_nodejs(tmp_path)
+    hide_base = tmp_path / "hide"
+    hide_base.mkdir()
+    new_paths = hide_nodejs(hide_base)
+    singularity_cache = tmp_path / "singularity"
+    singularity_cache.mkdir()
     factory = Factory()
     factory.loading_context.singularity = True
     factory.loading_context.debug = True
     factory.runtime_context.debug = True
     with monkeypatch.context() as m:
+        m.setenv("CWL_SINGULARITY_CACHE", str(singularity_cache))
         m.setenv("PATH", new_paths)
         echo = factory.make(get_data("tests/wf/vf-concat.cwl"))
         file = {"class": "File", "location": get_data("tests/wf/whale.txt")}

tests/test_provenance.py

@@ -591,8 +591,8 @@ def check_prov(
 
 
 @pytest.fixture
-def research_object() -> Generator[ResearchObject, None, None]:
-    re_ob = ResearchObject(StdFsAccess(""))
+def research_object(tmp_path: Path) -> Generator[ResearchObject, None, None]:
+    re_ob = ResearchObject(StdFsAccess(str(tmp_path / "ro")), temp_prefix_ro=str(tmp_path / "tmp"))
     yield re_ob
     re_ob.close()
 

tests/test_singularity.py

@@ -3,6 +3,8 @@
 from pathlib import Path
 from typing import Any
 
+import pytest
+
 from cwltool.main import main
 
 from .util import (
@@ -58,7 +60,10 @@ def test_singularity_workflow(tmp_path: Path) -> None:
     assert error_code == 0
 
 
-def test_singularity_iwdr() -> None:
+def test_singularity_iwdr(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    singularity_dir = tmp_path / "singularity"
+    singularity_dir.mkdir()
+    monkeypatch.setenv("CWL_SINGULARITY_CACHE", str(singularity_dir))
     result_code = main(
         [
             "--singularity",

tests/test_subgraph.py

@@ -219,10 +219,12 @@ def test_single_process_packed_subwf_step(tmp_path: Path) -> None:
 
 
 @needs_docker
-def test_single_process_subwf_subwf_inline_step() -> None:
+def test_single_process_subwf_subwf_inline_step(tmp_path: Path) -> None:
     """Test --single-process on an inline sub-sub-workflow step."""
     err_code, stdout, stderr = get_main_output(
         [
+            "--outdir",
+            str(tmp_path),
             "--single-process",
             "step1/stepX/stepY",
             get_data("tests/subgraph/count-lines17-wf.cwl.json"),

tox.ini

@@ -11,7 +11,7 @@ envlist =
 skip_missing_interpreters = True
 
 [pytest]
-addopts=--ignore cwltool/schemas --basetemp ./tmp -n auto
+addopts=--ignore cwltool/schemas -n auto
 testpaths = tests
 
 [gh-actions]