Improve pytask clean. (#32)

Author: tobiasraabe

docs/changes.rst

@@ -6,12 +6,18 @@ chronological order. Releases follow `semantic versioning <https://semver.org/>`
 all releases are available on `Anaconda.org <https://anaconda.org/pytask/pytask>`_.
 
 
+0.0.9 - 2020-xx-xx
+------------------
+
+- :gh:`31` adds ``pytask collect`` to show information on collected tasks.
+- :gh:`32` fixes ``pytask clean``.
+
+
 0.0.8 - 2020-10-04
 ------------------
 
 - :gh:`30` fixes or adds the session object to some hooks which was missing from the
   previous release.
-- :gh:`31` adds ``pytask collect`` to show information on collected tasks.
 
 
 0.0.7 - 2020-10-03

src/_pytask/build.py

@@ -114,5 +114,6 @@ def build(**config_from_cli):
     working directory for tasks to execute them. A report informs you on the results.
 
     """
+    config_from_cli["command"] = "build"
     session = main(config_from_cli)
     sys.exit(session.exit_code)

src/_pytask/clean.py

@@ -9,6 +9,7 @@
 import attr
 import click
 from _pytask.config import hookimpl
+from _pytask.config import IGNORED_TEMPORARY_FILES_AND_FOLDERS
 from _pytask.enums import ExitCode
 from _pytask.exceptions import CollectionError
 from _pytask.pluginmanager import get_plugin_manager
@@ -24,7 +25,7 @@
 )
 
 
-@hookimpl
+@hookimpl(tryfirst=True)
 def pytask_extend_command_line_interface(cli: click.Group):
     """Extend the command line interface."""
     cli.add_command(clean)
@@ -44,6 +45,15 @@ def pytask_parse_config(config, config_from_cli):
     )
 
 
+@hookimpl
+def pytask_post_parse(config):
+    """Correct ignore patterns such that caches, etc. will not be ignored."""
+    if config["command"] == "clean":
+        config["ignore"] = [
+            i for i in config["ignore"] if i not in IGNORED_TEMPORARY_FILES_AND_FOLDERS
+        ]
+
+
 @click.command()
 @click.argument(
     "paths", nargs=-1, type=click.Path(exists=True), callback=falsy_to_none_callback
@@ -58,11 +68,23 @@ def pytask_parse_config(config, config_from_cli):
 @click.option(
     "-q", "--quiet", is_flag=True, help="Do not print the names of the removed paths."
 )
+@click.option(
+    "--ignore",
+    type=str,
+    multiple=True,
+    help=(
+        "A pattern to ignore files or directories. For example, ``task_example.py`` or "
+        "``src/*``."
+    ),
+    callback=falsy_to_none_callback,
+)
 @click.option(
     "-c", "--config", type=click.Path(exists=True), help="Path to configuration file."
 )
 def clean(**config_from_cli):
     """Clean provided paths by removing files unknown to pytask."""
+    config_from_cli["command"] = "clean"
+
     try:
         # Duplication of the same mechanism in :func:`pytask.main.main`.
         pm = get_plugin_manager()
@@ -93,24 +115,29 @@ def clean(**config_from_cli):
                 session, known_paths, include_directories
             )
 
-            targets = "Files"
-            if session.config["directories"]:
-                targets += " and directories"
-            click.echo(f"\n{targets} which can be removed:\n")
-            for path in unknown_paths:
-                if session.config["mode"] == "dry-run":
-                    click.echo(f"Would remove {path}.")
-                else:
-                    should_be_deleted = session.config[
-                        "mode"
-                    ] == "force" or click.confirm(f"Would you like to remove {path}?")
-                    if should_be_deleted:
-                        if not session.config["quiet"]:
-                            click.echo(f"Remove {path}.")
-                        if path.is_dir():
-                            shutil.rmtree(path)
-                        else:
-                            path.unlink()
+            if unknown_paths:
+                targets = "Files"
+                if session.config["directories"]:
+                    targets += " and directories"
+                click.echo(f"\n{targets} which can be removed:\n")
+                for path in unknown_paths:
+                    if session.config["mode"] == "dry-run":
+                        click.echo(f"Would remove {path}")
+                    else:
+                        should_be_deleted = session.config[
+                            "mode"
+                        ] == "force" or click.confirm(
+                            f"Would you like to remove {path}?"
+                        )
+                        if should_be_deleted:
+                            if not session.config["quiet"]:
+                                click.echo(f"Remove {path}")
+                            if path.is_dir():
+                                shutil.rmtree(path)
+                            else:
+                                path.unlink()
+            else:
+                click.echo("\nThere are no files and directories which can be deleted.")
 
             click.echo("\n" + "=" * config["terminal_width"])
 
@@ -169,7 +196,7 @@ def _find_all_unknown_paths(session, known_paths, include_directories):
 
     """
     recursive_nodes = [
-        _RecursivePathNode.from_path(path, known_paths)
+        _RecursivePathNode.from_path(path, known_paths, session)
         for path in session.config["paths"]
     ]
     unknown_paths = list(
@@ -218,21 +245,38 @@ class _RecursivePathNode:
     is_unknown = attr.ib(type=bool)
 
     @classmethod
-    def from_path(cls, path: Path, known_paths: list):
+    def from_path(cls, path: Path, known_paths: list, session):
         """Create a node from a path.
 
         While instantiating the class, subordinate nodes are spawned for all paths
         inside a directory.
 
         """
         sub_nodes = (
-            [_RecursivePathNode.from_path(p, known_paths) for p in path.iterdir()]
+            [
+                _RecursivePathNode.from_path(p, known_paths, session)
+                for p in path.iterdir()
+            ]
             if path.is_dir()
+            # Do not collect sub files and folders for ignored folders.
+            and not session.hook.pytask_ignore_collect(path=path, config=session.config)
             else []
         )
-        is_unknown = (path.is_file() and path not in known_paths) or (
-            path.is_dir() and all(node.is_unknown for node in sub_nodes)
+
+        is_unknown_file = path.is_file() and not (
+            path in known_paths
+            # Ignored files are also known.
+            or session.hook.pytask_ignore_collect(path=path, config=session.config)
+        )
+        is_unknown_directory = (
+            path.is_dir()
+            # True for folders and ignored folders without any sub nodes.
+            and all(node.is_unknown for node in sub_nodes)
+            # True for not ignored paths.
+            and not session.hook.pytask_ignore_collect(path=path, config=session.config)
         )
+        is_unknown = is_unknown_file or is_unknown_directory
+
         return cls(path, sub_nodes, path.is_dir(), path.is_file(), is_unknown)
 
     def __repr__(self):

src/_pytask/config.py

@@ -18,12 +18,19 @@
 
 hookimpl = pluggy.HookimplMarker("pytask")
 
-
 IGNORED_FOLDERS = [
     ".git/*",
     ".hg/*",
     ".svn/*",
     ".venv/*",
+]
+
+IGNORED_FILES = [".pre-commit-config.yaml", ".readthedocs.yml", ".codecov.yml"]
+
+
+IGNORED_FILES_AND_FOLDERS = IGNORED_FILES + IGNORED_FOLDERS
+
+IGNORED_TEMPORARY_FILES_AND_FOLDERS = [
     "*.egg-info/*",
     ".ipynb_checkpoints/*",
     ".mypy_cache/*",
@@ -33,8 +40,8 @@
     "__pycache__/*",
     "build/*",
     "dist/*",
+    "pytest_cache/*",
 ]
-"""List[str]: List of expressions to ignore folders."""
 
 
 @hookimpl
@@ -93,6 +100,8 @@ def pytask_configure(pm, config_from_cli):
 @hookimpl
 def pytask_parse_config(config, config_from_cli, config_from_file):
     """Parse the configuration."""
+    config["command"] = config_from_cli.get("command", "build")
+
     config_from_file["ignore"] = parse_value_or_multiline_option(
         config_from_file.get("ignore")
     )
@@ -105,7 +114,8 @@ def pytask_parse_config(config, config_from_cli, config_from_file):
                 default=[],
             )
         )
-        + IGNORED_FOLDERS
+        + IGNORED_FILES_AND_FOLDERS
+        + IGNORED_TEMPORARY_FILES_AND_FOLDERS
     )
 
     config["debug_pytask"] = get_first_non_none_value(

src/_pytask/mark/__init__.py

@@ -42,6 +42,8 @@
 )
 def markers(**config_from_cli):
     """Show all registered markers."""
+    config_from_cli["command"] = "markers"
+
     try:
         # Duplication of the same mechanism in :func:`pytask.main.main`.
         pm = get_plugin_manager()
@@ -96,7 +98,9 @@ def pytask_extend_command_line_interface(cli: click.Group) -> None:
         ),
     ]
     cli.commands["build"].params.extend(additional_build_parameters)
+    cli.commands["clean"].params.extend(additional_build_parameters)
     cli.commands["collect"].params.extend(additional_build_parameters)
+
     cli.add_command(markers)
 
 

tests/test_clean.py

@@ -6,6 +6,7 @@
 
 @pytest.fixture()
 def sample_project_path(tmp_path):
+    """Create a sample project to be cleaned."""
     source = """
     import pytask
 
@@ -24,6 +25,39 @@ def task_dummy(produces):
     return tmp_path
 
 
+@pytest.mark.end_to_end
+def test_clean_with_ignored_file(sample_project_path, runner):
+    result = runner.invoke(
+        cli, ["clean", "--ignore", "*_1.txt", sample_project_path.as_posix()]
+    )
+
+    assert "to_be_deleted_file_1.txt" not in result.output
+    assert "to_be_deleted_file_2.txt" in result.output
+
+
+@pytest.mark.end_to_end
+def test_clean_with_ingored_directory(sample_project_path, runner):
+    result = runner.invoke(
+        cli,
+        [
+            "clean",
+            "--ignore",
+            "to_be_deleted_folder_1/*",
+            sample_project_path.as_posix(),
+        ],
+    )
+
+    assert "to_be_deleted_folder_1/" not in result.output
+    assert "to_be_deleted_file_1.txt" in result.output
+
+
+@pytest.mark.end_to_end
+def test_clean_with_nothing_to_remove(tmp_path, runner):
+    result = runner.invoke(cli, ["clean", "--ignore", "*", tmp_path.as_posix()])
+
+    assert "There are no files and directories which can be deleted." in result.output
+
+
 @pytest.mark.end_to_end
 def test_clean_dry_run(sample_project_path, runner):
     result = runner.invoke(cli, ["clean", sample_project_path.as_posix()])

tests/test_config.py

@@ -9,7 +9,6 @@
 
 
 _IGNORED_FOLDERS = [i.split("/")[0] for i in IGNORED_FOLDERS]
-_IGNORED_FOLDERS.remove("*.egg-info")
 
 
 @pytest.mark.unit