Implement pytask collect. (#31)

tobiasraabe · web-flow · commit 5ddf29c45663 · 2020-10-05T21:39:19.000+02:00
diff --git a/.conda/meta.yaml b/.conda/meta.yaml
@@ -5,7 +5,7 @@ package:
   version: {{ data.get('version') }}
 
 source:
-  # git_url is nice in that it won't capture devenv stuff.  However, it only captures
+  # git_url is nice in that it won't capture devenv stuff. However, it only captures
   # committed code, so pay attention.
   git_url: ../
 
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -8,7 +8,7 @@ repos:
     -   id: check-yaml
         exclude: meta.yaml
     -   id: debug-statements
-        exclude: (debugging\.py|build\.py|clean\.py|mark/__init__\.py)
+        exclude: (debugging\.py|build\.py|clean\.py|mark/__init__\.py|collect_command\.py)
     -   id: end-of-file-fixer
 -   repo: https://github.com/asottile/pyupgrade
     rev: v2.7.2
diff --git a/README.rst b/README.rst
@@ -4,6 +4,8 @@
 
 ------
 
+.. start-badges
+
 .. image:: https://anaconda.org/pytask/pytask/badges/version.svg
     :target: https://anaconda.org/pytask/pytask
 
@@ -23,6 +25,9 @@
     :target: https://github.com/psf/black
 
 
+.. end-badges
+
+
 Features
 --------
 
@@ -52,6 +57,8 @@ explanations/why_do_i_need_a_build_system.html>`_ if you do not know or are not
 convinced that you need a build system.
 
 
+.. start-installation
+
 Installation
 ------------
 
@@ -63,6 +70,7 @@ package with
     $ conda config --add channels conda-forge --add channels pytask
     $ conda install pytask
 
+.. end-installation
 
 Usage
 -----
diff --git a/docs/changes.rst b/docs/changes.rst
@@ -11,6 +11,7 @@ all releases are available on `Anaconda.org <https://anaconda.org/pytask/pytask>
 
 - :gh:`30` fixes or adds the session object to some hooks which was missing from the
   previous release.
+- :gh:`31` adds ``pytask collect`` to show information on collected tasks.
 
 
 0.0.7 - 2020-10-03
diff --git a/docs/index.rst b/docs/index.rst
@@ -1,34 +1,14 @@
 pytask
 ======
 
-.. image:: https://anaconda.org/pytask/pytask/badges/version.svg
-    :target: https://anaconda.org/pytask/pytask
+.. include:: ../README.rst
+   :start-after: start-badges
+   :end-before: end-badges
 
-.. image:: https://anaconda.org/pytask/pytask/badges/platforms.svg
-    :target: https://anaconda.org/pytask/pytask
 
-.. image:: https://readthedocs.org/projects/pytask-dev/badge/?version=latest
-    :target: https://pytask-dev.readthedocs.io/en/latest
-
-.. image:: https://github.com/pytask-dev/pytask/workflows/Continuous%20Integration%20Workflow/badge.svg?branch=main
-    :target: https://github.com/pytask-dev/pytask/actions?query=branch%3Amain
-
-
-.. image:: https://codecov.io/gh/pytask-dev/pytask/branch/main/graph/badge.svg
-    :target: https://codecov.io/gh/pytask-dev/pytask
-
-.. image:: https://img.shields.io/badge/code%20style-black-000000.svg
-    :target: https://github.com/psf/black
-
-----
-
-This is the documentation of pytask. You can install the package from `Anaconda.org
-<https://anaconda.org/pytask/pytask>`_ with
-
-.. code-block:: console
-
-    $ conda config --add channels conda-forge --add channels pytask
-    $ conda install pytask
+.. include:: ../README.rst
+   :start-after: start-installation
+   :end-before: end-installation
 
 The documentation has four central parts.
 
diff --git a/docs/reference_guides/hookspecs.rst b/docs/reference_guides/hookspecs.rst
@@ -38,6 +38,10 @@ Command Line Interface
 Configuration
 -------------
 
+The following hooks are used to configure pytask with the information from the command
+line and the configuration file as well as making sure that all options play nicely
+together.
+
 .. autofunction:: pytask_configure
    :noindex:
 
@@ -51,6 +55,8 @@ Configuration
 Collection
 ----------
 
+The following hooks traverse directories and collect tasks from files.
+
 .. autofunction:: pytask_collect
    :noindex:
 
@@ -99,9 +105,41 @@ duplicates are collected with :func:`pytask_collect_task`.
    :noindex:
 
 
+Resolving Dependencies
+----------------------
+
+The following hooks are designed to build a DAG from tasks and dependencies and check
+which files have changed and need to be re-run.
+
+.. warning::
+
+   This step is still experimental and likely to change in the future. If you are
+   planning to write a plugin which extends pytask in this dimension, please, start a
+   discussion before writing a plugin. It may make your life easier if changes in pytask
+   anticipate your plugin.
+
+
+.. autofunction:: pytask_resolve_dependencies
+   :noindex:
+
+.. autofunction:: pytask_resolve_dependencies_create_dag
+   :noindex:
+
+.. autofunction:: pytask_resolve_dependencies_validate_dag
+   :noindex:
+
+.. autofunction:: pytask_resolve_dependencies_select_execution_dag
+   :noindex:
+
+.. autofunction:: pytask_resolve_dependencies_log
+   :noindex:
+
+
 Execution
 ---------
 
+The following hooks execute the tasks and log information on the result in the terminal.
+
 .. autofunction:: pytask_execute
    :noindex:
 
diff --git a/docs/tutorials/how_to_clean.rst b/docs/tutorials/how_to_clean.rst
@@ -10,7 +10,7 @@ and type
 .. code-block:: console
 
     $ pytask clean
-    =============================== Start pytask session ===============================
+    ========================= Start pytask session =========================
     Platform: win32 -- Python x.y.z, pytask x.y.z, pluggy x.y.z
     Root: .
     Collected 3 task(s).
@@ -37,7 +37,7 @@ If you want to delete whole folders instead of only the files in them, use
 .. code-block:: console
 
     $ pytask clean -d
-    =============================== Start pytask session ===============================
+    ========================= Start pytask session =========================
     Platform: win32 -- Python x.y.z, pytask x.y.z, pluggy x.y.z
     Root: .
     Collected 3 task(s).
@@ -48,9 +48,7 @@ If you want to delete whole folders instead of only the files in them, use
     Would remove C:\Users\project\obsolete_folder.
 
 
-Command line options
---------------------
+Further reading
+---------------
 
-.. click:: _pytask.cli:cli
-    :commands: clean
-    :prog: pytask
+- :program:`pytask clean` in :doc:`../reference_guides/command_line_interface`.
diff --git a/docs/tutorials/how_to_collect.rst b/docs/tutorials/how_to_collect.rst
@@ -0,0 +1,62 @@
+How to collect tasks
+====================
+
+If you want to inspect your project and see a summary of all the tasks in the projects,
+you can use the :program:`pytask collect` command.
+
+For example, let us take the following task
+
+.. code-block:: python
+
+    # Content of task_dummy.py
+
+    import pytask
+
+
+    @pytask.mark.depends_on("in.txt")
+    @pytask.mark.produces("out.txt")
+    def task_write_file(depends_on, produces):
+        produces.write_text(depends_on.read_text())
+
+
+Now, running :program:`pytask collect` will produce the following output.
+
+.. code-block:: console
+
+    $ pytask collect
+    ========================= Start pytask session =========================
+    Platform: linux -- Python 3.x.y, pytask 0.x.y, pluggy 0.x.y
+    Root: xxx
+    Collected 1 task(s).
+
+    <Module /.../task_dummy.py>
+      <Function task_write_file>
+
+    ========================================================================
+
+If you want to have more information regarding dependencies and products of the task,
+append the ``--nodes`` flag.
+
+.. code-block:: console
+
+    $ pytask collect
+    ========================= Start pytask session =========================
+    Platform: linux -- Python 3.x.y, pytask 0.x.y, pluggy 0.x.y
+    Root: xxx
+    Collected 1 task(s).
+
+    <Module /.../task_dummy.py>
+      <Function task_write_file>
+        <Dependency /.../in.txt>
+        <Product /.../out.txt>
+
+    ========================================================================
+
+To restrict the set of tasks you are looking at, use markers, expression and ignore
+patterns as usual.
+
+
+Further reading
+---------------
+
+- :program:`pytask collect` in :doc:`../reference_guides/command_line_interface`.
diff --git a/docs/tutorials/index.rst b/docs/tutorials/index.rst
@@ -16,4 +16,5 @@ organize and start your own project.
    how_to_configure_pytask
    how_to_select_tasks
    how_to_clean
+   how_to_collect
    how_to_make_tasks_persist
diff --git a/src/_pytask/clean.py b/src/_pytask/clean.py
@@ -75,11 +75,11 @@ def clean(**config_from_cli):
         session = Session.from_config(config)
 
     except Exception:
-        traceback.print_exception(*sys.exc_info())
         session = Session({}, None)
         session.exit_code = ExitCode.CONFIGURATION_FAILED
 
         if config_from_cli.get("pdb"):
+            traceback.print_exception(*sys.exc_info())
             pdb.post_mortem()
 
     else:
@@ -112,13 +112,16 @@ def clean(**config_from_cli):
                         else:
                             path.unlink()
 
-            click.echo("")
+            click.echo("\n" + "=" * config["terminal_width"])
 
         except CollectionError:
             session.exit_code = ExitCode.COLLECTION_FAILED
 
         except Exception:
             traceback.print_exception(*sys.exc_info())
+            if config["pdb"]:
+                pdb.post_mortem()
+
             session.exit_code = ExitCode.FAILED
 
     sys.exit(session.exit_code)
diff --git a/src/_pytask/cli.py b/src/_pytask/cli.py
@@ -39,6 +39,7 @@ def pytask_add_hooks(pm):
     from _pytask import build
     from _pytask import clean
     from _pytask import collect
+    from _pytask import collect_command
     from _pytask import config
     from _pytask import database
     from _pytask import debugging
@@ -53,6 +54,7 @@ def pytask_add_hooks(pm):
     pm.register(build)
     pm.register(clean)
     pm.register(collect)
+    pm.register(collect_command)
     pm.register(config)
     pm.register(database)
     pm.register(debugging)
diff --git a/src/_pytask/collect_command.py b/src/_pytask/collect_command.py
diff --git a/src/_pytask/hookspecs.py b/src/_pytask/hookspecs.py
diff --git a/src/_pytask/mark/__init__.py b/src/_pytask/mark/__init__.py
diff --git a/tests/test_collect_command.py b/tests/test_collect_command.py
diff --git a/tox.ini b/tox.ini
