Skip to content

Commit 79cb0cd

Browse files
authored
Shorten task ids in execution error reports. (#41)
1 parent 64d457a commit 79cb0cd

12 files changed

+319
-121
lines changed

.github/workflows/continuous-integration-workflow.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,11 @@ jobs:
1818
fail-fast: false
1919
matrix:
2020
os: ['ubuntu-latest', 'macos-latest', 'windows-latest']
21-
python-version: ['3.6', '3.7', '3.8']
21+
python-version: ['3.6', '3.7', '3.8', '3.9']
2222

2323
steps:
2424
- uses: actions/checkout@v2
25-
- uses: goanpeca/setup-miniconda@v1
25+
- uses: conda-incubator/setup-miniconda@v2
2626
with:
2727
auto-update-conda: true
2828
python-version: ${{ matrix.python-version }}
@@ -88,7 +88,7 @@ jobs:
8888

8989
steps:
9090
- uses: actions/checkout@v2
91-
- uses: goanpeca/setup-miniconda@v1
91+
- uses: conda-incubator/setup-miniconda@v2
9292
with:
9393
auto-update-conda: true
9494
python-version: 3.8

.pre-commit-config.yaml

+4-4
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
repos:
22
- repo: https://github.com/pre-commit/pre-commit-hooks
3-
rev: v3.2.0
3+
rev: v3.3.0
44
hooks:
55
- id: check-added-large-files
66
args: ['--maxkb=25']
@@ -11,12 +11,12 @@ repos:
1111
exclude: (debugging\.py|build\.py|clean\.py|mark/__init__\.py|collect_command\.py)
1212
- id: end-of-file-fixer
1313
- repo: https://github.com/asottile/pyupgrade
14-
rev: v2.7.2
14+
rev: v2.7.3
1515
hooks:
1616
- id: pyupgrade
1717
args: [--py36-plus]
1818
- repo: https://github.com/asottile/reorder_python_imports
19-
rev: v2.3.5
19+
rev: v2.3.6
2020
hooks:
2121
- id: reorder-python-imports
2222
- repo: https://github.com/psf/black
@@ -52,7 +52,7 @@ repos:
5252
hooks:
5353
- id: doc8
5454
- repo: https://github.com/econchick/interrogate
55-
rev: 1.3.1
55+
rev: 1.3.2
5656
hooks:
5757
- id: interrogate
5858
args: [-v, --fail-under=40, src, tests]

docs/changes.rst

+1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ all releases are available on `Anaconda.org <https://anaconda.org/pytask/pytask>
1919
- :gh:`38` allows to pass dictionaries as dependencies and products and inside the
2020
function ``depends_on`` and ``produces`` become dictionaries.
2121
- :gh:`39` releases v0.0.9.
22+
- :gh:`41` shortens the task ids in the error reports for better readability.
2223

2324

2425
0.0.8 - 2020-10-04

environment.yml

+1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ dependencies:
1919
- pytest
2020

2121
# Misc
22+
- black
2223
- bump2version
2324
- jupyterlab
2425
- matplotlib

src/_pytask/collect.py

+24-20
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,8 @@
1313
from _pytask.mark import has_marker
1414
from _pytask.nodes import FilePathNode
1515
from _pytask.nodes import PythonFunctionTask
16+
from _pytask.nodes import shorten_node_name
1617
from _pytask.report import CollectionReport
17-
from _pytask.report import CollectionReportFile
18-
from _pytask.report import CollectionReportTask
1918
from _pytask.report import format_collect_footer
2019

2120

@@ -30,9 +29,7 @@ def pytask_collect(session):
3029
try:
3130
session.hook.pytask_collect_modify_tasks(session=session, tasks=tasks)
3231
except Exception:
33-
report = CollectionReport(
34-
" Modification of collected tasks failed ", sys.exc_info()
35-
)
32+
report = CollectionReport.from_exception(exc_info=sys.exc_info())
3633
reports.append(report)
3734

3835
session.collection_reports = reports
@@ -86,8 +83,8 @@ def pytask_collect_file_protocol(session, path, reports):
8683
session=session, path=path, reports=reports
8784
)
8885
except Exception:
89-
exc_info = sys.exc_info()
90-
reports = [CollectionReportFile.from_exception(path, exc_info)]
86+
node = FilePathNode.from_path(path)
87+
reports = [CollectionReport.from_exception(node=node, exc_info=sys.exc_info())]
9188

9289
return reports
9390

@@ -99,7 +96,7 @@ def pytask_collect_file(session, path, reports):
9996
spec = importlib.util.spec_from_file_location(path.stem, str(path))
10097

10198
if spec is None:
102-
raise ImportError(f"Can't find module {path.stem} at location {path}.")
99+
raise ImportError(f"Can't find module '{path.stem}' at location {path}.")
103100

104101
mod = importlib.util.module_from_spec(spec)
105102
spec.loader.exec_module(mod)
@@ -135,11 +132,12 @@ def pytask_collect_task_protocol(session, path, name, obj):
135132
)
136133
if task is not None:
137134
session.hook.pytask_collect_task_teardown(session=session, task=task)
138-
return CollectionReportTask.from_task(task)
135+
return CollectionReport.from_node(task)
139136

140137
except Exception:
141-
exc_info = sys.exc_info()
142-
return CollectionReportTask.from_exception(path, name, exc_info)
138+
return CollectionReport.from_exception(
139+
exc_info=sys.exc_info(), node=locals().get("task")
140+
)
143141

144142

145143
@hookimpl(trylast=True)
@@ -211,7 +209,7 @@ def valid_paths(paths, session):
211209

212210
def _extract_successful_tasks_from_reports(reports):
213211
"""Extract successful tasks from reports."""
214-
return [i.task for i in reports if i.successful]
212+
return [i.node for i in reports if i.successful]
215213

216214

217215
@hookimpl
@@ -233,20 +231,26 @@ def pytask_collect_log(session, reports, tasks):
233231
click.echo(f"{{:=^{tm_width}}}".format(" Failures during collection "))
234232

235233
for report in failed_reports:
236-
click.echo(f"{{:_^{tm_width}}}".format(report.format_title()))
234+
if report.node is None:
235+
header = " Error "
236+
else:
237+
shortened_name = shorten_node_name(report.node, session.config["paths"])
238+
header = f" Could not collect {shortened_name} "
239+
240+
click.echo(f"{{:_^{tm_width}}}".format(header))
237241

238242
click.echo("")
239243

240244
traceback.print_exception(*report.exc_info)
241245

242246
click.echo("")
243247

244-
duration = round(session.collection_end - session.collection_start, 2)
245-
click.echo(
246-
format_collect_footer(
247-
len(tasks), len(failed_reports), n_deselected, duration, tm_width
248-
),
249-
nl=True,
250-
)
248+
duration = round(session.collection_end - session.collection_start, 2)
249+
click.echo(
250+
format_collect_footer(
251+
len(tasks), len(failed_reports), n_deselected, duration, tm_width
252+
),
253+
nl=True,
254+
)
251255

252256
raise CollectionError

src/_pytask/execute.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
from _pytask.exceptions import NodeNotFoundError
1414
from _pytask.mark import Mark
1515
from _pytask.nodes import FilePathNode
16+
from _pytask.nodes import shorten_node_name
1617
from _pytask.report import ExecutionReport
1718
from _pytask.report import format_execute_footer
1819

@@ -167,7 +168,8 @@ def pytask_execute_log_end(session, reports):
167168
for report in reports:
168169
if not report.success:
169170

170-
message = f" Task {report.task.name} failed "
171+
task_name = shorten_node_name(report.task, session.config["paths"])
172+
message = f" Task {task_name} failed "
171173
if len(message) > tm_width:
172174
click.echo("_" * tm_width)
173175
click.echo(message)

src/_pytask/nodes.py

+106-17
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
from pathlib import Path
99
from typing import Any
1010
from typing import Iterable
11+
from typing import List
1112
from typing import Union
1213

1314
import attr
@@ -45,24 +46,30 @@ def produces(objects: Union[Any, Iterable[Any]]) -> Union[Any, Iterable[Any]]:
4546
return objects
4647

4748

48-
class MetaTask(metaclass=ABCMeta):
49-
"""The base class for tasks."""
49+
class MetaNode(metaclass=ABCMeta):
50+
"""Meta class for nodes."""
5051

5152
@abstractmethod
52-
def execute(self):
53-
"""Execute the task."""
53+
def state(self):
54+
"""Return a value which indicates whether a node has changed or not."""
5455
pass
5556

57+
58+
class MetaTask(MetaNode):
59+
"""The base class for tasks."""
60+
5661
@abstractmethod
57-
def state(self):
58-
"""Return a value to check whether the task definition has changed."""
62+
def execute(self):
63+
"""Execute the task."""
5964
pass
6065

6166

6267
@attr.s
6368
class PythonFunctionTask(MetaTask):
6469
"""The class for tasks which are Python functions."""
6570

71+
base_name = attr.ib(type=str)
72+
"""str: The base name of the task."""
6673
name = attr.ib(type=str)
6774
"""str: The unique identifier for a task."""
6875
path = attr.ib(type=Path)
@@ -95,8 +102,9 @@ def from_path_name_function_session(cls, path, name, function, session):
95102
]
96103

97104
return cls(
105+
base_name=name,
106+
name=_create_task_name(path, name),
98107
path=path,
99-
name=path.as_posix() + "::" + name,
100108
function=function,
101109
depends_on=dependencies,
102110
produces=products,
@@ -134,15 +142,6 @@ def add_report_section(self, when: str, key: str, content: str):
134142
self._report_sections.append((when, key, content))
135143

136144

137-
class MetaNode(metaclass=ABCMeta):
138-
"""Meta class for nodes."""
139-
140-
@abstractmethod
141-
def state(self):
142-
"""Return a value which indicates whether a node has changed or not."""
143-
pass
144-
145-
146145
@attr.s
147146
class FilePathNode(MetaNode):
148147
"""The class for a node which is a path."""
@@ -153,6 +152,9 @@ class FilePathNode(MetaNode):
153152
value = attr.ib()
154153
"""Any: Value passed to the decorator which can be requested inside the function."""
155154

155+
path = attr.ib()
156+
"""pathlib.Path: Path to the FilePathNode."""
157+
156158
@classmethod
157159
@functools.lru_cache()
158160
def from_path(cls, path: pathlib.Path):
@@ -162,7 +164,7 @@ def from_path(cls, path: pathlib.Path):
162164
163165
"""
164166
path = path.resolve()
165-
return cls(path.as_posix(), path)
167+
return cls(path.as_posix(), path, path)
166168

167169
def state(self):
168170
"""Return the last modified date for file path."""
@@ -280,3 +282,90 @@ def _convert_nodes_to_dictionary(list_of_tuples):
280282
nodes[node_name] = tuple_[0]
281283

282284
return nodes
285+
286+
287+
def _create_task_name(path: Path, base_name: str):
288+
"""Create the name of a task from a path and the task's base name.
289+
290+
Examples
291+
--------
292+
>>> from pathlib import Path
293+
>>> _create_task_name(Path("module.py"), "task_dummy")
294+
'module.py::task_dummy'
295+
296+
"""
297+
return path.as_posix() + "::" + base_name
298+
299+
300+
def _relative_to(path: Path, source: Path, include_source: bool = True):
301+
"""Make a path relative to another path.
302+
303+
In contrast to :meth:`pathlib.Path.relative_to`, this function allows to keep the
304+
name of the source path.
305+
306+
Examples
307+
--------
308+
>>> from pathlib import Path
309+
>>> _relative_to(Path("folder", "file.py"), Path("folder")).as_posix()
310+
'folder/file.py'
311+
>>> _relative_to(Path("folder", "file.py"), Path("folder"), False).as_posix()
312+
'file.py'
313+
314+
"""
315+
return Path(source.name if include_source else "", path.relative_to(source))
316+
317+
318+
def _find_closest_ancestor(path: Path, potential_ancestors: List[Path]):
319+
"""Find the closest ancestor of a path.
320+
321+
Examples
322+
--------
323+
>>> from pathlib import Path
324+
>>> _find_closest_ancestor(Path("folder", "file.py"), [Path("folder")]).as_posix()
325+
'folder'
326+
327+
>>> paths = [Path("folder"), Path("folder", "subfolder")]
328+
>>> _find_closest_ancestor(Path("folder", "subfolder", "file.py"), paths).as_posix()
329+
'folder/subfolder'
330+
331+
"""
332+
closest_ancestor = None
333+
for ancestor in potential_ancestors:
334+
if ancestor == path:
335+
closest_ancestor = path
336+
break
337+
if ancestor in path.parents:
338+
if closest_ancestor is None or (
339+
len(path.relative_to(ancestor).parts)
340+
< len(path.relative_to(closest_ancestor).parts)
341+
):
342+
closest_ancestor = ancestor
343+
344+
return closest_ancestor
345+
346+
347+
def shorten_node_name(node, paths: List[Path]):
348+
"""Shorten the node name.
349+
350+
The whole name of the node - which includes the drive letter - can be very long
351+
when using nested folder structures in bigger projects.
352+
353+
Thus, the part of the name which contains the path is replace by the relative
354+
path from one path in ``session.config["paths"]`` to the node.
355+
356+
"""
357+
ancestor = _find_closest_ancestor(node.path, paths)
358+
if ancestor is None:
359+
raise ValueError("A node must be defined in a child of 'paths'.")
360+
elif isinstance(node, MetaTask):
361+
if ancestor == node.path:
362+
name = _create_task_name(Path(node.path.name), node.base_name)
363+
else:
364+
shortened_path = _relative_to(node.path, ancestor)
365+
name = _create_task_name(shortened_path, node.base_name)
366+
elif isinstance(node, MetaNode):
367+
name = _relative_to(node.path, ancestor).as_posix()
368+
else:
369+
raise ValueError(f"Unknown node {node} with type '{type(node)}'.")
370+
371+
return name

0 commit comments

Comments
 (0)