Skip to content

Commit f79bce1

Browse files
mr-cGlassOfWhiskey
andauthored
implement the loop construct extension (#1641)
Loop construct prototype implemented as an extension, with tests Based upon @GlassOfWhiskey 's work in common-workflow-language/common-workflow-language#495 (comment) With comments from @tetron @mr-c Co-authored-by: GlassOfWhiskey <[email protected]>
1 parent 0f4459c commit f79bce1

35 files changed

+1918
-30
lines changed

MANIFEST.in

+2
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ include *requirements.txt mypy.ini tox.ini
55
include gittaggers.py Makefile cwltool.py
66
recursive-include mypy-stubs *.pyi *.py
77
include tests/*
8+
include tests/loop/*
89
include tests/tmp1/tmp2/tmp3/.gitkeep
910
include tests/tmp4/alpha/*
1011
include tests/wf/*
@@ -54,6 +55,7 @@ include cwltool/schemas/v1.2/salad/schema_salad/metaschema/*.yml
5455
include cwltool/schemas/v1.2/salad/schema_salad/metaschema/*.md
5556
include cwltool/extensions.yml
5657
include cwltool/extensions-v1.1.yml
58+
include cwltool/extensions-v1.2.yml
5759
include cwltool/jshint/jshint_wrapper.js
5860
include cwltool/jshint/jshint.js
5961
include cwltool/hello.simg

cwltool/checker.py

+51
Original file line numberDiff line numberDiff line change
@@ -389,6 +389,11 @@ def check_all_types(
389389
message="Source is from conditional step, but pickValue is not used",
390390
)
391391
)
392+
if is_all_output_method_loop_step(param_to_step, parm_id):
393+
src_dict[parm_id]["type"] = {
394+
"type": "array",
395+
"items": src_dict[parm_id]["type"],
396+
}
392397
else:
393398
parm_id = cast(str, sink[sourceField])
394399
if parm_id not in src_dict:
@@ -430,6 +435,12 @@ def check_all_types(
430435

431436
srcs_of_sink[0]["type"] = src_typ
432437

438+
if is_all_output_method_loop_step(param_to_step, parm_id):
439+
src_dict[parm_id]["type"] = {
440+
"type": "array",
441+
"items": src_dict[parm_id]["type"],
442+
}
443+
433444
for src in srcs_of_sink:
434445
check_result = check_types(src, sink, linkMerge, valueFrom)
435446
if check_result == "warning":
@@ -517,3 +528,43 @@ def is_conditional_step(param_to_step: Dict[str, CWLObjectType], parm_id: str) -
517528
if source_step.get("when") is not None:
518529
return True
519530
return False
531+
532+
533+
def is_all_output_method_loop_step(
534+
param_to_step: Dict[str, CWLObjectType], parm_id: str
535+
) -> bool:
536+
"""Check if a step contains a http://commonwl.org/cwltool#Loop requirement with `all` outputMethod."""
537+
source_step: Optional[MutableMapping[str, Any]] = param_to_step.get(parm_id)
538+
if source_step is not None:
539+
for requirement in source_step.get("requirements", []):
540+
if (
541+
requirement["class"] == "http://commonwl.org/cwltool#Loop"
542+
and requirement.get("outputMethod") == "all"
543+
):
544+
return True
545+
return False
546+
547+
548+
def loop_checker(steps: List[MutableMapping[str, Any]]) -> None:
549+
"""Check http://commonwl.org/cwltool#Loop requirement compatibility with other directives."""
550+
exceptions = []
551+
for step in steps:
552+
requirements = {
553+
**{h["class"]: h for h in step.get("hints", [])},
554+
**{r["class"]: r for r in step.get("requirements", [])},
555+
}
556+
if "http://commonwl.org/cwltool#Loop" in requirements:
557+
if "when" in step:
558+
exceptions.append(
559+
SourceLine(step, "id").makeError(
560+
"The `cwltool:Loop` clause is not compatible with the `when` directive."
561+
)
562+
)
563+
if "scatter" in step:
564+
exceptions.append(
565+
SourceLine(step, "id").makeError(
566+
"The `cwltool:Loop` clause is not compatible with the `scatter` directive."
567+
)
568+
)
569+
if exceptions:
570+
raise ValidationException("\n".join(exceptions))

cwltool/extensions-v1.2.yml

+242
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,242 @@
1+
$base: http://commonwl.org/cwltool#
2+
$namespaces:
3+
cwl: "https://w3id.org/cwl/cwl#"
4+
$graph:
5+
- $import: https://w3id.org/cwl/CommonWorkflowLanguage.yml
6+
7+
- name: Secrets
8+
type: record
9+
inVocab: false
10+
extends: cwl:ProcessRequirement
11+
fields:
12+
class:
13+
type: string
14+
doc: "Always 'Secrets'"
15+
jsonldPredicate:
16+
"_id": "@type"
17+
"_type": "@vocab"
18+
secrets:
19+
type: string[]
20+
doc: |
21+
List one or more input parameters that are sensitive (such as passwords)
22+
which will be deliberately obscured from logging.
23+
jsonldPredicate:
24+
"_type": "@id"
25+
refScope: 0
26+
27+
28+
- name: ProcessGenerator
29+
type: record
30+
inVocab: true
31+
extends: cwl:Process
32+
documentRoot: true
33+
fields:
34+
- name: class
35+
jsonldPredicate:
36+
"_id": "@type"
37+
"_type": "@vocab"
38+
type: string
39+
- name: run
40+
type: [string, cwl:Process]
41+
jsonldPredicate:
42+
_id: "cwl:run"
43+
_type: "@id"
44+
subscope: run
45+
doc: |
46+
Specifies the process to run.
47+
48+
- name: MPIRequirement
49+
type: record
50+
inVocab: false
51+
extends: cwl:ProcessRequirement
52+
doc: |
53+
Indicates that a process requires an MPI runtime.
54+
fields:
55+
- name: class
56+
type: string
57+
doc: "Always 'MPIRequirement'"
58+
jsonldPredicate:
59+
"_id": "@type"
60+
"_type": "@vocab"
61+
- name: processes
62+
type: [int, cwl:Expression]
63+
doc: |
64+
The number of MPI processes to start. If you give a string,
65+
this will be evaluated as a CWL Expression and it must
66+
evaluate to an integer.
67+
68+
- name: CUDARequirement
69+
type: record
70+
extends: cwl:ProcessRequirement
71+
inVocab: false
72+
doc: |
73+
Require support for NVIDA CUDA (GPU hardware acceleration).
74+
fields:
75+
class:
76+
type: string
77+
doc: 'cwltool:CUDARequirement'
78+
jsonldPredicate:
79+
_id: "@type"
80+
_type: "@vocab"
81+
cudaVersionMin:
82+
type: string
83+
doc: |
84+
Minimum CUDA version to run the software, in X.Y format. This
85+
corresponds to a CUDA SDK release. When running directly on
86+
the host (not in a container) the host must have a compatible
87+
CUDA SDK (matching the exact version, or, starting with CUDA
88+
11.3, matching major version). When run in a container, the
89+
container image should provide the CUDA runtime, and the host
90+
driver is injected into the container. In this case, because
91+
CUDA drivers are backwards compatible, it is possible to
92+
use an older SDK with a newer driver across major versions.
93+
94+
See https://docs.nvidia.com/deploy/cuda-compatibility/ for
95+
details.
96+
cudaComputeCapability:
97+
type:
98+
- 'string'
99+
- 'string[]'
100+
doc: |
101+
CUDA hardware capability required to run the software, in X.Y
102+
format.
103+
104+
* If this is a single value, it defines only the minimum
105+
compute capability. GPUs with higher capability are also
106+
accepted.
107+
108+
* If it is an array value, then only select GPUs with compute
109+
capabilities that explicitly appear in the array.
110+
cudaDeviceCountMin:
111+
type: ['null', int, cwl:Expression]
112+
default: 1
113+
doc: |
114+
Minimum number of GPU devices to request. If not specified,
115+
same as `cudaDeviceCountMax`. If neither are specified,
116+
default 1.
117+
cudaDeviceCountMax:
118+
type: ['null', int, cwl:Expression]
119+
doc: |
120+
Maximum number of GPU devices to request. If not specified,
121+
same as `cudaDeviceCountMin`.
122+
123+
- name: LoopInput
124+
type: record
125+
fields:
126+
id:
127+
type: string?
128+
jsonldPredicate: "@id"
129+
doc: "It must reference the `id` of one of the elements in the `in` field of the step."
130+
loopSource:
131+
doc: |
132+
Specifies one or more of the step output parameters that will
133+
provide input to the loop iterations after the first one (inputs
134+
of the first iteration are the step input parameters).
135+
type:
136+
- string?
137+
- string[]?
138+
jsonldPredicate:
139+
"_type": "@id"
140+
refScope: 1
141+
linkMerge:
142+
type: cwl:LinkMergeMethod?
143+
jsonldPredicate: "cwl:linkMerge"
144+
default: merge_nested
145+
doc: |
146+
The method to use to merge multiple inbound links into a single array.
147+
If not specified, the default method is "merge_nested".
148+
pickValue:
149+
type: ["null", cwl:PickValueMethod]
150+
jsonldPredicate: "cwl:pickValue"
151+
doc: |
152+
The method to use to choose non-null elements among multiple sources.
153+
default:
154+
type: ["null", Any]
155+
doc: |
156+
The default value for this parameter to use if either there is no
157+
`source` field, or the value produced by the `source` is `null`. The
158+
default must be applied prior to scattering or evaluating `valueFrom`.
159+
jsonldPredicate:
160+
_id: "sld:default"
161+
noLinkCheck: true
162+
valueFrom:
163+
type:
164+
- "null"
165+
- string
166+
- cwl:Expression
167+
jsonldPredicate: "cwl:valueFrom"
168+
doc: |
169+
To use valueFrom, [StepInputExpressionRequirement](#StepInputExpressionRequirement) must
170+
be specified in the workflow or workflow step requirements.
171+
172+
If `valueFrom` is a constant string value, use this as the value for
173+
this input parameter.
174+
175+
If `valueFrom` is a parameter reference or expression, it must be
176+
evaluated to yield the actual value to be assigned to the input field.
177+
178+
The `self` value in the parameter reference or expression must be
179+
`null` if there is no `loopSource` field, or the value of the
180+
parameter(s) specified in the `loopSource` field.
181+
182+
The value of `inputs` in the parameter reference or expression must be
183+
the input object to the previous iteration of the workflow step (or the initial
184+
inputs for the first iteration).
185+
186+
- name: Loop
187+
type: record
188+
extends: cwl:ProcessRequirement
189+
inVocab: false
190+
doc: |
191+
Prototype to enable workflow-level looping of a step.
192+
193+
Valid only under `requirements` of a https://www.commonwl.org/v1.2/Workflow.html#WorkflowStep.
194+
Unlike other CWL requirements, Loop requirement is not propagated to inner steps.
195+
196+
`loopWhen` is an expansion of the CWL v1.2 `when` construct which controls
197+
conditional execution.
198+
199+
Using `loopWhen` and `when` for the same step will produce an error.
200+
201+
`loopWhen` is not compatible with `scatter` at this time and combining the
202+
two in the same step will produce an error.
203+
fields:
204+
class:
205+
type: string
206+
doc: 'cwltool:Loop'
207+
jsonldPredicate:
208+
_id: "@type"
209+
_type: "@vocab"
210+
loopWhen:
211+
type: cwl:Expression
212+
doc: |
213+
Only run the step while the expression evaluates to `true`.
214+
If `false` and no iteration has been performed, the step is skipped.
215+
216+
A skipped step produces a `null` on each output.
217+
218+
The `inputs` value in the expression must be the step input object.
219+
220+
It is an error if this expression returns a value other than `true` or `false`.
221+
loop:
222+
type: LoopInput[]
223+
jsonldPredicate:
224+
_id: "cwltool:loop"
225+
mapSubject: id
226+
mapPredicate: loopSource
227+
doc: |
228+
Defines the input parameters of the loop iterations after the first one
229+
(inputs of the first iteration are the step input parameters). If no
230+
`loop` rule is specified for a given step `in` field, the initial value
231+
is kept constant among all iterations.
232+
outputMethod:
233+
type:
234+
type: enum
235+
name: LoopOutputModes
236+
symbols: [ last, all ]
237+
default: last
238+
doc:
239+
- Specify the desired method of dealing with loop outputs
240+
- Default. Propagates only the last computed element to the subsequent steps when the loop terminates.
241+
- Propagates a single array with all output values to the subsequent steps when the loop terminates.
242+

cwltool/load_tool.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import re
88
import urllib
99
import uuid
10+
from functools import partial
1011
from typing import (
1112
Any,
1213
Dict,
@@ -18,7 +19,6 @@
1819
Union,
1920
cast,
2021
)
21-
from functools import partial
2222

2323
from cwl_utils.parser import cwl_v1_2, cwl_v1_2_utils
2424
from ruamel.yaml.comments import CommentedMap, CommentedSeq

cwltool/main.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -661,9 +661,11 @@ def setup_schema(
661661
ext10 = res.read().decode("utf-8")
662662
with pkg_resources.resource_stream(__name__, "extensions-v1.1.yml") as res:
663663
ext11 = res.read().decode("utf-8")
664+
with pkg_resources.resource_stream(__name__, "extensions-v1.2.yml") as res:
665+
ext12 = res.read().decode("utf-8")
664666
use_custom_schema("v1.0", "http://commonwl.org/cwltool", ext10)
665667
use_custom_schema("v1.1", "http://commonwl.org/cwltool", ext11)
666-
use_custom_schema("v1.2", "http://commonwl.org/cwltool", ext11)
668+
use_custom_schema("v1.2", "http://commonwl.org/cwltool", ext12)
667669
use_custom_schema("v1.2.0-dev1", "http://commonwl.org/cwltool", ext11)
668670
use_custom_schema("v1.2.0-dev2", "http://commonwl.org/cwltool", ext11)
669671
use_custom_schema("v1.2.0-dev3", "http://commonwl.org/cwltool", ext11)

0 commit comments

Comments
 (0)