Skip to content

Commit ed9dd4c

Browse files
jmfernandezmr-c
jmfernandez
and
mr-c
authored
Allow packing Operation records (#1467)
* cwltool now packs `Operation` definitions CWL 1.2 introduced [Operation](https://www.commonwl.org/v1.2/Workflow.html#Operation) in order to model abstract workflow steps, needed to use CWL to describe a workflow which could not exist or be implemented in a different way than CWL. As `cwltool --pack` was ignoring records of class `Operation`, packed workflows were not correct (they did not validate with `cwltool --validate`). The issue was explained at #1466. This commit fixes the issue which arises while packing workflows having `Operation` records. Co-authored-by: Michael R. Crusoe <michael.crusoe@gmail.com>
1 parent 56d5120 commit ed9dd4c

File tree

6 files changed

+241
-45
lines changed

6 files changed

+241
-45
lines changed

‎MANIFEST.in‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ include tests/*
77
include tests/tmp1/tmp2/tmp3/.gitkeep
88
include tests/tmp4/alpha/*
99
include tests/wf/*
10+
include tests/wf/operation/*
1011
include tests/override/*
1112
include tests/checker_wf/*
1213
include tests/subgraph/*

‎cwltool/pack.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -251,7 +251,12 @@ def rewrite_id(r: str, mainuri: str) -> None:
251251
if "$schemas" in metadata:
252252
for s in metadata["$schemas"]:
253253
schemas.add(s)
254-
if dcr.get("class") not in ("Workflow", "CommandLineTool", "ExpressionTool"):
254+
if dcr.get("class") not in (
255+
"Workflow",
256+
"CommandLineTool",
257+
"ExpressionTool",
258+
"Operation",
259+
):
255260
continue
256261
dc = cast(Dict[str, Any], copy.deepcopy(dcr))
257262
v = rewrite[r]

‎tests/test_pack.py‎

Lines changed: 28 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -21,57 +21,41 @@
2121
from .util import get_data, needs_docker
2222

2323

24-
def test_pack() -> None:
25-
loadingContext, workflowobj, uri = fetch_document(get_data("tests/wf/revsort.cwl"))
26-
yaml = YAML(typ="safe", pure=True)
27-
28-
with open(get_data("tests/wf/expect_packed.cwl")) as packed_file:
29-
expect_packed = yaml.load(packed_file)
30-
31-
packed = cwltool.pack.pack(loadingContext, uri)
32-
adjustFileObjs(
33-
packed, partial(make_relative, os.path.abspath(get_data("tests/wf")))
34-
)
35-
adjustDirObjs(packed, partial(make_relative, os.path.abspath(get_data("tests/wf"))))
36-
37-
assert "$schemas" in packed
38-
packed_schemas = packed["$schemas"]
39-
assert isinstance(packed_schemas, Sized)
40-
assert len(packed_schemas) == len(expect_packed["$schemas"])
41-
del packed["$schemas"]
42-
del expect_packed["$schemas"]
43-
44-
assert packed == expect_packed
45-
46-
47-
def test_pack_input_named_name() -> None:
48-
loadingContext, workflowobj, uri = fetch_document(
49-
get_data("tests/wf/trick_revsort.cwl")
50-
)
24+
@pytest.mark.parametrize(
25+
"unpacked,expected",
26+
[
27+
("tests/wf/revsort.cwl", "tests/wf/expect_packed.cwl"),
28+
(
29+
"tests/wf/operation/operation-single.cwl",
30+
"tests/wf/operation/expect_operation-single_packed.cwl",
31+
),
32+
("tests/wf/trick_revsort.cwl", "tests/wf/expect_trick_packed.cwl"),
33+
],
34+
)
35+
def test_packing(unpacked: str, expected: str) -> None:
36+
"""Compare expected version reality with various workflows and --pack."""
37+
loadingContext, workflowobj, uri = fetch_document(get_data(unpacked))
5138
loadingContext.do_update = False
5239
loadingContext, uri = resolve_and_validate_document(
5340
loadingContext, workflowobj, uri
5441
)
55-
loader = loadingContext.loader
56-
assert loader
57-
loader.resolve_ref(uri)[0]
58-
59-
yaml = YAML()
60-
with open(get_data("tests/wf/expect_trick_packed.cwl")) as packed_file:
61-
expect_packed = yaml.load(packed_file)
6242

6343
packed = cwltool.pack.pack(loadingContext, uri)
64-
adjustFileObjs(
65-
packed, partial(make_relative, os.path.abspath(get_data("tests/wf")))
66-
)
67-
adjustDirObjs(packed, partial(make_relative, os.path.abspath(get_data("tests/wf"))))
44+
context_dir = os.path.abspath(os.path.dirname(get_data(unpacked)))
45+
adjustFileObjs(packed, partial(make_relative, context_dir))
46+
adjustDirObjs(packed, partial(make_relative, context_dir))
6847

69-
assert "$schemas" in packed
70-
packed_schemas = packed["$schemas"]
71-
assert isinstance(packed_schemas, Sized)
72-
assert len(packed_schemas) == len(expect_packed["$schemas"])
73-
del packed["$schemas"]
74-
del expect_packed["$schemas"]
48+
yaml = YAML(typ="safe", pure=True)
49+
with open(get_data(expected)) as packed_file:
50+
expect_packed = yaml.load(packed_file)
51+
52+
if "$schemas" in expect_packed:
53+
assert "$schemas" in packed
54+
packed_schemas = packed["$schemas"]
55+
assert isinstance(packed_schemas, Sized)
56+
assert len(packed_schemas) == len(expect_packed["$schemas"])
57+
del packed["$schemas"]
58+
del expect_packed["$schemas"]
7559

7660
assert packed == expect_packed
7761

‎tests/wf/operation/abstract-cosifer.cwl‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
class: Operation
2+
cwlVersion: v1.2
3+
4+
requirements:
5+
DockerRequirement:
6+
dockerPull: 'tsenit/cosifer:b4d5af45d2fc54b6bff2a9153a8e9054e560302e'
7+
8+
inputs:
9+
data_matrix:
10+
type: File
11+
separator:
12+
type: string?
13+
doc: The separator used in the data_matrix file
14+
index_col:
15+
type: int?
16+
gmt_filepath:
17+
type: File?
18+
outdir:
19+
type: string?
20+
samples_on_rows:
21+
type: boolean?
22+
23+
outputs:
24+
resdir:
25+
type: Directory

‎tests/wf/operation/expect_operation-single_packed.cwl‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
{
2+
"$graph": [
3+
{
4+
"class": "Operation",
5+
"requirements": [
6+
{
7+
"dockerPull": "tsenit/cosifer:b4d5af45d2fc54b6bff2a9153a8e9054e560302e",
8+
"class": "DockerRequirement"
9+
}
10+
],
11+
"inputs": [
12+
{
13+
"type": "File",
14+
"id": "#abstract-cosifer.cwl/data_matrix"
15+
},
16+
{
17+
"type": [
18+
"null",
19+
"File"
20+
],
21+
"id": "#abstract-cosifer.cwl/gmt_filepath"
22+
},
23+
{
24+
"type": [
25+
"null",
26+
"int"
27+
],
28+
"id": "#abstract-cosifer.cwl/index_col"
29+
},
30+
{
31+
"type": [
32+
"null",
33+
"string"
34+
],
35+
"id": "#abstract-cosifer.cwl/outdir"
36+
},
37+
{
38+
"type": [
39+
"null",
40+
"boolean"
41+
],
42+
"id": "#abstract-cosifer.cwl/samples_on_rows"
43+
},
44+
{
45+
"type": [
46+
"null",
47+
"string"
48+
],
49+
"doc": "The separator used in the data_matrix file",
50+
"id": "#abstract-cosifer.cwl/separator"
51+
}
52+
],
53+
"outputs": [
54+
{
55+
"type": "Directory",
56+
"id": "#abstract-cosifer.cwl/resdir"
57+
}
58+
],
59+
"id": "#abstract-cosifer.cwl"
60+
},
61+
{
62+
"class": "Workflow",
63+
"id": "#main",
64+
"label": "abstract-cosifer-workflow",
65+
"inputs": [
66+
{
67+
"type": "File",
68+
"doc": "Gene expression data matrix",
69+
"id": "#data_matrix"
70+
},
71+
{
72+
"type": [
73+
"null",
74+
"File"
75+
],
76+
"doc": "Optional GMT file to perform inference on multiple gene sets",
77+
"id": "#gmt_filepath"
78+
},
79+
{
80+
"type": [
81+
"null",
82+
"int"
83+
],
84+
"doc": "Column index in the data. Defaults to None, a.k.a., no index",
85+
"id": "#index_col"
86+
},
87+
{
88+
"type": "string",
89+
"doc": "Path to the output directory",
90+
"id": "#outdir"
91+
},
92+
{
93+
"type": [
94+
"null",
95+
"boolean"
96+
],
97+
"doc": "Flag that indicates that data contain the samples on rows. Defaults to False.",
98+
"id": "#samples_on_rows"
99+
},
100+
{
101+
"type": [
102+
"null",
103+
"string"
104+
],
105+
"doc": "Separator for the data. Defaults to .",
106+
"id": "#separator"
107+
}
108+
],
109+
"outputs": [
110+
{
111+
"type": "Directory",
112+
"outputSource": "#/abstract_cosifer/resdir",
113+
"id": "#resdir"
114+
}
115+
],
116+
"steps": [
117+
{
118+
"run": "#abstract-cosifer.cwl",
119+
"in": [
120+
{
121+
"source": "#data_matrix",
122+
"id": "#abstract_cosifer/data_matrix"
123+
},
124+
{
125+
"source": "#gmt_filepath",
126+
"id": "#abstract_cosifer/gmt_filepath"
127+
},
128+
{
129+
"source": "#index_col",
130+
"id": "#abstract_cosifer/index_col"
131+
},
132+
{
133+
"source": "#outdir",
134+
"id": "#abstract_cosifer/outdir"
135+
},
136+
{
137+
"source": "#samples_on_rows",
138+
"id": "#abstract_cosifer/samples_on_rows"
139+
},
140+
{
141+
"source": "#separator",
142+
"id": "#abstract_cosifer/separator"
143+
}
144+
],
145+
"out": [
146+
"#/abstract_cosifer/resdir"
147+
],
148+
"id": "#abstract_cosifer"
149+
}
150+
]
151+
}
152+
],
153+
"cwlVersion": "v1.2"
154+
}

‎tests/wf/operation/operation-single.cwl‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
class: Workflow
2+
cwlVersion: v1.2
3+
id: abstract_cosifer_workflow
4+
label: abstract-cosifer-workflow
5+
6+
inputs:
7+
data_matrix: {type: File, doc: "Gene expression data matrix"}
8+
gmt_filepath: {type: "File?", doc: "Optional GMT file to perform inference on multiple gene sets"}
9+
index_col: {type: "int?", doc: "Column index in the data. Defaults to None, a.k.a., no index"}
10+
outdir: {type: string, doc: "Path to the output directory"}
11+
separator: {type: "string?", doc: "Separator for the data. Defaults to ."}
12+
samples_on_rows: {type: "boolean?", doc: "Flag that indicates that data contain the samples on rows. Defaults to False."}
13+
14+
outputs:
15+
resdir: {type: Directory, outputSource: abstract_cosifer/resdir}
16+
17+
steps:
18+
abstract_cosifer:
19+
run: abstract-cosifer.cwl
20+
in:
21+
data_matrix: data_matrix
22+
separator: separator
23+
index_col: index_col
24+
gmt_filepath: gmt_filepath
25+
outdir: outdir
26+
samples_on_rows: samples_on_rows
27+
out: [resdir]

0 commit comments

Comments
 (0)