From 999ba7e01e540a4d49fd8dd736a06de5b8db0074 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Wed, 1 Mar 2017 10:11:13 -0500 Subject: [PATCH 01/12] Support Dockstore lookups for tools. --- cwltool/resolver.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/cwltool/resolver.py b/cwltool/resolver.py index a67f21003..5836966e8 100644 --- a/cwltool/resolver.py +++ b/cwltool/resolver.py @@ -25,8 +25,20 @@ def resolve_local(document_loader, uri): def tool_resolver(document_loader, uri): - for r in [resolve_local]: + for r in [resolve_local, resolve_ga4gh_tool]: ret = r(document_loader, uri) if ret is not None: return ret return file_uri(os.path.abspath(uri), split_frag=True) + + +def resolve_ga4gh_tool(document_loader, uri): + ds = "https://staging.dockstore.org:8443/api/ga4gh/v1/tools/%s/versions/master/plain-CWL/descriptor" % urllib.quote(uri, "") + print ds + try: + resp = document_loader.session.head(ds) + resp.raise_for_status() + return ds + except Exception: + pass + return None From 7ceb4e5700a6941f22c55cbc12f877bc423adf78 Mon Sep 17 00:00:00 2001 From: Denis Yuen Date: Fri, 30 Sep 2016 16:17:08 -0400 Subject: [PATCH 02/12] Point at production site and allow for specification of tool versions --- cwltool/resolver.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cwltool/resolver.py b/cwltool/resolver.py index 5836966e8..3e7a55c5e 100644 --- a/cwltool/resolver.py +++ b/cwltool/resolver.py @@ -33,7 +33,10 @@ def tool_resolver(document_loader, uri): def resolve_ga4gh_tool(document_loader, uri): - ds = "https://staging.dockstore.org:8443/api/ga4gh/v1/tools/%s/versions/master/plain-CWL/descriptor" % urllib.quote(uri, "") + path, version = uri.partition(":")[::2] + if not version: + version = "latest" + ds = "https://dockstore.org:8443/api/ga4gh/v1/tools/{0}/versions/{1}/plain-CWL/descriptor".format(urllib.quote(path, "") , urllib.quote(version, "")) print ds try: resp = document_loader.session.head(ds) From be0870598df419057d08ad77ad62e2cb48c81c8e Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Wed, 1 Mar 2017 10:12:16 -0500 Subject: [PATCH 03/12] Formatting changes --- cwltool/resolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cwltool/resolver.py b/cwltool/resolver.py index 3e7a55c5e..dce29ff6e 100644 --- a/cwltool/resolver.py +++ b/cwltool/resolver.py @@ -36,7 +36,7 @@ def resolve_ga4gh_tool(document_loader, uri): path, version = uri.partition(":")[::2] if not version: version = "latest" - ds = "https://dockstore.org:8443/api/ga4gh/v1/tools/{0}/versions/{1}/plain-CWL/descriptor".format(urllib.quote(path, "") , urllib.quote(version, "")) + ds = "https://dockstore.org:8443/api/ga4gh/v1/tools/{0}/versions/{1}/plain-CWL/descriptor".format(urllib.quote(path, ""), urllib.quote(version, "")) print ds try: resp = document_loader.session.head(ds) From 7d6cad7070a7cc166cca4697c0eb54bf950df63d Mon Sep 17 00:00:00 2001 From: Denis Yuen Date: Mon, 3 Oct 2016 14:33:47 -0400 Subject: [PATCH 04/12] Remove print statement breaking tests --- cwltool/resolver.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cwltool/resolver.py b/cwltool/resolver.py index dce29ff6e..f597c9644 100644 --- a/cwltool/resolver.py +++ b/cwltool/resolver.py @@ -37,7 +37,6 @@ def resolve_ga4gh_tool(document_loader, uri): if not version: version = "latest" ds = "https://dockstore.org:8443/api/ga4gh/v1/tools/{0}/versions/{1}/plain-CWL/descriptor".format(urllib.quote(path, ""), urllib.quote(version, "")) - print ds try: resp = document_loader.session.head(ds) resp.raise_for_status() From 050078a8af9c34d62b6f2bc49a8785d15a32f118 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Wed, 1 Mar 2017 10:13:46 -0500 Subject: [PATCH 05/12] Update README.rst to add reference to discovery mechanism --- README.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.rst b/README.rst index b351bc5f0..e960b90e3 100644 --- a/README.rst +++ b/README.rst @@ -86,3 +86,16 @@ documents using absolute or relative local filesytem paths. If a relative path is referenced and that document isn't found in the current directory then the following locations will be searched: http://www.commonwl.org/v1.0/CommandLineTool.html#Discovering_CWL_documents_on_a_local_filesystem + +Use with Dockstore +------------------ + +This allows cwltool to launch tools from https://dockstore.org/ in one of two ways :: + + cwltool --non-strict quay.io/collaboratory/dockstore-tool-bamstats:master test.json + +and (defaults to latest when a version is not specified) :: + + cwltool --non-strict quay.io/collaboratory/dockstore-tool-bamstats test.json + +For this example, grab the test.json (and input file) from https://github.com/CancerCollaboratory/dockstore-tool-bamstats From 522943b575e8ec98b176ba8f06b6821f4f8b00f6 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Wed, 1 Mar 2017 10:33:01 -0500 Subject: [PATCH 06/12] Add --enable/disable-tool-registry and --add-tool-registry --- cwltool/main.py | 16 ++++++++++++++-- cwltool/resolver.py | 16 +++++++++------- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/cwltool/main.py b/cwltool/main.py index 56e0bc18b..24d098bd6 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -27,7 +27,7 @@ from .load_tool import fetch_document, validate_document, make_tool from .pack import pack from .process import shortname, Process, getListing, relocateOutputs, cleanIntermediate, scandeps, normalizeFilesDirs -from .resolver import tool_resolver +from .resolver import tool_resolver, tool_registries from .stdfsaccess import StdFsAccess _logger = logging.getLogger("cwltool") @@ -162,7 +162,14 @@ def arg_parser(): # type: () -> argparse.ArgumentParser help="Will be passed to `docker run` as the '--net' " "parameter. Implies '--enable-net'.") - parser.add_argument("--on-error", type=str, + parser.add_argument("--enable-tool-registry", action="store_true", help="Enable resolution using tool registry", + dest="enable_tool_registry") + parser.add_argument("--disable-tool-registry", action="store_false", help="Disable resolution using registry", + dest="enable_tool_registry") + parser.add_argument("--add-tool-registry", action="append", help="Add a tool registry to use for resolution, default %s" % , + dest="tool_registries", default=[]) + + parser.add_argument("--on-error", type=Text, help="Desired workflow behavior when a step fails. One of 'stop' or 'continue'. " "Default is 'stop'.", default="stop", choices=("stop", "continue")) @@ -635,6 +642,11 @@ def main(argsl=None, # type: List[str] if args.relax_path_checks: draft2tool.ACCEPTLIST_RE = draft2tool.ACCEPTLIST_EN_RELAXED_RE + if args.tool_registries: + tool_registries[:] = args.tool_registries + if not args.enable_tool_registry: + del tool_registries[:] + try: document_loader, workflowobj, uri = fetch_document(args.workflow, resolver=resolver, fetcher_constructor=fetcher_constructor) diff --git a/cwltool/resolver.py b/cwltool/resolver.py index f597c9644..2121ad710 100644 --- a/cwltool/resolver.py +++ b/cwltool/resolver.py @@ -31,16 +31,18 @@ def tool_resolver(document_loader, uri): return ret return file_uri(os.path.abspath(uri), split_frag=True) +tool_registries = ["https://dockstore.org:8443"] def resolve_ga4gh_tool(document_loader, uri): path, version = uri.partition(":")[::2] if not version: version = "latest" - ds = "https://dockstore.org:8443/api/ga4gh/v1/tools/{0}/versions/{1}/plain-CWL/descriptor".format(urllib.quote(path, ""), urllib.quote(version, "")) - try: - resp = document_loader.session.head(ds) - resp.raise_for_status() - return ds - except Exception: - pass + for reg in tool_registries: + ds = "{0}/api/ga4gh/v1/tools/{1}/versions/{2}/plain-CWL/descriptor".format(reg, urllib.quote(path, ""), urllib.quote(version, "")) + try: + resp = document_loader.session.head(ds) + resp.raise_for_status() + return ds + except Exception: + pass return None From 5edd681571d0ae55cb835bd2358e8c20c7abed72 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Wed, 1 Mar 2017 10:40:08 -0500 Subject: [PATCH 07/12] Fix --- cwltool/main.py | 12 +++++++----- cwltool/resolver.py | 1 + 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/cwltool/main.py b/cwltool/main.py index 24d098bd6..b02d3d282 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -162,11 +162,13 @@ def arg_parser(): # type: () -> argparse.ArgumentParser help="Will be passed to `docker run` as the '--net' " "parameter. Implies '--enable-net'.") - parser.add_argument("--enable-tool-registry", action="store_true", help="Enable resolution using tool registry", - dest="enable_tool_registry") - parser.add_argument("--disable-tool-registry", action="store_false", help="Disable resolution using registry", - dest="enable_tool_registry") - parser.add_argument("--add-tool-registry", action="append", help="Add a tool registry to use for resolution, default %s" % , + exgroup = parser.add_mutually_exclusive_group() + exgroup.add_argument("--enable-tool-registry", action="store_true", help="Enable resolution using tool registry", + dest="enable_tool_registry", default=True) + exgroup.add_argument("--disable-tool-registry", action="store_false", help="Disable resolution using registry", + dest="enable_tool_registry", default=True) + + parser.add_argument("--add-tool-registry", action="append", help="Add a tool registry to use for resolution, default %s" % tool_registries, dest="tool_registries", default=[]) parser.add_argument("--on-error", type=Text, diff --git a/cwltool/resolver.py b/cwltool/resolver.py index 2121ad710..9e893508f 100644 --- a/cwltool/resolver.py +++ b/cwltool/resolver.py @@ -1,5 +1,6 @@ import logging import os +import urllib from schema_salad.ref_resolver import file_uri From 05b6efda0ce0a67202f4ebcc01ef88d96d8dbb69 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Wed, 1 Mar 2017 11:05:31 -0500 Subject: [PATCH 08/12] Fix mypy --- cwltool/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cwltool/main.py b/cwltool/main.py index b02d3d282..05cf0027c 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -171,7 +171,7 @@ def arg_parser(): # type: () -> argparse.ArgumentParser parser.add_argument("--add-tool-registry", action="append", help="Add a tool registry to use for resolution, default %s" % tool_registries, dest="tool_registries", default=[]) - parser.add_argument("--on-error", type=Text, + parser.add_argument("--on-error", help="Desired workflow behavior when a step fails. One of 'stop' or 'continue'. " "Default is 'stop'.", default="stop", choices=("stop", "continue")) From 5d70973be91c045fca64bf59880458294817e623 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Thu, 2 Mar 2017 17:04:32 -0500 Subject: [PATCH 09/12] Update "Use with Dockstore" to "Use with GA4GH Tool Registry API" --- README.rst | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index e960b90e3..6f33f086a 100644 --- a/README.rst +++ b/README.rst @@ -87,10 +87,14 @@ is referenced and that document isn't found in the current directory then the following locations will be searched: http://www.commonwl.org/v1.0/CommandLineTool.html#Discovering_CWL_documents_on_a_local_filesystem -Use with Dockstore ------------------- +Use with GA4GH Tool Registry API +-------------------------------- -This allows cwltool to launch tools from https://dockstore.org/ in one of two ways :: +Cwltool can launch tools directly from `GA4GH Tool Registry API`_ endpoints. + +By default, cwltool searches https://dockstore.org/ . Use --add-tool-registry to add other registries to the search path. + +For example :: cwltool --non-strict quay.io/collaboratory/dockstore-tool-bamstats:master test.json @@ -99,3 +103,5 @@ and (defaults to latest when a version is not specified) :: cwltool --non-strict quay.io/collaboratory/dockstore-tool-bamstats test.json For this example, grab the test.json (and input file) from https://github.com/CancerCollaboratory/dockstore-tool-bamstats + +.. _`GA4GH Tool Registry API`: https://github.com/ga4gh/tool-registry-schemas From 621946eae4e7ae64eb2edd281ed5ce99ca9a1e8e Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Thu, 2 Mar 2017 17:13:24 -0500 Subject: [PATCH 10/12] Change tool_registries symbols to be explicit about ga4gh_tool_registries. --- cwltool/main.py | 22 +++++++++++----------- cwltool/resolver.py | 4 ++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/cwltool/main.py b/cwltool/main.py index 05cf0027c..5cc409b19 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -27,7 +27,7 @@ from .load_tool import fetch_document, validate_document, make_tool from .pack import pack from .process import shortname, Process, getListing, relocateOutputs, cleanIntermediate, scandeps, normalizeFilesDirs -from .resolver import tool_resolver, tool_registries +from .resolver import tool_resolver, ga4gh_tool_registries from .stdfsaccess import StdFsAccess _logger = logging.getLogger("cwltool") @@ -163,13 +163,13 @@ def arg_parser(): # type: () -> argparse.ArgumentParser "parameter. Implies '--enable-net'.") exgroup = parser.add_mutually_exclusive_group() - exgroup.add_argument("--enable-tool-registry", action="store_true", help="Enable resolution using tool registry", - dest="enable_tool_registry", default=True) - exgroup.add_argument("--disable-tool-registry", action="store_false", help="Disable resolution using registry", - dest="enable_tool_registry", default=True) + exgroup.add_argument("--enable-ga4gh-tool-registry", action="store_true", help="Enable resolution using GA4GH tool registry API", + dest="enable_ga4gh_tool_registry", default=True) + exgroup.add_argument("--disable-ga4gh-tool-registry", action="store_false", help="Disable resolution using GA4GH tool registry API", + dest="enable_ga4gh_tool_registry", default=True) - parser.add_argument("--add-tool-registry", action="append", help="Add a tool registry to use for resolution, default %s" % tool_registries, - dest="tool_registries", default=[]) + parser.add_argument("--add-ga4gh-tool-registry", action="append", help="Add a GA4GH tool registry endpoint to use for resolution, default %s" % ga4gh_tool_registries, + dest="ga4gh_tool_registries", default=[]) parser.add_argument("--on-error", help="Desired workflow behavior when a step fails. One of 'stop' or 'continue'. " @@ -644,10 +644,10 @@ def main(argsl=None, # type: List[str] if args.relax_path_checks: draft2tool.ACCEPTLIST_RE = draft2tool.ACCEPTLIST_EN_RELAXED_RE - if args.tool_registries: - tool_registries[:] = args.tool_registries - if not args.enable_tool_registry: - del tool_registries[:] + if args.ga4gh_tool_registries: + ga4gh_tool_registries[:] = args.ga4gh_tool_registries + if not args.enable_ga4gh_tool_registry: + del ga4gh_tool_registries[:] try: document_loader, workflowobj, uri = fetch_document(args.workflow, resolver=resolver, diff --git a/cwltool/resolver.py b/cwltool/resolver.py index 9e893508f..7ea419562 100644 --- a/cwltool/resolver.py +++ b/cwltool/resolver.py @@ -32,13 +32,13 @@ def tool_resolver(document_loader, uri): return ret return file_uri(os.path.abspath(uri), split_frag=True) -tool_registries = ["https://dockstore.org:8443"] +ga4gh_tool_registries = ["https://dockstore.org:8443"] def resolve_ga4gh_tool(document_loader, uri): path, version = uri.partition(":")[::2] if not version: version = "latest" - for reg in tool_registries: + for reg in ga4gh_tool_registries: ds = "{0}/api/ga4gh/v1/tools/{1}/versions/{2}/plain-CWL/descriptor".format(reg, urllib.quote(path, ""), urllib.quote(version, "")) try: resp = document_loader.session.head(ds) From 7a6dd2e184fc454e9ea87f9020d110dee34a2fb8 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Fri, 3 Mar 2017 13:24:21 +0200 Subject: [PATCH 11/12] format fix --- cwltool/resolver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cwltool/resolver.py b/cwltool/resolver.py index 7ea419562..c0fb069fd 100644 --- a/cwltool/resolver.py +++ b/cwltool/resolver.py @@ -34,6 +34,7 @@ def tool_resolver(document_loader, uri): ga4gh_tool_registries = ["https://dockstore.org:8443"] + def resolve_ga4gh_tool(document_loader, uri): path, version = uri.partition(":")[::2] if not version: From a9a4a5170bb57c53bb9e7fa5aad224b9eb2602de Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Wed, 8 Mar 2017 09:52:13 -0500 Subject: [PATCH 12/12] Fix pylint. --- cwltool/resolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cwltool/resolver.py b/cwltool/resolver.py index c0fb069fd..98571e183 100644 --- a/cwltool/resolver.py +++ b/cwltool/resolver.py @@ -32,8 +32,8 @@ def tool_resolver(document_loader, uri): return ret return file_uri(os.path.abspath(uri), split_frag=True) -ga4gh_tool_registries = ["https://dockstore.org:8443"] +ga4gh_tool_registries = ["https://dockstore.org:8443"] def resolve_ga4gh_tool(document_loader, uri): path, version = uri.partition(":")[::2]