Support file system and Dockstore lookups for tools.

Author: Peter Amstutz

cwltool/load_tool.py

@@ -19,8 +19,8 @@
 
 _logger = logging.getLogger("cwltool")
 
-def fetch_document(argsworkflow):
-    # type: (Union[Text, Text, dict[Text, Any]]) -> Tuple[Loader, Dict[Text, Any], Text]
+def fetch_document(argsworkflow, resolver=None):
+    # type: (Union[Text, dict[Text, Any]], Any) -> Tuple[Loader, Dict[Text, Any], Text]
     """Retrieve a CWL document."""
     document_loader = Loader({"cwl": "https://w3id.org/cwl/cwl#", "id": "@id"})
 
@@ -30,8 +30,17 @@ def fetch_document(argsworkflow):
         split = urlparse.urlsplit(argsworkflow)
         if split.scheme:
             uri = argsworkflow
-        else:
+        elif os.path.exists(os.path.abspath(argsworkflow)):
             uri = "file://" + os.path.abspath(argsworkflow)
+        elif resolver:
+            uri = resolver(document_loader, argsworkflow)
+
+        if uri is None:
+            raise ValidationException("Not found: '%s'" % argsworkflow)
+
+        if argsworkflow != uri:
+            _logger.info("Resolved '%s' to '%s'", argsworkflow, uri)
+
         fileuri = urlparse.urldefrag(uri)[0]
         workflowobj = document_loader.fetch(fileuri)
     elif isinstance(argsworkflow, dict):
@@ -193,9 +202,10 @@ def make_tool(document_loader, avsc_names, metadata, uri, makeTool, kwargs):
 
 def load_tool(argsworkflow, makeTool, kwargs=None,
               enable_dev=False,
-              strict=True):
-    # type: (Union[Text, dict[Text,Any]], Callable[...,Process], Dict[AnyStr, Any], bool, bool) -> Any
-    document_loader, workflowobj, uri = fetch_document(argsworkflow)
+              strict=True,
+              resolver=None):
+    # type: (Union[Text, dict[Text, Any]], Callable[...,Process], Dict[AnyStr, Any], bool, bool, Any) -> Any
+    document_loader, workflowobj, uri = fetch_document(argsworkflow, resolver=resolver)
     document_loader, avsc_names, processobj, metadata, uri = validate_document(
         document_loader, workflowobj, uri, enable_dev=enable_dev,
         strict=strict)

cwltool/main.py

@@ -28,6 +28,7 @@
 from .process import shortname, Process, getListing, relocateOutputs, cleanIntermediate, scandeps, normalizeFilesDirs
 from .load_tool import fetch_document, validate_document, make_tool
 from . import draft2tool
+from .resolver import tool_resolver
 from .builder import adjustFileObjs, adjustDirObjs
 from .stdfsaccess import StdFsAccess
 from .pack import pack
@@ -617,7 +618,7 @@ def main(argsl=None,
                 return 1
 
         try:
-            document_loader, workflowobj, uri = fetch_document(args.workflow)
+            document_loader, workflowobj, uri = fetch_document(args.workflow, resolver=tool_resolver)
 
             if args.print_deps:
                 printdeps(workflowobj, document_loader, stdout, args.relative_deps, uri)

cwltool/resolver.py

@@ -0,0 +1,41 @@
+import os
+import logging
+import urllib
+import urlparse
+
+_logger = logging.getLogger("cwltool")
+
+def resolve_local(document_loader, uri):
+    if uri.startswith("/"):
+        return None
+    shares = [os.environ.get("XDG_DATA_HOME", os.path.join(os.environ["HOME"], ".local", "share"))]
+    shares.extend(os.environ.get("XDG_DATA_DIRS", "/usr/local/share/:/usr/share/").split(":"))
+    shares = [os.path.join(s, "commonwl", uri) for s in shares]
+    shares.insert(0, os.path.join(os.getcwd(), uri))
+
+    _logger.debug("Search path is %s", shares)
+
+    for s in shares:
+        if os.path.exists(s):
+            return ("file://%s" % s)
+        if os.path.exists("%s.cwl" % s):
+            return ("file://%s.cwl" % s)
+    return None
+
+def resolve_ga4gh_tool(document_loader, uri):
+    ds = "https://staging.dockstore.org:8443/api/ga4gh/v1/tools/%s/versions/master/plain-CWL/descriptor" % urllib.quote(uri, "")
+    print ds
+    try:
+        resp = document_loader.session.head(ds)
+        resp.raise_for_status()
+        return ds
+    except Exception:
+        pass
+    return None
+
+def tool_resolver(document_loader, uri):
+    for r in [resolve_local, resolve_ga4gh_tool]:
+        ret = r(document_loader, uri)
+        if ret is not None:
+            return ret
+    return uri