Merge pull request #224 from common-workflow-language/local-resolver

Local resolver

Author: mr-c

README.rst

@@ -14,29 +14,43 @@ The reference implementation consists of two packages.  The "cwltool" package
 is the primary Python module containing the reference implementation in the
 "cwltool" module and console executable by the same name.
 
-The "cwl-runner" package is optional and provides an additional entry point
+The "cwlref-runner" package is optional and provides an additional entry point
 under the alias "cwl-runner", which is the implementation-agnostic name for the
 default CWL interpreter installed on a host.
 
 Install
 -------
 
-Installing the official package from PyPi (will install "cwltool" package as well)::
+Installing the official package from PyPi (will install "cwltool" package as
+well)::
 
   pip install cwlref-runner
 
-Or from source::
+If installling alongside another CWL implementation then::
+
+  pip instal cwltool
+
+To install from source::
 
   git clone https://github.com/common-workflow-language/cwltool.git
   cd cwltool && python setup.py install
-  cd cwlref-runner && python setup.py install
+  cd cwlref-runner && python setup.py install  # co-installing? skip this
+
+Remember, if co-installing multiple CWL implementations then you need to
+maintain which implementation ``cwl-runner`` points to via a symbolic file
+system link or [another facility](https://wiki.debian.org/DebianAlternatives).
 
 Run on the command line
 -----------------------
 
 Simple command::
 
-  cwl-runner [tool] [job]
+  cwl-runner [tool-or-workflow-description] [input-job-settings]
+
+Or if you have multiple CWL implementations installed and you want to override
+the default cwl-runner use::
+
+  cwltool [tool-or-workflow-description] [input-job-settings]
 
 Import as a module
 ----------------
@@ -60,3 +74,15 @@ and ``--tmp-outdir-prefix`` to somewhere under ``/Users``::
 
 .. |Build Status| image:: https://ci.commonwl.org/buildStatus/icon?job=cwltool-conformance
    :target: https://ci.commonwl.org/job/cwltool-conformance/
+
+Tool or workflow loading from remote or local locations
+-------------------------------------------------------
+
+``cwltool`` can run tool and workflow descriptions on both local and remote
+systems via its support for HTTP[S] URLs.
+
+Input job files and Workflow steps (via the `run` directive) can reference CWL
+documents using absolute or relative local filesytem paths. If a relative path
+is referenced and that document isn't found in the current directory then the
+following locations will be searched:
+http://www.commonwl.org/v1.0/CommandLineTool.html#Discovering_CWL_documents_on_a_local_filesystem

cwltool/load_tool.py

@@ -19,8 +19,8 @@
 
 _logger = logging.getLogger("cwltool")
 
-def fetch_document(argsworkflow):
-    # type: (Union[Text, Text, dict[Text, Any]]) -> Tuple[Loader, Dict[Text, Any], Text]
+def fetch_document(argsworkflow, resolver=None):
+    # type: (Union[Text, dict[Text, Any]], Any) -> Tuple[Loader, Dict[Text, Any], Text]
     """Retrieve a CWL document."""
     document_loader = Loader({"cwl": "https://w3id.org/cwl/cwl#", "id": "@id"})
 
@@ -30,8 +30,17 @@ def fetch_document(argsworkflow):
         split = urlparse.urlsplit(argsworkflow)
         if split.scheme:
             uri = argsworkflow
-        else:
+        elif os.path.exists(os.path.abspath(argsworkflow)):
             uri = "file://" + os.path.abspath(argsworkflow)
+        elif resolver:
+            uri = resolver(document_loader, argsworkflow)
+
+        if uri is None:
+            raise ValidationException("Not found: '%s'" % argsworkflow)
+
+        if argsworkflow != uri:
+            _logger.info("Resolved '%s' to '%s'", argsworkflow, uri)
+
         fileuri = urlparse.urldefrag(uri)[0]
         workflowobj = document_loader.fetch(fileuri)
     elif isinstance(argsworkflow, dict):
@@ -193,9 +202,10 @@ def make_tool(document_loader, avsc_names, metadata, uri, makeTool, kwargs):
 
 def load_tool(argsworkflow, makeTool, kwargs=None,
               enable_dev=False,
-              strict=True):
-    # type: (Union[Text, dict[Text,Any]], Callable[...,Process], Dict[AnyStr, Any], bool, bool) -> Any
-    document_loader, workflowobj, uri = fetch_document(argsworkflow)
+              strict=True,
+              resolver=None):
+    # type: (Union[Text, dict[Text, Any]], Callable[...,Process], Dict[AnyStr, Any], bool, bool, Any) -> Any
+    document_loader, workflowobj, uri = fetch_document(argsworkflow, resolver=resolver)
     document_loader, avsc_names, processobj, metadata, uri = validate_document(
         document_loader, workflowobj, uri, enable_dev=enable_dev,
         strict=strict)

cwltool/main.py

@@ -28,6 +28,7 @@
 from .process import shortname, Process, getListing, relocateOutputs, cleanIntermediate, scandeps, normalizeFilesDirs
 from .load_tool import fetch_document, validate_document, make_tool
 from . import draft2tool
+from .resolver import tool_resolver
 from .builder import adjustFileObjs, adjustDirObjs
 from .stdfsaccess import StdFsAccess
 from .pack import pack
@@ -617,7 +618,7 @@ def main(argsl=None,
                 return 1
 
         try:
-            document_loader, workflowobj, uri = fetch_document(args.workflow)
+            document_loader, workflowobj, uri = fetch_document(args.workflow, resolver=tool_resolver)
 
             if args.print_deps:
                 printdeps(workflowobj, document_loader, stdout, args.relative_deps, uri)

cwltool/resolver.py

@@ -0,0 +1,30 @@
+import os
+import logging
+import urllib
+import urlparse
+
+_logger = logging.getLogger("cwltool")
+
+def resolve_local(document_loader, uri):
+    if uri.startswith("/"):
+        return None
+    shares = [os.environ.get("XDG_DATA_HOME", os.path.join(os.environ["HOME"], ".local", "share"))]
+    shares.extend(os.environ.get("XDG_DATA_DIRS", "/usr/local/share/:/usr/share/").split(":"))
+    shares = [os.path.join(s, "commonwl", uri) for s in shares]
+    shares.insert(0, os.path.join(os.getcwd(), uri))
+
+    _logger.debug("Search path is %s", shares)
+
+    for s in shares:
+        if os.path.exists(s):
+            return ("file://%s" % s)
+        if os.path.exists("%s.cwl" % s):
+            return ("file://%s.cwl" % s)
+    return None
+
+def tool_resolver(document_loader, uri):
+    for r in [resolve_local]:
+        ret = r(document_loader, uri)
+        if ret is not None:
+            return ret
+    return "file://" + os.path.abspath(uri)