podman/singularity + nodejs

Fixes #803

Author: mr-c

Makefile

@@ -148,11 +148,11 @@ diff-cover.html: coverage.xml
 
 ## test        : run the ${MODULE} test suite
 test: check-python3 $(PYSOURCES)
-	python -m pytest ${PYTEST_EXTRA}
+	python -m pytest -rs ${PYTEST_EXTRA}
 
 ## testcov     : run the ${MODULE} test suite and collect coverage
 testcov: check-python3 $(PYSOURCES)
-	python -m pytest --cov --cov-config=.coveragerc --cov-report= ${PYTEST_EXTRA}
+	python -m pytest -rs --cov --cov-config=.coveragerc --cov-report= ${PYTEST_EXTRA}
 
 sloccount.sc: $(PYSOURCES) Makefile
 	sloccount --duplicates --wide --details $^ > $@
@@ -177,7 +177,7 @@ mypy: $(filter-out setup.py gittagger.py,$(PYSOURCES))
 
 mypyc: $(PYSOURCES)
 	MYPYPATH=typeshed CWLTOOL_USE_MYPYC=1 pip install --verbose -e . \
-		 && pytest -vv ${PYTEST_EXTRA}
+		 && pytest -rs -vv ${PYTEST_EXTRA}
 
 shellcheck: FORCE
 	shellcheck build-cwltool-docker.sh cwl-docker.sh release-test.sh conformance-test.sh \

cwltool/builder.py

@@ -179,6 +179,7 @@ def __init__(
         tmpdir: str,
         stagedir: str,
         cwlVersion: str,
+        container_engine: str,
     ) -> None:
         """Initialize this Builder."""
         self.job = job
@@ -215,6 +216,7 @@ def __init__(
         self.pathmapper = None  # type: Optional[PathMapper]
         self.prov_obj = None  # type: Optional[ProvenanceProfile]
         self.find_default_container = None  # type: Optional[Callable[[], str]]
+        self.container_engine = container_engine
 
     def build_job_script(self, commands: List[str]) -> Optional[str]:
         if self.job_script_provider is not None:
@@ -746,4 +748,5 @@ def do_eval(
             force_docker_pull=self.force_docker_pull,
             strip_whitespace=strip_whitespace,
             cwlVersion=self.cwlVersion,
+            container_engine=self.container_engine,
         )

cwltool/context.py

@@ -75,6 +75,8 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None:
         self.jobdefaults = None  # type: Optional[CommentedMap]
         self.doc_cache = True  # type: bool
         self.relax_path_checks = False  # type: bool
+        self.singularity = False  # type: bool
+        self.podman = False  # type: bool
 
         super().__init__(kwargs)
 

cwltool/expression.py

@@ -204,6 +204,7 @@ def evaluator(
     force_docker_pull: bool = False,
     debug: bool = False,
     js_console: bool = False,
+    container_engine: str = "docker",
 ) -> Optional[CWLOutputType]:
     match = param_re.match(ex)
 
@@ -238,6 +239,7 @@ def evaluator(
             force_docker_pull=force_docker_pull,
             debug=debug,
             js_console=js_console,
+            container_engine=container_engine,
         )
     else:
         if expression_parse_exception is not None:
@@ -270,6 +272,7 @@ def interpolate(
     strip_whitespace: bool = True,
     escaping_behavior: int = 2,
     convert_to_expression: bool = False,
+    container_engine: str = "docker",
 ) -> Optional[CWLOutputType]:
     """
     Interpolate and evaluate.
@@ -303,6 +306,7 @@ def interpolate(
                     force_docker_pull=force_docker_pull,
                     debug=debug,
                     js_console=js_console,
+                    container_engine=container_engine,
                 )
                 if w[0] == 0 and w[1] == len(scan) and len(parts) <= 1:
                     return e
@@ -367,6 +371,7 @@ def do_eval(
     js_console: bool = False,
     strip_whitespace: bool = True,
     cwlVersion: str = "",
+    container_engine: str = "docker",
 ) -> Optional[CWLOutputType]:
 
     runtime = cast(MutableMapping[str, Union[int, str, None]], copy.deepcopy(resources))
@@ -409,6 +414,7 @@ def do_eval(
                     "v1.2.0-dev3",
                 )
                 else 2,
+                container_engine=container_engine,
             )
 
         except Exception as e:

cwltool/factory.py

@@ -40,6 +40,9 @@ def __call__(self, **kwargs):
 class Factory:
     """Easy way to load a CWL document for execution."""
 
+    loading_context: LoadingContext
+    runtime_context: RuntimeContext
+
     def __init__(
         self,
         executor: Optional[JobExecutor] = None,
@@ -51,13 +54,16 @@ def __init__(
         if executor is None:
             executor = SingleJobExecutor()
         self.executor = executor
-        self.loading_context = loading_context
-        if loading_context is None:
-            self.loading_context = LoadingContext()
         if runtime_context is None:
             self.runtime_context = RuntimeContext()
         else:
             self.runtime_context = runtime_context
+        if loading_context is None:
+            self.loading_context = LoadingContext()
+            self.loading_context.singularity = self.runtime_context.singularity
+            self.loading_context.podman = self.runtime_context.podman
+        else:
+            self.loading_context = loading_context
 
     def make(self, cwl: Union[str, Dict[str, Any]]) -> Callable:
         """Instantiate a CWL object from a CWl document."""

cwltool/loghandler.py

@@ -1,7 +1,38 @@
 """Shared logger for cwltool."""
 import logging
 
+import coloredlogs
+
 _logger = logging.getLogger("cwltool")  # pylint: disable=invalid-name
 defaultStreamHandler = logging.StreamHandler()  # pylint: disable=invalid-name
 _logger.addHandler(defaultStreamHandler)
 _logger.setLevel(logging.INFO)
+
+
+def configure_logging(
+    stderr_handler: logging.Handler,
+    quiet: bool,
+    debug: bool,
+    enable_color: bool,
+    timestamps: bool,
+    base_logger: logging.Logger = _logger,
+) -> None:
+    """Configure logging."""
+    rdflib_logger = logging.getLogger("rdflib.term")
+    rdflib_logger.addHandler(stderr_handler)
+    rdflib_logger.setLevel(logging.ERROR)
+    if quiet:
+        # Silence STDERR, not an eventual provenance log file
+        stderr_handler.setLevel(logging.WARN)
+    if debug:
+        # Increase to debug for both stderr and provenance log file
+        base_logger.setLevel(logging.DEBUG)
+        stderr_handler.setLevel(logging.DEBUG)
+        rdflib_logger.setLevel(logging.DEBUG)
+    fmtclass = coloredlogs.ColoredFormatter if enable_color else logging.Formatter
+    formatter = fmtclass("%(levelname)s %(message)s")
+    if timestamps:
+        formatter = fmtclass(
+            "[%(asctime)s] %(levelname)s %(message)s", "%Y-%m-%d %H:%M:%S"
+        )
+    stderr_handler.setFormatter(formatter)

cwltool/main.py

@@ -60,7 +60,7 @@
     resolve_overrides,
     resolve_tool_uri,
 )
-from .loghandler import _logger, defaultStreamHandler
+from .loghandler import _logger, configure_logging, defaultStreamHandler
 from .mpi import MpiConfig
 from .mutation import MutationManager
 from .pack import pack
@@ -624,31 +624,6 @@ def supported_cwl_versions(enable_dev: bool) -> List[str]:
     return versions
 
 
-def configure_logging(
-    args: argparse.Namespace,
-    stderr_handler: logging.Handler,
-    runtimeContext: RuntimeContext,
-) -> None:
-    rdflib_logger = logging.getLogger("rdflib.term")
-    rdflib_logger.addHandler(stderr_handler)
-    rdflib_logger.setLevel(logging.ERROR)
-    if args.quiet:
-        # Silence STDERR, not an eventual provenance log file
-        stderr_handler.setLevel(logging.WARN)
-    if runtimeContext.debug:
-        # Increase to debug for both stderr and provenance log file
-        _logger.setLevel(logging.DEBUG)
-        stderr_handler.setLevel(logging.DEBUG)
-        rdflib_logger.setLevel(logging.DEBUG)
-    fmtclass = coloredlogs.ColoredFormatter if args.enable_color else logging.Formatter
-    formatter = fmtclass("%(levelname)s %(message)s")
-    if args.timestamps:
-        formatter = fmtclass(
-            "[%(asctime)s] %(levelname)s %(message)s", "%Y-%m-%d %H:%M:%S"
-        )
-    stderr_handler.setFormatter(formatter)
-
-
 def setup_schema(
     args: argparse.Namespace, custom_schema_callback: Optional[Callable[[], None]]
 ) -> None:
@@ -724,6 +699,8 @@ def setup_loadingContext(
 ) -> LoadingContext:
     if loadingContext is None:
         loadingContext = LoadingContext(vars(args))
+        loadingContext.singularity = runtimeContext.singularity
+        loadingContext.podman = runtimeContext.podman
     else:
         loadingContext = loadingContext.copy()
     loadingContext.loader = default_loader(
@@ -966,7 +943,13 @@ def main(
             if not hasattr(args, key):
                 setattr(args, key, val)
 
-        configure_logging(args, stderr_handler, runtimeContext)
+        configure_logging(
+            stderr_handler,
+            args.quiet,
+            runtimeContext.debug,
+            args.enable_color,
+            args.timestamps,
+        )
 
         if args.version:
             print(versionfunc())

cwltool/process.py

@@ -697,6 +697,12 @@ def __init__(
             )
             make_avsc_object(convert_to_dict(self.outputs_record_schema), self.names)
 
+        self.container_engine = "docker"
+        if loadingContext.podman:
+            self.container_engine = "podman"
+        elif loadingContext.singularity:
+            self.container_engine = "singularity"
+
         if toolpath_object.get("class") is not None and not getdefault(
             loadingContext.disable_js_validation, False
         ):
@@ -722,6 +728,7 @@ def __init__(
                     toolpath_object,
                     self.doc_schema.names[avroname],
                     validate_js_options,
+                    self.container_engine,
                 )
 
         dockerReq, is_req = self.get_requirement("DockerRequirement")
@@ -903,6 +910,7 @@ def inc(d):  # type: (List[int]) -> None
             tmpdir,
             stagedir,
             cwl_version,
+            self.container_engine,
         )
 
         bindings.extend(