fixes from conformance tests and varying Singularity version

Author: mr-c

README.rst

@@ -138,10 +138,14 @@ or
 
   cwltool --user-space-docker-cmd=dx-docker https://raw.githubusercontent.com/common-workflow-language/common-workflow-language/master/v1.0/v1.0/test-cwl-out2.cwl https://github.com/common-workflow-language/common-workflow-language/blob/master/v1.0/v1.0/empty.json
 
-``cwltool`` can use `Singularity <http://singularity.lbl.gov/>`_ as a Docker container runtime, an experimental feature.
-Singularity will run software containers specified in ``DockerRequirement`` and therefore works with Docker images only,
-native Singularity images are not supported.
-To use Singularity as the Docker container runtime, provide ``--singularity`` command line option to ``cwltool``.
+``cwltool`` can use `Singularity <http://singularity.lbl.gov/>`_ version 2.6.1
+or later as a Docker container runtime.
+``cwltool`` with Singularity will run software containers specified in
+``DockerRequirement`` and therefore works with Docker images only, native
+Singularity images are not supported. To use Singularity as the Docker container
+runtime, provide ``--singularity`` command line option to ``cwltool``.
+With Singularity, ``cwltool`` can pass all CWL v1.0 conformance tests, except
+those involving Docker container ENTRYPOINTs.
 
 
 .. code:: bash

cwltool/singularity.py

@@ -38,6 +38,7 @@
     pass
 
 _USERNS = None
+_SINGULARITY_VERSION = None
 
 def _singularity_supports_userns():  # type: ()->bool
     global _USERNS  # pylint: disable=global-statement
@@ -53,6 +54,25 @@ def _singularity_supports_userns():  # type: ()->bool
             _USERNS = False
     return _USERNS
 
+
+def get_version():
+    global _SINGULARITY_VERSION  # pylint: disable=global-statement
+    if not _SINGULARITY_VERSION:
+       _SINGULARITY_VERSION = check_output(["singularity", "--version"], text=True)
+       if _SINGULARITY_VERSION.startswith("singularity version "):
+           _SINGULARITY_VERSION = _SINGULARITY_VERSION[20:]
+    return _SINGULARITY_VERSION
+
+def is_version_2_6():
+    return get_version().startswith("2.6")
+
+def is_version_3_or_newer():
+    return int(get_version()[0]) >= 3
+
+def is_version_3_1_or_newer():  # OCI compatible ??
+    version = get_version().split('.')
+    return int(version[0]) >= 4 or (int(version[0]) == 3 and int(version[1]) >= 1)
+
 def _normalize_image_id(string):  # type: (Text)->Text
     return string.replace('/', '_') + '.img'
 
@@ -78,30 +98,40 @@ def get_image(dockerRequirement,  # type: Dict[Text, Text]
 
         candidates = []
 
+        cache_folder = None
+        if "CWL_SINGULARITY_CACHE" in os.environ:
+            cache_folder = os.environ["CWL_SINGULARITY_CACHE"]
+        elif is_version_2_6() and "SINGULARITY_PULLFOLDER" in os.environ:
+            cache_folder = os.environ["SINGULARITY_PULLFOLDER"]
+
         if "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement:
             match = re.search(pattern=r'([a-z]*://)', string=dockerRequirement["dockerPull"])
-            candidate_image = _normalize_image_id(dockerRequirement['dockerPull'])
-            candidates.append(candidate_image)
-            candidate_sif = _normalize_sif_id(dockerRequirement['dockerPull'])
-            candidates.append(candidate_sif)
-            dockerRequirement['dockerImageId'] = candidate_image
+            img_name = _normalize_image_id(dockerRequirement['dockerPull'])
+            candidates.append(img_name)
+            if is_version_3_or_newer():
+                sif_name = _normalize_sif_id(dockerRequirement['dockerPull'])
+                candidates.append(sif_name)
+                dockerRequirement["dockerImageId"] = sif_name
+            else:
+                dockerRequirement["dockerImageId"] = img_name
             if not match:
                 dockerRequirement["dockerPull"] = "docker://" + dockerRequirement["dockerPull"]
         elif "dockerImageId" in dockerRequirement:
             candidates.append(dockerRequirement['dockerImageId'])
             candidates.append(_normalize_image_id(dockerRequirement['dockerImageId']))
+            if is_version_3_or_newer():
+                candidates.append(_normalize_sif_id(dockerRequirement['dockerPull']))
 
-        # check if Singularity image is available in $SINGULARITY_CACHEDIR
-        # or any subdirs created in cachedir
         targets = [os.getcwd()]
-        for env in ("SINGULARITY_CACHEDIR", "SINGULARITY_PULLFOLDER"):
-            if env in os.environ:
-                targets.append(os.environ[env])
+        if "CWL_SINGULARITY_CACHE" in os.environ:
+            targets.append(os.environ["CWL_SINGULARITY_CACHE"])
+        if is_version_2_6() and "SINGULARITY_PULLFOLDER" in os.environ:
+            targets.append(os.environ["SINGULARITY_PULLFOLDER"])
         for target in targets:
             for dirpath, subdirs, files in os.walk(target):
-                for sif in files:
-                    if sif in candidates:
-                        path = os.path.join(dirpath, sif)
+                for entry in files:
+                    if entry in candidates:
+                        path = os.path.join(dirpath, entry)
                         if os.path.isfile(path):
                             _logger.info(
                                 "Using local copy of Singularity image found in %s",
@@ -111,15 +141,24 @@ def get_image(dockerRequirement,  # type: Dict[Text, Text]
         if (force_pull or not found) and pull_image:
             cmd = []  # type: List[Text]
             if "dockerPull" in dockerRequirement:
-                if "SINGULARITY_PULLFOLDER" in os.environ:
-                    pull_folder = str(os.environ["SINGULARITY_PULLFOLDER"])
-                    cmd = ["singularity", "pull", "--force", "--name",
-                           str('%s/' %pull_folder) + str(dockerRequirement["dockerImageId"]),
-                           str(dockerRequirement["dockerPull"])]
+                if cache_folder:
+                    env = os.environ.copy()
+                    if is_version_2_6():
+                        env['SINGULARITY_PULLFOLDER'] = cache_folder
+                        cmd = ["singularity", "pull", "--force", "--name",
+                               dockerRequirement["dockerImageId"],
+                               str(dockerRequirement["dockerPull"])]
+                    else:
+                        cmd = ["singularity", "pull", "--force", "--name",
+                               "{}/{}".format(
+                                   cache_folder,
+                                   dockerRequirement["dockerImageId"]),
+                               str(dockerRequirement["dockerPull"])]
+
                     _logger.info(Text(cmd))
-                    check_call(cmd, stdout=sys.stderr)  # nosec
-                    dockerRequirement["dockerImageId"] = pull_folder + '/' + \
-                                                         str(dockerRequirement["dockerImageId"])
+                    check_call(cmd, env=env, stdout=sys.stderr)  # nosec
+                    dockerRequirement["dockerImageId"] = '{}/{}'.format(
+                            cache_folder, dockerRequirement["dockerImageId"])
                     found = True
                 else:
                     cmd = ["singularity", "pull", "--force", "--name",
@@ -135,10 +174,21 @@ def get_image(dockerRequirement,  # type: Dict[Text, Text]
                         "dockerFile is not currently supported when using the "
                         "Singularity runtime for Docker containers."))
             elif "dockerLoad" in dockerRequirement:
+                if is_version_3_1_or_newer():
+                    if 'dockerImageId' in dockerRequirement:
+                        name = "{}.sif".format(dockerRequirement["dockerImageId"])
+                    else:
+                        name = "{}.sif".format(dockerRequirement["dockerLoad"])
+                    cmd ["singularity", "build", name,
+                         "docker-archive://{}".format(dockerRequirement["dockerLoad"])]
+                    _logger.info(Text(cmd))
+                    check_call(cmd, stdout=sys.stderr)  # nosec
+                    found = True
+                    dockerRequirement['dockerImageId'] = name
                 raise WorkflowException(SourceLine(
                     dockerRequirement, 'dockerLoad').makeError(
                         "dockerLoad is not currently supported when using the "
-                        "Singularity runtime for Docker containers."))
+                        "Singularity runtime (version less than 3.1) for Docker containers."))
             elif "dockerImport" in dockerRequirement:
                 raise WorkflowException(SourceLine(
                     dockerRequirement, 'dockerImport').makeError(
@@ -166,10 +216,7 @@ def get_from_requirements(self,
             raise WorkflowException(u"Container image {} not "
                                     "found".format(r["dockerImageId"]))
 
-        if "SINGULARITY_PULLFOLDER" in os.environ:
-            return str(r["dockerImageId"])
-        else:
-            return os.path.abspath(r["dockerImageId"])
+        return os.path.abspath(r["dockerImageId"])
 
     @staticmethod
     def append_volume(runtime, source, target, writable=False):
@@ -282,10 +329,16 @@ def create_runtime(self,
                    u"--ipc"]
         if _singularity_supports_userns():
             runtime.append(u"--userns")
-        runtime.append(u"--bind")
-        runtime.append(u"{}:{}:rw".format(
-            docker_windows_path_adjust(os.path.realpath(self.outdir)),
-            self.builder.outdir))
+        if is_version_3_1_or_newer():
+            runtime.append(u"--home")
+            runtime.append(u"{}:{}".format(
+                docker_windows_path_adjust(os.path.realpath(self.outdir)),
+                self.builder.outdir))
+        else:
+            runtime.append(u"--bind")
+            runtime.append(u"{}:{}:rw".format(
+                docker_windows_path_adjust(os.path.realpath(self.outdir)),
+                self.builder.outdir))
         runtime.append(u"--bind")
         tmpdir = "/tmp"  # nosec
         runtime.append(u"{}:{}:rw".format(

tests/test_singularity.py

@@ -9,12 +9,12 @@
 from cwltool.main import main
 
 from .util import (get_data, get_main_output, needs_singularity,
-                   working_directory)
+                   needs_singularity_2_6, working_directory)
 
 sys.argv = ['']
 
 
-@needs_singularity
+@needs_singularity_2_6
 def test_singularity_pullfolder(tmp_path):
     workdir = tmp_path / "working_dir_new"
     workdir.mkdir()
@@ -25,8 +25,11 @@ def test_singularity_pullfolder(tmp_path):
     env["SINGULARITY_PULLFOLDER"] = str(pullfolder)
     result_code, stdout, stderr = get_main_output(
         ['--singularity', get_data("tests/sing_pullfolder_test.cwl"), "--message", "hello"], env=env)
+    print(stdout)
+    print(stderr)
     assert result_code == 0
-    # TODO, confirm that the image is in the pullfolder
+    image = pullfolder/"debian.img"
+    assert image.exists()
 
 @needs_singularity
 def test_singularity_workflow(tmpdir):

tests/util.py

@@ -18,7 +18,7 @@
 from cwltool.factory import Factory
 from cwltool.resolver import Path
 from cwltool.utils import onWindows, subprocess, windows_default_container_id
-
+from cwltool.singularity import is_version_2_6, is_version_3_or_newer
 
 
 def get_windows_safe_factory(runtime_context=None,  # type: RuntimeContext
@@ -55,9 +55,18 @@ def get_data(filename):
                                   reason="Requires the docker executable on the "
                                   "system path.")
 
-needs_singularity = pytest.mark.skipif(not bool(distutils.spawn.find_executable('singularity')),
-                                       reason="Requires the singularity executable on the "
-                                       "system path.")
+needs_singularity = pytest.mark.skipif(
+    not bool(distutils.spawn.find_executable('singularity')),
+    reason="Requires the singularity executable on the system path.")
+
+needs_singularity_2_6 = pytest.mark.skipif(
+    not is_version_2_6(),
+    reason="Requires that version 2.6.x of singularity executable version is on the system path.")
+
+needs_singularity_3_or_newer = pytest.mark.skipif(
+    not is_version_3_or_newer,
+    reason="Requires that version 2.6.x of singularity executable version is on the system path.")
+
 
 windows_needs_docker = pytest.mark.skipif(
     onWindows() and not bool(distutils.spawn.find_executable('docker')),