Merge pull request #1113 from drkennetz/singularity23_fix

mr-c · web-flow · commit 26870e38cec8 · 2019-06-03T08:16:40.000+02:00
Singularity23 fix
diff --git a/README.rst b/README.rst
@@ -138,10 +138,14 @@ or
 
   cwltool --user-space-docker-cmd=dx-docker https://raw.githubusercontent.com/common-workflow-language/common-workflow-language/master/v1.0/v1.0/test-cwl-out2.cwl https://github.com/common-workflow-language/common-workflow-language/blob/master/v1.0/v1.0/empty.json
 
-``cwltool`` can use `Singularity <http://singularity.lbl.gov/>`_ as a Docker container runtime, an experimental feature.
-Singularity will run software containers specified in ``DockerRequirement`` and therefore works with Docker images only,
-native Singularity images are not supported.
-To use Singularity as the Docker container runtime, provide ``--singularity`` command line option to ``cwltool``.
+``cwltool`` can use `Singularity <http://singularity.lbl.gov/>`_ version 2.6.1
+or later as a Docker container runtime.
+``cwltool`` with Singularity will run software containers specified in
+``DockerRequirement`` and therefore works with Docker images only, native
+Singularity images are not supported. To use Singularity as the Docker container
+runtime, provide ``--singularity`` command line option to ``cwltool``.
+With Singularity, ``cwltool`` can pass all CWL v1.0 conformance tests, except
+those involving Docker container ENTRYPOINTs.
 
 
 .. code:: bash
diff --git a/cwltool/argparser.py b/cwltool/argparser.py
@@ -211,7 +211,7 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
     dockergroup.add_argument("--singularity", action="store_true",
                              default=False, help="[experimental] Use "
                              "Singularity runtime for running containers. "
-                             "Requires Singularity v2.3.2+ and Linux with kernel "
+                             "Requires Singularity v2.6.1+ and Linux with kernel "
                              "version v3.18+ or with overlayfs support "
                              "backported.")
     dockergroup.add_argument("--no-container", action="store_false",
diff --git a/cwltool/singularity.py b/cwltool/singularity.py
@@ -38,6 +38,7 @@
     pass
 
 _USERNS = None
+_SINGULARITY_VERSION = None
 
 def _singularity_supports_userns():  # type: ()->bool
     global _USERNS  # pylint: disable=global-statement
@@ -53,10 +54,30 @@ def _singularity_supports_userns():  # type: ()->bool
             _USERNS = False
     return _USERNS
 
+
+def get_version():  # type: ()->Text
+    global _SINGULARITY_VERSION  # pylint: disable=global-statement
+    if not _SINGULARITY_VERSION:
+       _SINGULARITY_VERSION = check_output(["singularity", "--version"], universal_newlines=True)
+       if _SINGULARITY_VERSION.startswith("singularity version "):
+           _SINGULARITY_VERSION = _SINGULARITY_VERSION[20:]
+    return _SINGULARITY_VERSION
+
+def is_version_2_6():  # type: ()->bool
+    return get_version().startswith("2.6")
+
+def is_version_3_or_newer():  # type: ()->bool
+    return int(get_version()[0]) >= 3
+
+def is_version_3_1_or_newer():  # type: ()->bool
+    version = get_version().split('.')
+    return int(version[0]) >= 4 or (int(version[0]) == 3 and int(version[1]) >= 1)
+
 def _normalize_image_id(string):  # type: (Text)->Text
-    candidate = re.sub(pattern=r'([a-z]*://)', repl=r'', string=string)
-    return re.sub(pattern=r'[:/]', repl=r'-', string=candidate) + ".img"
+    return string.replace('/', '_') + '.img'
 
+def _normalize_sif_id(string): # type: (Text)->Text
+    return string.replace('/', '_') + '.sif'
 
 class SingularityCommandLineJob(ContainerCommandLineJob):
 
@@ -77,51 +98,97 @@ def get_image(dockerRequirement,  # type: Dict[Text, Text]
 
         candidates = []
 
+        cache_folder = None
+        if "CWL_SINGULARITY_CACHE" in os.environ:
+            cache_folder = os.environ["CWL_SINGULARITY_CACHE"]
+        elif is_version_2_6() and "SINGULARITY_PULLFOLDER" in os.environ:
+            cache_folder = os.environ["SINGULARITY_PULLFOLDER"]
+
         if "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement:
             match = re.search(pattern=r'([a-z]*://)', string=dockerRequirement["dockerPull"])
-            candidate = _normalize_image_id(dockerRequirement['dockerPull'])
-            candidates.append(candidate)
-            dockerRequirement['dockerImageId'] = candidate
+            img_name = _normalize_image_id(dockerRequirement['dockerPull'])
+            candidates.append(img_name)
+            if is_version_3_or_newer():
+                sif_name = _normalize_sif_id(dockerRequirement['dockerPull'])
+                candidates.append(sif_name)
+                dockerRequirement["dockerImageId"] = sif_name
+            else:
+                dockerRequirement["dockerImageId"] = img_name
             if not match:
                 dockerRequirement["dockerPull"] = "docker://" + dockerRequirement["dockerPull"]
         elif "dockerImageId" in dockerRequirement:
             candidates.append(dockerRequirement['dockerImageId'])
             candidates.append(_normalize_image_id(dockerRequirement['dockerImageId']))
+            if is_version_3_or_newer():
+                candidates.append(_normalize_sif_id(dockerRequirement['dockerPull']))
 
-        # check if Singularity image is available in $SINGULARITY_CACHEDIR
         targets = [os.getcwd()]
-        for env in ("SINGULARITY_CACHEDIR", "SINGULARITY_PULLFOLDER"):
-            if env in os.environ:
-                targets.append(os.environ[env])
+        if "CWL_SINGULARITY_CACHE" in os.environ:
+            targets.append(os.environ["CWL_SINGULARITY_CACHE"])
+        if is_version_2_6() and "SINGULARITY_PULLFOLDER" in os.environ:
+            targets.append(os.environ["SINGULARITY_PULLFOLDER"])
         for target in targets:
-            for candidate in candidates:
-                path = os.path.join(target, candidate)
-                if os.path.isfile(path):
-                    _logger.info(
-                        "Using local copy of Singularity image found in %s",
-                        target)
-                    dockerRequirement["dockerImageId"] = path
-                    found = True
-
+            for dirpath, subdirs, files in os.walk(target):
+                for entry in files:
+                    if entry in candidates:
+                        path = os.path.join(dirpath, entry)
+                        if os.path.isfile(path):
+                            _logger.info(
+                                "Using local copy of Singularity image found in %s",
+                                dirpath)
+                            dockerRequirement["dockerImageId"] = path
+                            found = True
         if (force_pull or not found) and pull_image:
             cmd = []  # type: List[Text]
             if "dockerPull" in dockerRequirement:
-                cmd = ["singularity", "pull", "--force", "--name",
-                       str(dockerRequirement["dockerImageId"]),
-                       str(dockerRequirement["dockerPull"])]
-                _logger.info(Text(cmd))
-                check_call(cmd, stdout=sys.stderr)  # nosec
-                found = True
+                if cache_folder:
+                    env = os.environ.copy()
+                    if is_version_2_6():
+                        env['SINGULARITY_PULLFOLDER'] = cache_folder
+                        cmd = ["singularity", "pull", "--force", "--name",
+                               dockerRequirement["dockerImageId"],
+                               str(dockerRequirement["dockerPull"])]
+                    else:
+                        cmd = ["singularity", "pull", "--force", "--name",
+                               "{}/{}".format(
+                                   cache_folder,
+                                   dockerRequirement["dockerImageId"]),
+                               str(dockerRequirement["dockerPull"])]
+
+                    _logger.info(Text(cmd))
+                    check_call(cmd, env=env, stdout=sys.stderr)  # nosec
+                    dockerRequirement["dockerImageId"] = '{}/{}'.format(
+                            cache_folder, dockerRequirement["dockerImageId"])
+                    found = True
+                else:
+                    cmd = ["singularity", "pull", "--force", "--name",
+                           str(dockerRequirement["dockerImageId"]),
+                           str(dockerRequirement["dockerPull"])]
+                    _logger.info(Text(cmd))
+                    check_call(cmd, stdout=sys.stderr)  # nosec
+                    found = True
+
             elif "dockerFile" in dockerRequirement:
                 raise WorkflowException(SourceLine(
                     dockerRequirement, 'dockerFile').makeError(
                         "dockerFile is not currently supported when using the "
                         "Singularity runtime for Docker containers."))
             elif "dockerLoad" in dockerRequirement:
+                if is_version_3_1_or_newer():
+                    if 'dockerImageId' in dockerRequirement:
+                        name = "{}.sif".format(dockerRequirement["dockerImageId"])
+                    else:
+                        name = "{}.sif".format(dockerRequirement["dockerLoad"])
+                    cmd = ["singularity", "build", name,
+                         "docker-archive://{}".format(dockerRequirement["dockerLoad"])]
+                    _logger.info(Text(cmd))
+                    check_call(cmd, stdout=sys.stderr)  # nosec
+                    found = True
+                    dockerRequirement['dockerImageId'] = name
                 raise WorkflowException(SourceLine(
                     dockerRequirement, 'dockerLoad').makeError(
                         "dockerLoad is not currently supported when using the "
-                        "Singularity runtime for Docker containers."))
+                        "Singularity runtime (version less than 3.1) for Docker containers."))
             elif "dockerImport" in dockerRequirement:
                 raise WorkflowException(SourceLine(
                     dockerRequirement, 'dockerImport').makeError(
@@ -252,8 +319,8 @@ def add_writable_directory_volume(self,
 
 
     def create_runtime(self,
-                       env,            # type: MutableMapping[Text, Text]
-                       runtime_context  # type: RuntimeContext
+                       env,              # type: MutableMapping[Text, Text]
+                       runtime_context   # type: RuntimeContext
                       ):  # type: (...) -> Tuple[List, Optional[Text]]
         """ Returns the Singularity runtime list of commands and options."""
         any_path_okay = self.builder.get_requirement("DockerRequirement")[1] \
@@ -262,10 +329,16 @@ def create_runtime(self,
                    u"--ipc"]
         if _singularity_supports_userns():
             runtime.append(u"--userns")
-        runtime.append(u"--bind")
-        runtime.append(u"{}:{}:rw".format(
-            docker_windows_path_adjust(os.path.realpath(self.outdir)),
-            self.builder.outdir))
+        if is_version_3_1_or_newer():
+            runtime.append(u"--home")
+            runtime.append(u"{}:{}".format(
+                docker_windows_path_adjust(os.path.realpath(self.outdir)),
+                self.builder.outdir))
+        else:
+            runtime.append(u"--bind")
+            runtime.append(u"{}:{}:rw".format(
+                docker_windows_path_adjust(os.path.realpath(self.outdir)),
+                self.builder.outdir))
         runtime.append(u"--bind")
         tmpdir = "/tmp"  # nosec
         runtime.append(u"{}:{}:rw".format(
@@ -283,6 +356,7 @@ def create_runtime(self,
         runtime.append(u"--pwd")
         runtime.append(u"%s" % (docker_windows_path_adjust(self.builder.outdir)))
 
+
         if runtime_context.custom_net:
             raise UnsupportedRequirement(
                 "Singularity implementation does not support custom networking")
diff --git a/tests/debian_image_id.cwl b/tests/debian_image_id.cwl
@@ -0,0 +1,14 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.0
+class: CommandLineTool
+
+requirements:
+  DockerRequirement:
+    dockerImageId: 'debian.img'
+
+inputs:
+  message: string
+
+outputs: []
+
+baseCommand: echo
diff --git a/tests/sing_pullfolder_test.cwl b/tests/sing_pullfolder_test.cwl
@@ -0,0 +1,14 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.0
+class: CommandLineTool
+
+requirements:
+  DockerRequirement:
+    dockerPull: debian
+
+inputs:
+  message: string
+
+outputs: []
+
+baseCommand: echo
diff --git a/tests/test_pack.py b/tests/test_pack.py
@@ -154,7 +154,7 @@ def test_packed_workflow_execution(wf_path, job_path, namespaced, tmpdir):
     packed_output = StringIO()
 
     normal_params = ['--outdir', str(tmpdir), get_data(wf_path), get_data(job_path)]
-    packed_params = ['--outdir', str(tmpdir), '--debug', get_data(wf_packed_path), get_data(job_path)]
+    packed_params = ['--outdir', str(tmpdir), '--debug', wf_packed_path, get_data(job_path)]
 
     assert main(normal_params, stdout=normal_output) == 0
     assert main(packed_params, stdout=packed_output) == 0
diff --git a/tests/test_singularity.py b/tests/test_singularity.py
@@ -1,4 +1,5 @@
 import sys
+import os
 import pytest
 
 import distutils.spawn
@@ -8,11 +9,28 @@
 from cwltool.main import main
 
 from .util import (get_data, get_main_output, needs_singularity,
-                   working_directory)
+                   needs_singularity_2_6, working_directory)
 
 sys.argv = ['']
 
 
+@needs_singularity_2_6
+def test_singularity_pullfolder(tmp_path):
+    workdir = tmp_path / "working_dir_new"
+    workdir.mkdir()
+    os.chdir(str(workdir))
+    pullfolder = tmp_path / "pullfolder"
+    pullfolder.mkdir()
+    env = os.environ.copy()
+    env["SINGULARITY_PULLFOLDER"] = str(pullfolder)
+    result_code, stdout, stderr = get_main_output(
+        ['--singularity', get_data("tests/sing_pullfolder_test.cwl"), "--message", "hello"], env=env)
+    print(stdout)
+    print(stderr)
+    assert result_code == 0
+    image = pullfolder/"debian.img"
+    assert image.exists()
+
 @needs_singularity
 def test_singularity_workflow(tmpdir):
     with working_directory(str(tmpdir)):
@@ -38,3 +56,23 @@ def test_singularity_incorrect_image_pull():
         ['--singularity', '--default-container', 'non-existant-weird-image',
          get_data("tests/wf/hello-workflow.cwl"), "--usermessage", "hello"])
     assert result_code != 0
+
+@needs_singularity
+def test_singularity_local(tmp_path):
+    workdir = tmp_path / "working_dir"
+    workdir.mkdir()
+    os.chdir(str(workdir))
+    result_code, stdout, stderr = get_main_output(
+        ['--singularity', get_data("tests/sing_pullfolder_test.cwl"), "--message", "hello"])
+    assert result_code == 0
+
+@needs_singularity_2_6
+def test_singularity_docker_image_id_in_tool(tmp_path):
+    workdir = tmp_path / "working_dir"
+    workdir.mkdir()
+    os.chdir(str(workdir))
+    result_code, stdout, stderr = get_main_output(
+        ['--singularity', get_data("tests/sing_pullfolder_test.cwl"), "--message", "hello"])
+    result_code1, stdout, stderr = get_main_output(
+        ['--singularity', get_data("tests/debian_image_id.cwl"), "--message", "hello"])
+    assert result_code1 == 0
diff --git a/tests/test_udocker.py b/tests/test_udocker.py
@@ -51,7 +51,7 @@ def test_udocker_usage_should_not_write_cid_file(self, tmpdir):
 
         tmpdir.remove(ignore_errors=True)
 
-        assert "completed success" in stderr
+        assert "completed success" in stderr, stderr
         assert cidfiles_count == 0
 
     @pytest.mark.skipif(TRAVIS, reason='Not reliable on single threaded test on travis.')
@@ -63,5 +63,5 @@ def test_udocker_should_display_memory_usage(self, tmpdir):
         cwd.chdir()
         tmpdir.remove(ignore_errors=True)
 
-        assert "completed success" in stderr
-        assert "Max memory" in stderr
+        assert "completed success" in stderr, stderr
+        assert "Max memory" in stderr, stderr
diff --git a/tests/util.py b/tests/util.py
@@ -18,7 +18,7 @@
 from cwltool.factory import Factory
 from cwltool.resolver import Path
 from cwltool.utils import onWindows, subprocess, windows_default_container_id
-
+from cwltool.singularity import is_version_2_6, is_version_3_or_newer
 
 
 def get_windows_safe_factory(runtime_context=None,  # type: RuntimeContext
@@ -55,19 +55,28 @@ def get_data(filename):
                                   reason="Requires the docker executable on the "
                                   "system path.")
 
-needs_singularity = pytest.mark.skipif(not bool(distutils.spawn.find_executable('singularity')),
-                                       reason="Requires the singularity executable on the "
-                                       "system path.")
+needs_singularity = pytest.mark.skipif(
+    not bool(distutils.spawn.find_executable('singularity')),
+    reason="Requires the singularity executable on the system path.")
+
+needs_singularity_2_6 = pytest.mark.skipif(
+    not (distutils.spawn.find_executable('singularity') and is_version_2_6()),
+    reason="Requires that version 2.6.x of singularity executable version is on the system path.")
+
+needs_singularity_3_or_newer = pytest.mark.skipif(
+    not (distutils.spawn.find_executable('singularity') and is_version_3_or_newer),
+    reason="Requires that version 2.6.x of singularity executable version is on the system path.")
+
 
 windows_needs_docker = pytest.mark.skipif(
     onWindows() and not bool(distutils.spawn.find_executable('docker')),
     reason="Running this test on MS Windows requires the docker executable "
     "on the system path.")
 
-def get_main_output(args):
+def get_main_output(args, env=None):
     process = subprocess.Popen(
         [sys.executable, "-m", "cwltool"] + args,
-        stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
 
     stdout, stderr = process.communicate()
     return process.returncode, stdout.decode(), stderr.decode()
