Skip to content

Commit 470d561

Browse files
jmchiltonjmchilton
committed
Refactor utilities for dealing with SoftwareRequirements into own module.
Per great suggestion from Michael on #217.
1 parent dec58df commit 470d561

File tree

2 files changed

+124
-101
lines changed

2 files changed

+124
-101
lines changed

‎cwltool/main.py

+7-101
Original file line numberDiff line numberDiff line change
@@ -32,16 +32,10 @@
3232
relocateOutputs, scandeps, shortname, use_custom_schema,
3333
use_standard_schema)
3434
from .resolver import ga4gh_tool_registries, tool_resolver
35+
from .software_requirements import DependenciesConfiguration
3536
from .stdfsaccess import StdFsAccess
3637
from .update import ALLUPDATES, UPDATES
37-
from .utils import get_feature
38-
try:
39-
from galaxy.tools.deps.requirements import ToolRequirement, ToolRequirements
40-
from galaxy.tools import deps
41-
except ImportError:
42-
ToolRequirement = None # type: ignore
43-
ToolRequirements = None # type: ignore
44-
deps = None
38+
4539

4640
_logger = logging.getLogger("cwltool")
4741

@@ -863,102 +857,14 @@ def locToPath(p):
863857
_logger.addHandler(defaultStreamHandler)
864858

865859

866-
COMMAND_WITH_DEPENDENCIES_TEMPLATE = string.Template("""#!/bin/bash
867-
$handle_dependencies
868-
python "run_job.py" "job.json"
869-
""")
870-
871-
872860
def find_default_container(args, builder):
861+
default_container = None
873862
if args.default_container:
874-
return args.default_container
863+
default_container = args.default_container
875864
elif args.beta_use_biocontainers:
876-
try:
877-
from galaxy.tools.deps.containers import ContainerRegistry, AppInfo, ToolInfo, DOCKER_CONTAINER_TYPE
878-
except ImportError:
879-
raise Exception("galaxy-lib not found")
880-
881-
app_info = AppInfo(
882-
involucro_auto_init=True,
883-
enable_beta_mulled_containers=True,
884-
container_image_cache_path=".",
885-
) # type: AppInfo
886-
container_registry = ContainerRegistry(app_info) # type: ContainerRegistry
887-
requirements = _get_dependencies(builder)
888-
tool_info = ToolInfo(requirements=requirements) # type: ToolInfo
889-
container_description = container_registry.find_best_container_description([DOCKER_CONTAINER_TYPE], tool_info)
890-
if container_description:
891-
return container_description.identifier
892-
893-
return None
894-
895-
896-
class DependenciesConfiguration(object):
897-
898-
def __init__(self, args):
899-
# type: (argparse.Namespace) -> None
900-
conf_file = getattr(args, "beta_dependency_resolvers_configuration", None)
901-
tool_dependency_dir = getattr(args, "beta_dependencies_directory", None)
902-
conda_dependencies = getattr(args, "beta_conda_dependencies", None)
903-
if conf_file is not None and os.path.exists(conf_file):
904-
self.use_tool_dependencies = True
905-
if not tool_dependency_dir:
906-
tool_dependency_dir = os.path.abspath(os.path.dirname(conf_file))
907-
self.tool_dependency_dir = tool_dependency_dir
908-
self.dependency_resolvers_config_file = conf_file
909-
elif conda_dependencies:
910-
if not tool_dependency_dir:
911-
tool_dependency_dir = os.path.abspath("./cwltool_deps")
912-
self.tool_dependency_dir = tool_dependency_dir
913-
self.use_tool_dependencies = True
914-
self.dependency_resolvers_config_file = None
915-
else:
916-
self.use_tool_dependencies = False
917-
918-
@property
919-
def config_dict(self):
920-
return {
921-
'conda_auto_install': True,
922-
'conda_auto_init': True,
923-
}
924-
925-
def build_job_script(self, builder, command):
926-
# type: (Any, List[str]) -> Text
927-
if deps is None:
928-
raise Exception("galaxy-lib not found")
929-
tool_dependency_manager = deps.build_dependency_manager(self) # type: deps.DependencyManager
930-
dependencies = _get_dependencies(builder)
931-
handle_dependencies = "" # str
932-
if dependencies:
933-
handle_dependencies = "\n".join(tool_dependency_manager.dependency_shell_commands(dependencies, job_directory=builder.tmpdir))
934-
935-
template_kwds = dict(handle_dependencies=handle_dependencies) # type: Dict[str, str]
936-
job_script = COMMAND_WITH_DEPENDENCIES_TEMPLATE.substitute(template_kwds)
937-
return job_script
938-
939-
940-
def _get_dependencies(builder):
941-
# type: (Any) -> List[ToolRequirement]
942-
(software_requirement, _) = get_feature(builder, "SoftwareRequirement")
943-
dependencies = [] # type: List[ToolRequirement]
944-
if software_requirement and software_requirement.get("packages"):
945-
packages = software_requirement.get("packages")
946-
for package in packages:
947-
version = package.get("version", None)
948-
if isinstance(version, list):
949-
if version:
950-
version = version[0]
951-
else:
952-
version = None
953-
specs = [{"uri": s} for s in package.get("specs", [])]
954-
dependencies.append(ToolRequirement.from_dict(dict(
955-
name=package["package"].split("#")[-1],
956-
version=version,
957-
type="package",
958-
specs=specs,
959-
)))
960-
961-
return ToolRequirements.from_list(dependencies)
865+
default_container = get_container_from_software_requirements(args, builder)
866+
867+
return default_container
962868

963869

964870
if __name__ == "__main__":

‎cwltool/software_requirements.py

+117
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
"""This module handles resolution of SoftwareRequirement hints.
2+
3+
This is accomplished mainly by adapting cwltool internals to galaxy-lib's
4+
concept of "dependencies". Despite the name, galaxy-lib is a light weight
5+
library that can be used to map SoftwareRequirements in all sorts of ways -
6+
Homebrew, Conda, custom scripts, environment modules. We'd be happy to find
7+
ways to adapt new packages managers and such as well.
8+
"""
9+
10+
import os
11+
import string
12+
13+
try:
14+
from galaxy.tools.deps.requirements import ToolRequirement, ToolRequirements
15+
from galaxy.tools import deps
16+
except ImportError:
17+
ToolRequirement = None # type: ignore
18+
ToolRequirements = None # type: ignore
19+
deps = None
20+
21+
from .utils import get_feature
22+
23+
24+
COMMAND_WITH_DEPENDENCIES_TEMPLATE = string.Template("""#!/bin/bash
25+
$handle_dependencies
26+
python "run_job.py" "job.json"
27+
""")
28+
29+
30+
class DependenciesConfiguration(object):
31+
32+
def __init__(self, args):
33+
# type: (argparse.Namespace) -> None
34+
conf_file = getattr(args, "beta_dependency_resolvers_configuration", None)
35+
tool_dependency_dir = getattr(args, "beta_dependencies_directory", None)
36+
conda_dependencies = getattr(args, "beta_conda_dependencies", None)
37+
if conf_file is not None and os.path.exists(conf_file):
38+
self.use_tool_dependencies = True
39+
if not tool_dependency_dir:
40+
tool_dependency_dir = os.path.abspath(os.path.dirname(conf_file))
41+
self.tool_dependency_dir = tool_dependency_dir
42+
self.dependency_resolvers_config_file = conf_file
43+
elif conda_dependencies:
44+
if not tool_dependency_dir:
45+
tool_dependency_dir = os.path.abspath("./cwltool_deps")
46+
self.tool_dependency_dir = tool_dependency_dir
47+
self.use_tool_dependencies = True
48+
self.dependency_resolvers_config_file = None
49+
else:
50+
self.use_tool_dependencies = False
51+
52+
@property
53+
def config_dict(self):
54+
return {
55+
'conda_auto_install': True,
56+
'conda_auto_init': True,
57+
}
58+
59+
def build_job_script(self, builder, command):
60+
# type: (Any, List[str]) -> Text
61+
if deps is None:
62+
raise Exception("galaxy-lib not found")
63+
tool_dependency_manager = deps.build_dependency_manager(self) # type: deps.DependencyManager
64+
dependencies = get_dependencies(builder)
65+
handle_dependencies = "" # str
66+
if dependencies:
67+
handle_dependencies = "\n".join(tool_dependency_manager.dependency_shell_commands(dependencies, job_directory=builder.tmpdir))
68+
69+
template_kwds = dict(handle_dependencies=handle_dependencies) # type: Dict[str, str]
70+
job_script = COMMAND_WITH_DEPENDENCIES_TEMPLATE.substitute(template_kwds)
71+
return job_script
72+
73+
74+
def get_dependencies(builder):
75+
# type: (Any) -> List[ToolRequirement]
76+
(software_requirement, _) = get_feature(builder, "SoftwareRequirement")
77+
dependencies = [] # type: List[ToolRequirement]
78+
if software_requirement and software_requirement.get("packages"):
79+
packages = software_requirement.get("packages")
80+
for package in packages:
81+
version = package.get("version", None)
82+
if isinstance(version, list):
83+
if version:
84+
version = version[0]
85+
else:
86+
version = None
87+
specs = [{"uri": s} for s in package.get("specs", [])]
88+
dependencies.append(ToolRequirement.from_dict(dict(
89+
name=package["package"].split("#")[-1],
90+
version=version,
91+
type="package",
92+
specs=specs,
93+
)))
94+
95+
return ToolRequirements.from_list(dependencies)
96+
97+
98+
def get_container_from_software_requirements(args, builder):
99+
if args.beta_use_biocontainers:
100+
try:
101+
from galaxy.tools.deps.containers import ContainerRegistry, AppInfo, ToolInfo, DOCKER_CONTAINER_TYPE
102+
except ImportError:
103+
raise Exception("Optional requirement galaxy-lib not found, it is required for this configuration.")
104+
105+
app_info = AppInfo(
106+
involucro_auto_init=True,
107+
enable_beta_mulled_containers=True,
108+
container_image_cache_path=".",
109+
) # type: AppInfo
110+
container_registry = ContainerRegistry(app_info) # type: ContainerRegistry
111+
requirements = get_dependencies(builder)
112+
tool_info = ToolInfo(requirements=requirements) # type: ToolInfo
113+
container_description = container_registry.find_best_container_description([DOCKER_CONTAINER_TYPE], tool_info)
114+
if container_description:
115+
return container_description.identifier
116+
117+
return None

0 commit comments

Comments
 (0)