diff --git a/README.md b/README.md index 29b3c45..c52e090 100755 --- a/README.md +++ b/README.md @@ -14,18 +14,18 @@ for filesystem mounts with FUSE, a UFTP driver for and a UNICORE implementation of a [Dask Cluster](https://distributed.dask.org/en/stable/) -This project has received funding from the European Union’s -Horizon 2020 Framework Programme for Research and Innovation under the -Specific Grant Agreement Nos. 720270, 785907 and 945539 +This project has received funding from the European Union’s +Horizon 2020 Framework Programme for Research and Innovation under the +Specific Grant Agreement Nos. 720270, 785907 and 945539 (Human Brain Project SGA 1, 2 and 3) See LICENSE file for licensing information ## Documentation -The complete documentation of PyUNICORE can be viewed +The complete documentation of PyUNICORE can be viewed [here](https://pyunicore.readthedocs.io/en/latest/) - + ## Installation Install from PyPI with @@ -61,7 +61,7 @@ client = uc_client.Client(credential, base_url) print(json.dumps(client.properties, indent = 2)) ``` -PyUNICORE supports a variety of +PyUNICORE supports a variety of [authentication options](https://pyunicore.readthedocs.io/en/latest/authentication.html). ### Run a job and read result files @@ -188,7 +188,7 @@ unicore-cwl-runner echo.cwl hello_world.yml > hello_world.u ## Helpers -The `pyunicore.helpers` module provides a set of higher-level APIs: +The `pyunicore.helpers` module provides helper code for: * Connecting to * a Registry (`pyunicore.helpers.connect_to_registry`). diff --git a/pyunicore/cli/__init__.py b/pyunicore/cli/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/pyunicore/cli/base.py b/pyunicore/cli/base.py new file mode 100644 index 0000000..69a1947 --- /dev/null +++ b/pyunicore/cli/base.py @@ -0,0 +1,176 @@ +""" Base command class """ +import argparse +import getpass +import json +import os.path +from base64 import b64decode + +import pyunicore.client +import pyunicore.credentials + + +class Base: + """Base command class with support for common commandline args""" + + def __init__(self, password_source=None): + self.parser = argparse.ArgumentParser( + prog="unicore", description="A commandline client for UNICORE" + ) + self.config = {"verbose": False} + self.args = None + self.credential = None + self.registry = None + self.add_base_args() + self.add_command_args() + if password_source: + self.password_source = password_source + else: + self.password_source = getpass.getpass + + def _value(self, value: str): + if value.lower() == "true": + return True + if value.lower() == "false": + return False + return value + + def load_user_properties(self): + with open(self.config_file) as f: + for line in f.readlines(): + line = line.strip() + if line.startswith("#"): + continue + try: + key, value = line.split("=", 1) + self.config[key] = self._value(value) + except ValueError: + pass + + def add_base_args(self): + self.parser.add_argument( + "-v", "--verbose", required=False, action="store_true", help="Be verbose" + ) + self.parser.add_argument( + "-c", + "--configuration", + metavar="CONFIG", + default=f"{os.getenv('HOME')}/.ucc/properties", + help="Configuration file", + ) + + def add_command_args(self): + pass + + def run(self, args): + self.args = self.parser.parse_args(args) + self.is_verbose = self.args.verbose + self.config_file = self.args.configuration + self.load_user_properties() + self.create_credential() + self.registry = self.create_registry() + + def get_synopsis(self): + return "N/A" + + def create_credential(self): + auth_method = self.config.get("authentication-method", "USERNAME").upper() + if "USERNAME" == auth_method: + username = self.config["username"] + password = self._get_password() + self.credential = pyunicore.credentials.create_credential(username, password) + + def _get_password(self, key="password") -> str: + password = self.config.get(key) + if password is None: + _p = os.getenv("UCC_PASSWORD") + if not _p: + pwd_prompt = "Enter password: " + password = self.password_source(pwd_prompt) + else: + password = _p + return password + + def create_registry(self) -> pyunicore.client.Registry: + self.create_credential() + return pyunicore.client.Registry(self.credential, self.config["registry"]) + + def verbose(self, msg): + if self.is_verbose: + print(msg) + + def human_readable(self, value, decimals=0): + for unit in ["B", "KB", "MB", "GB"]: + if value < 1024.0 or unit == "GB": + break + value /= 1024.0 + return f"{value:.{decimals}f} {unit}" + + +class IssueToken(Base): + def add_command_args(self): + self.parser.prog = "unicore issue-token" + self.parser.description = self.get_synopsis() + self.parser.add_argument("URL", help="Token endpoint URL") + self.parser.add_argument("-s", "--sitename", required=False, type=str, help="Site name") + self.parser.add_argument( + "-l", + "--lifetime", + required=False, + type=int, + default=-1, + help="Initial lifetime (in seconds) for token.", + ) + self.parser.add_argument( + "-R", + "--renewable", + required=False, + action="store_true", + help="Token can be used to get a fresh token.", + ) + self.parser.add_argument( + "-L", + "--limited", + required=False, + action="store_true", + help="Token should be limited to the issuing server.", + ) + self.parser.add_argument( + "-I", "--inspect", required=False, action="store_true", help="Inspect the issued token." + ) + + def get_synopsis(self): + return """Get a JWT token from a UNICORE/X server""" + + def run(self, args): + super().run(args) + endpoint = self.args.URL + site_name = self.args.sitename + if site_name: + endpoint = self.registry.site(site_name).resource_url + else: + if endpoint is None: + raise ValueError("Either --sitename or URL must be given.") + endpoint = endpoint.split("/token")[0] + token = self.issue_token( + url=endpoint, + lifetime=self.args.lifetime, + limited=self.args.limited, + renewable=self.args.renewable, + ) + if self.args.inspect: + self.show_token_details(token) + print(token) + + def issue_token(self, url: str, lifetime: int, limited: bool, renewable: bool) -> str: + client = pyunicore.client.Client(self.credential, site_url=url) + return client.issue_auth_token(lifetime, renewable, limited) + + def show_token_details(self, token: str): + _p = token.split(".")[1] + _p += "=" * (-len(_p) % 4) # padding + payload = json.loads(b64decode(_p)) + print(f"Subject: {payload['sub']}") + print(f"Lifetime (s): {payload['exp']-payload['iat']}") + print(f"Issued by: {payload['iss']}") + print(f"Valid for: {payload.get('aud', '')}") + print(f"Renewable: {payload.get('renewable', 'no')}") diff --git a/pyunicore/cli/main.py b/pyunicore/cli/main.py new file mode 100644 index 0000000..bc7df86 --- /dev/null +++ b/pyunicore/cli/main.py @@ -0,0 +1,66 @@ +""" Main client class """ +import platform +import sys + +import pyunicore.cli.base + +_commands = { + "issue-token": pyunicore.cli.base.IssueToken, +} + + +def get_command(name): + return _commands.get(name)() + + +def show_version(): + print( + "UNICORE Commandline Client (pyUNICORE) " + "%s, https://www.unicore.eu" % pyunicore._version.get_versions().get("version", "n/a") + ) + print("Python %s" % sys.version) + print("OS: %s" % platform.platform()) + + +def help(): + s = """UNICORE Commandline Client (pyUNICORE) %s, https://www.unicore.eu +Usage: unicore [OPTIONS] +The following commands are available:""" % pyunicore._version.get_versions().get( + "version", "n/a" + ) + print(s) + for cmd in sorted(_commands): + print(f" {cmd:20} - {get_command(cmd).get_synopsis()}") + print("Enter 'unicore -h' for help on a particular command.") + + +def run(args): + _help = ["help", "-h", "--help"] + if len(args) < 1 or args[0] in _help: + help() + return + _version = ["version", "-V", "--version"] + if args[0] in _version: + show_version() + return + + command = None + cmd = args[0] + for k in _commands: + if k.startswith(cmd): + command = get_command(k) + break + if command is None: + raise ValueError(f"No such command: {cmd}") + command.run(args[1:]) + + +def main(): + """ + Main entry point + """ + run(sys.argv[1:]) + + +if __name__ == "__main__": + main() diff --git a/pyunicore/client.py b/pyunicore/client.py index ceceb78..0cd8c2f 100755 --- a/pyunicore/client.py +++ b/pyunicore/client.py @@ -426,7 +426,7 @@ def execute(self, cmd, login_node=None): return Job(self.transport, job_url) - def issue_auth_token(self, lifetime=-1, renewable=False, limited=False): + def issue_auth_token(self, lifetime=-1, renewable=False, limited=False) -> str: """ Issue an authentication token (JWT) from this UNICORE server Args: diff --git a/pyunicore/cwl/__init__.py b/pyunicore/cwl/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/pyunicore/cwlconverter.py b/pyunicore/cwl/cwlconverter.py similarity index 100% rename from pyunicore/cwlconverter.py rename to pyunicore/cwl/cwlconverter.py diff --git a/pyunicore/cwltool.py b/pyunicore/cwl/cwltool.py similarity index 92% rename from pyunicore/cwltool.py rename to pyunicore/cwl/cwltool.py index ace46c8..b5b3da6 100644 --- a/pyunicore/cwltool.py +++ b/pyunicore/cwl/cwltool.py @@ -4,7 +4,7 @@ import yaml -import pyunicore.cwlconverter +import pyunicore.cwl.cwlconverter as cwlconverter def read_cwl_files(cwl_doc_path, cwl_inputs_object_path=None, debug=False): @@ -51,7 +51,7 @@ def main(): unicore_job, file_list, outputs_list, - ) = pyunicore.cwlconverter.convert_cmdline_tool(cwl_doc, cwl_inputs_object, debug=debug) + ) = cwlconverter.convert_cmdline_tool(cwl_doc, cwl_inputs_object, debug=debug) print(json.dumps(unicore_job, indent=2, sort_keys=True)) sys.exit(0) diff --git a/pyunicore/testing/__init__.py b/pyunicore/testing/__init__.py deleted file mode 100644 index 18c3abf..0000000 --- a/pyunicore/testing/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from pyunicore.testing.contexts import expect_raise_if_exception -from pyunicore.testing.pyunicore import FakeClient -from pyunicore.testing.pyunicore import FakeJob -from pyunicore.testing.pyunicore import FakeRegistry -from pyunicore.testing.pyunicore import FakeTransport diff --git a/pyunicore/uftp/__init__.py b/pyunicore/uftp/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/pyunicore/uftp.py b/pyunicore/uftp/uftp.py similarity index 100% rename from pyunicore/uftp.py rename to pyunicore/uftp/uftp.py diff --git a/pyunicore/uftpfs.py b/pyunicore/uftp/uftpfs.py similarity index 98% rename from pyunicore/uftpfs.py rename to pyunicore/uftp/uftpfs.py index 5831972..e4ea3c2 100644 --- a/pyunicore/uftpfs.py +++ b/pyunicore/uftp/uftpfs.py @@ -4,7 +4,7 @@ from fs.opener import Opener import pyunicore.credentials as uc_credentials -from pyunicore.uftp import UFTP +from pyunicore.uftp.uftp import UFTP class UFTPFS(FTPFS): diff --git a/pyunicore/uftpfuse.py b/pyunicore/uftp/uftpfuse.py similarity index 99% rename from pyunicore/uftpfuse.py rename to pyunicore/uftp/uftpfuse.py index 0a351f5..c1615d7 100644 --- a/pyunicore/uftpfuse.py +++ b/pyunicore/uftp/uftpfuse.py @@ -14,7 +14,7 @@ from fuse import FuseOSError from fuse import Operations -from pyunicore.uftp import UFTP +from pyunicore.uftp.uftp import UFTP class UFTPFile: diff --git a/setup.py b/setup.py index a27b805..4b77da0 100755 --- a/setup.py +++ b/setup.py @@ -10,10 +10,10 @@ Visit https://github.com/HumanBrainProject/pyunicore for more information. """ -python_requires = ">=3" +python_requires = ">=3.7" install_requires = [ - "PyJWT>=2.0", + "pyjwt>=2.8", "requests>=2.5", ] @@ -37,12 +37,13 @@ extras_require=extras_require, entry_points={ "fs.opener": [ - "uftp = pyunicore.uftpfs:UFTPOpener", + "uftp = pyunicore.uftp.uftpfs:UFTPOpener", ], "console_scripts": [ "unicore-port-forwarder=pyunicore.forwarder:main", - "unicore-cwl-runner=pyunicore.cwltool:main", - "unicore-fusedriver=pyunicore.uftpfuse:main", + "unicore-cwl-runner=pyunicore.cwl.cwltool:main", + "unicore-fusedriver=pyunicore.uftp.uftpfuse:main", + "unicore=pyunicore.cli.main:main", ], }, license="License :: OSI Approved :: BSD", diff --git a/tests/integration/cli/preferences b/tests/integration/cli/preferences new file mode 100644 index 0000000..8e1eb0d --- /dev/null +++ b/tests/integration/cli/preferences @@ -0,0 +1,18 @@ +# User preferences for UCC + +authentication-method=USERNAME +username=demouser +password=test123 + +verbose=true + +# +# The address(es) of the registries to contact +# (space separated list) +registry=https://localhost:8080/DEMO-SITE/rest/registries/default_registry +contact-registry=true + +# +# default directory for output +# +output=/tmp diff --git a/tests/integration/cli/test_base.py b/tests/integration/cli/test_base.py new file mode 100644 index 0000000..503f8cf --- /dev/null +++ b/tests/integration/cli/test_base.py @@ -0,0 +1,24 @@ +import unittest + +import pyunicore.cli.base as base + + +class TestBase(unittest.TestCase): + def test_base_setup(self): + cmd = base.Base() + cmd.config_file = "tests/integration/cli/preferences" + cmd.load_user_properties() + registry = cmd.create_registry() + self.assertTrue(len(registry.site_urls) > 0) + print(registry.site_urls) + + def test_issue_token(self): + cmd = base.IssueToken() + config_file = "tests/integration/cli/preferences" + ep = "https://localhost:8080/DEMO-SITE/rest/core" + args = ["-c", config_file, ep, "--lifetime", "700", "--inspect", "--limited", "--renewable"] + cmd.run(args) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/testing/__init__.py b/tests/testing/__init__.py new file mode 100644 index 0000000..42cb4e0 --- /dev/null +++ b/tests/testing/__init__.py @@ -0,0 +1,5 @@ +from tests.testing.contexts import expect_raise_if_exception +from tests.testing.pyunicore import FakeClient +from tests.testing.pyunicore import FakeJob +from tests.testing.pyunicore import FakeRegistry +from tests.testing.pyunicore import FakeTransport diff --git a/pyunicore/testing/contexts.py b/tests/testing/contexts.py similarity index 100% rename from pyunicore/testing/contexts.py rename to tests/testing/contexts.py diff --git a/pyunicore/testing/pyunicore.py b/tests/testing/pyunicore.py similarity index 100% rename from pyunicore/testing/pyunicore.py rename to tests/testing/pyunicore.py diff --git a/tests/unit/cli/preferences b/tests/unit/cli/preferences new file mode 100644 index 0000000..8e1eb0d --- /dev/null +++ b/tests/unit/cli/preferences @@ -0,0 +1,18 @@ +# User preferences for UCC + +authentication-method=USERNAME +username=demouser +password=test123 + +verbose=true + +# +# The address(es) of the registries to contact +# (space separated list) +registry=https://localhost:8080/DEMO-SITE/rest/registries/default_registry +contact-registry=true + +# +# default directory for output +# +output=/tmp diff --git a/tests/unit/cli/test_base.py b/tests/unit/cli/test_base.py new file mode 100644 index 0000000..47e8de2 --- /dev/null +++ b/tests/unit/cli/test_base.py @@ -0,0 +1,21 @@ +import unittest + +import pyunicore.cli.base as base +from pyunicore.credentials import UsernamePassword + + +class TestBase(unittest.TestCase): + def test_base_setup(self): + cmd = base.Base() + cmd.config_file = "tests/unit/cli/preferences" + cmd.load_user_properties() + self.assertEqual( + "https://localhost:8080/DEMO-SITE/rest/registries/default_registry", + cmd.config["registry"], + ) + cmd.create_credential() + self.assertTrue(type(cmd.credential) == UsernamePassword) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/cli/test_main.py b/tests/unit/cli/test_main.py new file mode 100644 index 0000000..acfd277 --- /dev/null +++ b/tests/unit/cli/test_main.py @@ -0,0 +1,29 @@ +import unittest + +import pyunicore.cli.main as main + + +class TestMain(unittest.TestCase): + def test_help(self): + main.help() + main.show_version() + for cmd in main._commands: + print("\n*** %s *** " % cmd) + c = main.get_command(cmd) + print(c.get_synopsis()) + c.parser.print_usage() + c.parser.print_help() + + def test_run_args(self): + main.run([]) + main.run(["--version"]) + main.run(["--help"]) + try: + main.run(["no-such-cmd"]) + self.fail() + except ValueError: + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/helpers/connection/test_registry_helper.py b/tests/unit/helpers/connection/test_registry_helper.py index d0240dd..3f92b68 100644 --- a/tests/unit/helpers/connection/test_registry_helper.py +++ b/tests/unit/helpers/connection/test_registry_helper.py @@ -6,7 +6,7 @@ import pyunicore.client import pyunicore.credentials as credentials import pyunicore.helpers.connection.registry as _registry -import pyunicore.testing as testing +import tests.testing as testing @pytest.fixture() diff --git a/tests/unit/helpers/connection/test_site.py b/tests/unit/helpers/connection/test_site.py index fc73bcb..d7ef4ac 100644 --- a/tests/unit/helpers/connection/test_site.py +++ b/tests/unit/helpers/connection/test_site.py @@ -5,7 +5,7 @@ import pyunicore.client as pyunicore import pyunicore.credentials as credentials import pyunicore.helpers.connection.site as _connect -import pyunicore.testing as testing +import tests.testing as testing @pytest.fixture() diff --git a/tests/unit/helpers/workflows/test_variable.py b/tests/unit/helpers/workflows/test_variable.py index ea157fc..b445f5c 100644 --- a/tests/unit/helpers/workflows/test_variable.py +++ b/tests/unit/helpers/workflows/test_variable.py @@ -1,7 +1,7 @@ import pytest -from pyunicore import testing from pyunicore.helpers.workflows import variable +from tests import testing class TestVariable: diff --git a/tests/unit/test_cwl1.py b/tests/unit/test_cwl1.py index e0e8268..4e03c3f 100644 --- a/tests/unit/test_cwl1.py +++ b/tests/unit/test_cwl1.py @@ -1,8 +1,8 @@ import json import unittest -import pyunicore.cwlconverter as cwlconverter -import pyunicore.cwltool as cwltool +import pyunicore.cwl.cwlconverter as cwlconverter +import pyunicore.cwl.cwltool as cwltool class TestCWL1(unittest.TestCase): diff --git a/tests/unit/test_uftpfs.py b/tests/unit/test_uftpfs.py index bb93a74..c8db6ee 100644 --- a/tests/unit/test_uftpfs.py +++ b/tests/unit/test_uftpfs.py @@ -1,7 +1,7 @@ import unittest from os import environ -from pyunicore.uftpfs import UFTPOpener +from pyunicore.uftp.uftpfs import UFTPOpener class TestUFTPFS(unittest.TestCase):