re-organise code; start work on command-line client

Author: BerndSchuller

README.md

@@ -14,18 +14,18 @@ for filesystem mounts with FUSE, a UFTP driver for
 and a UNICORE implementation of a
 [Dask Cluster](https://distributed.dask.org/en/stable/)
 
-This project has received funding from the European Union’s 
-Horizon 2020 Framework Programme for Research and Innovation under the 
-Specific Grant Agreement Nos. 720270, 785907 and 945539 
+This project has received funding from the European Union’s
+Horizon 2020 Framework Programme for Research and Innovation under the
+Specific Grant Agreement Nos. 720270, 785907 and 945539
 (Human Brain Project SGA 1, 2 and 3)
 
 See LICENSE file for licensing information
 
 ## Documentation
 
-The complete documentation of PyUNICORE can be viewed 
+The complete documentation of PyUNICORE can be viewed
 [here](https://pyunicore.readthedocs.io/en/latest/)
- 
+
 ## Installation
 
 Install from PyPI with
@@ -61,7 +61,7 @@ client = uc_client.Client(credential, base_url)
 print(json.dumps(client.properties, indent = 2))
 ```
 
-PyUNICORE supports a variety of 
+PyUNICORE supports a variety of
 [authentication options](https://pyunicore.readthedocs.io/en/latest/authentication.html).
 
 ### Run a job and read result files
@@ -188,7 +188,7 @@ unicore-cwl-runner echo.cwl hello_world.yml > hello_world.u
 
 ## Helpers
 
-The `pyunicore.helpers` module provides a set of higher-level APIs:
+The `pyunicore.helpers` module provides helper code for:
 
 * Connecting to
   * a Registry (`pyunicore.helpers.connect_to_registry`).

pyunicore/cli/__init__.py

@@ -0,0 +1,176 @@
+""" Base command class """
+import argparse
+import getpass
+import json
+import os.path
+from base64 import b64decode
+
+import pyunicore.client
+import pyunicore.credentials
+
+
+class Base:
+    """Base command class with support for common commandline args"""
+
+    def __init__(self, password_source=None):
+        self.parser = argparse.ArgumentParser(
+            prog="unicore", description="A commandline client for UNICORE"
+        )
+        self.config = {"verbose": False}
+        self.args = None
+        self.credential = None
+        self.registry = None
+        self.add_base_args()
+        self.add_command_args()
+        if password_source:
+            self.password_source = password_source
+        else:
+            self.password_source = getpass.getpass
+
+    def _value(self, value: str):
+        if value.lower() == "true":
+            return True
+        if value.lower() == "false":
+            return False
+        return value
+
+    def load_user_properties(self):
+        with open(self.config_file) as f:
+            for line in f.readlines():
+                line = line.strip()
+                if line.startswith("#"):
+                    continue
+                try:
+                    key, value = line.split("=", 1)
+                    self.config[key] = self._value(value)
+                except ValueError:
+                    pass
+
+    def add_base_args(self):
+        self.parser.add_argument(
+            "-v", "--verbose", required=False, action="store_true", help="Be verbose"
+        )
+        self.parser.add_argument(
+            "-c",
+            "--configuration",
+            metavar="CONFIG",
+            default=f"{os.getenv('HOME')}/.ucc/properties",
+            help="Configuration file",
+        )
+
+    def add_command_args(self):
+        pass
+
+    def run(self, args):
+        self.args = self.parser.parse_args(args)
+        self.is_verbose = self.args.verbose
+        self.config_file = self.args.configuration
+        self.load_user_properties()
+        self.create_credential()
+        self.registry = self.create_registry()
+
+    def get_synopsis(self):
+        return "N/A"
+
+    def create_credential(self):
+        auth_method = self.config.get("authentication-method", "USERNAME").upper()
+        if "USERNAME" == auth_method:
+            username = self.config["username"]
+            password = self._get_password()
+        self.credential = pyunicore.credentials.create_credential(username, password)
+
+    def _get_password(self, key="password") -> str:
+        password = self.config.get(key)
+        if password is None:
+            _p = os.getenv("UCC_PASSWORD")
+            if not _p:
+                pwd_prompt = "Enter password: "
+                password = self.password_source(pwd_prompt)
+            else:
+                password = _p
+        return password
+
+    def create_registry(self) -> pyunicore.client.Registry:
+        self.create_credential()
+        return pyunicore.client.Registry(self.credential, self.config["registry"])
+
+    def verbose(self, msg):
+        if self.is_verbose:
+            print(msg)
+
+    def human_readable(self, value, decimals=0):
+        for unit in ["B", "KB", "MB", "GB"]:
+            if value < 1024.0 or unit == "GB":
+                break
+            value /= 1024.0
+        return f"{value:.{decimals}f} {unit}"
+
+
+class IssueToken(Base):
+    def add_command_args(self):
+        self.parser.prog = "unicore issue-token"
+        self.parser.description = self.get_synopsis()
+        self.parser.add_argument("URL", help="Token endpoint URL")
+        self.parser.add_argument("-s", "--sitename", required=False, type=str, help="Site name")
+        self.parser.add_argument(
+            "-l",
+            "--lifetime",
+            required=False,
+            type=int,
+            default=-1,
+            help="Initial lifetime (in seconds) for token.",
+        )
+        self.parser.add_argument(
+            "-R",
+            "--renewable",
+            required=False,
+            action="store_true",
+            help="Token can be used to get a fresh token.",
+        )
+        self.parser.add_argument(
+            "-L",
+            "--limited",
+            required=False,
+            action="store_true",
+            help="Token should be limited to the issuing server.",
+        )
+        self.parser.add_argument(
+            "-I", "--inspect", required=False, action="store_true", help="Inspect the issued token."
+        )
+
+    def get_synopsis(self):
+        return """Get a JWT token from a UNICORE/X server"""
+
+    def run(self, args):
+        super().run(args)
+        endpoint = self.args.URL
+        site_name = self.args.sitename
+        if site_name:
+            endpoint = self.registry.site(site_name).resource_url
+        else:
+            if endpoint is None:
+                raise ValueError("Either --sitename or URL must be given.")
+            endpoint = endpoint.split("/token")[0]
+        token = self.issue_token(
+            url=endpoint,
+            lifetime=self.args.lifetime,
+            limited=self.args.limited,
+            renewable=self.args.renewable,
+        )
+        if self.args.inspect:
+            self.show_token_details(token)
+        print(token)
+
+    def issue_token(self, url: str, lifetime: int, limited: bool, renewable: bool) -> str:
+        client = pyunicore.client.Client(self.credential, site_url=url)
+        return client.issue_auth_token(lifetime, renewable, limited)
+
+    def show_token_details(self, token: str):
+        _p = token.split(".")[1]
+        _p += "=" * (-len(_p) % 4)  # padding
+        payload = json.loads(b64decode(_p))
+        print(f"Subject:      {payload['sub']}")
+        print(f"Lifetime (s): {payload['exp']-payload['iat']}")
+        print(f"Issued by:    {payload['iss']}")
+        print(f"Valid for:    {payload.get('aud', '<unlimited>')}")
+        print(f"Renewable:    {payload.get('renewable', 'no')}")

pyunicore/cli/base.py

@@ -0,0 +1,66 @@
+""" Main client class """
+import platform
+import sys
+
+import pyunicore.cli.base
+
+_commands = {
+    "issue-token": pyunicore.cli.base.IssueToken,
+}
+
+
+def get_command(name):
+    return _commands.get(name)()
+
+
+def show_version():
+    print(
+        "UNICORE Commandline Client (pyUNICORE) "
+        "%s, https://www.unicore.eu" % pyunicore._version.get_versions().get("version", "n/a")
+    )
+    print("Python %s" % sys.version)
+    print("OS: %s" % platform.platform())
+
+
+def help():
+    s = """UNICORE Commandline Client (pyUNICORE) %s, https://www.unicore.eu
+Usage: unicore <command> [OPTIONS] <args>
+The following commands are available:""" % pyunicore._version.get_versions().get(
+        "version", "n/a"
+    )
+    print(s)
+    for cmd in sorted(_commands):
+        print(f" {cmd:20} - {get_command(cmd).get_synopsis()}")
+    print("Enter 'unicore <command> -h' for help on a particular command.")
+
+
+def run(args):
+    _help = ["help", "-h", "--help"]
+    if len(args) < 1 or args[0] in _help:
+        help()
+        return
+    _version = ["version", "-V", "--version"]
+    if args[0] in _version:
+        show_version()
+        return
+
+    command = None
+    cmd = args[0]
+    for k in _commands:
+        if k.startswith(cmd):
+            command = get_command(k)
+            break
+    if command is None:
+        raise ValueError(f"No such command: {cmd}")
+    command.run(args[1:])
+
+
+def main():
+    """
+    Main entry point
+    """
+    run(sys.argv[1:])
+
+
+if __name__ == "__main__":
+    main()

pyunicore/cli/main.py

@@ -426,7 +426,7 @@ def execute(self, cmd, login_node=None):
 
         return Job(self.transport, job_url)
 
-    def issue_auth_token(self, lifetime=-1, renewable=False, limited=False):
+    def issue_auth_token(self, lifetime=-1, renewable=False, limited=False) -> str:
         """
         Issue an authentication token (JWT) from this UNICORE server
         Args:

pyunicore/client.py

@@ -4,7 +4,7 @@
 
 import yaml
 
-import pyunicore.cwlconverter
+import pyunicore.cwl.cwlconverter as cwlconverter
 
 
 def read_cwl_files(cwl_doc_path, cwl_inputs_object_path=None, debug=False):
@@ -51,7 +51,7 @@ def main():
         unicore_job,
         file_list,
         outputs_list,
-    ) = pyunicore.cwlconverter.convert_cmdline_tool(cwl_doc, cwl_inputs_object, debug=debug)
+    ) = cwlconverter.convert_cmdline_tool(cwl_doc, cwl_inputs_object, debug=debug)
     print(json.dumps(unicore_job, indent=2, sort_keys=True))
 
     sys.exit(0)

pyunicore/cwl/__init__.py

@@ -4,7 +4,7 @@
 from fs.opener import Opener
 
 import pyunicore.credentials as uc_credentials
-from pyunicore.uftp import UFTP
+from pyunicore.uftp.uftp import UFTP
 
 
 class UFTPFS(FTPFS):

pyunicore/cwlconverter.py renamed to pyunicore/cwl/cwlconverter.py

@@ -14,7 +14,7 @@
     from fuse import FuseOSError
     from fuse import Operations
 
-from pyunicore.uftp import UFTP
+from pyunicore.uftp.uftp import UFTP
 
 
 class UFTPFile:

pyunicore/cwltool.py renamed to pyunicore/cwl/cwltool.py

@@ -10,10 +10,10 @@
 Visit https://github.com/HumanBrainProject/pyunicore for more information.
 """
 
-python_requires = ">=3"
+python_requires = ">=3.7"
 
 install_requires = [
-    "PyJWT>=2.0",
+    "pyjwt>=2.8",
     "requests>=2.5",
 ]
 
@@ -37,12 +37,13 @@
     extras_require=extras_require,
     entry_points={
         "fs.opener": [
-            "uftp = pyunicore.uftpfs:UFTPOpener",
+            "uftp = pyunicore.uftp.uftpfs:UFTPOpener",
         ],
         "console_scripts": [
             "unicore-port-forwarder=pyunicore.forwarder:main",
-            "unicore-cwl-runner=pyunicore.cwltool:main",
-            "unicore-fusedriver=pyunicore.uftpfuse:main",
+            "unicore-cwl-runner=pyunicore.cwl.cwltool:main",
+            "unicore-fusedriver=pyunicore.uftp.uftpfuse:main",
+            "unicore=pyunicore.cli.main:main",
         ],
     },
     license="License :: OSI Approved :: BSD",

pyunicore/testing/__init__.py

@@ -0,0 +1,18 @@
+# User preferences for UCC
+
+authentication-method=USERNAME
+username=demouser
+password=test123
+
+verbose=true
+
+#
+# The address(es) of the registries to contact
+# (space separated list)
+registry=https://localhost:8080/DEMO-SITE/rest/registries/default_registry
+contact-registry=true
+
+#
+# default directory for output
+#
+output=/tmp