From c035442be01cc4b610fef6ffcfb8f17ebe9844b0 Mon Sep 17 00:00:00 2001 From: vsoch Date: Tue, 30 Aug 2022 19:14:37 -0600 Subject: [PATCH] start of work to add private remotes Signed-off-by: vsoch --- CHANGELOG.md | 1 + shpc/client/sync.py | 3 +- shpc/main/modules/base.py | 16 +++-------- shpc/main/registry/__init__.py | 2 +- shpc/main/registry/remote.py | 50 ++++++++++++++++++++++++++++++---- shpc/tests/test_sync.py | 29 ++++++++++++++++++++ shpc/version.py | 2 +- 7 files changed, 82 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb3d078f0..9ad14a5c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and **Merged pull requests**. Critical items to know are: The versions coincide with releases on pip. Only major versions will be released as tags on Github. ## [0.0.x](https://github.com/singularityhub/singularity-hpc/tree/main) (0.0.x) + - Support for remotes that do not expose library.json (0.0.12) - Update add to return container yaml (0.1.11) - Fixing bug with writing package file in update (0.1.1) - Add support for remote registry and sync commands --all (0.1.0) diff --git a/shpc/client/sync.py b/shpc/client/sync.py index 6037a9a97..98cee4a8e 100644 --- a/shpc/client/sync.py +++ b/shpc/client/sync.py @@ -2,9 +2,10 @@ __copyright__ = "Copyright 2021-2022, Vanessa Sochat" __license__ = "MPL 2.0" +import os + import shpc.logger as logger import shpc.utils -import os def sync_registry(args, parser, extra, subparser): diff --git a/shpc/main/modules/base.py b/shpc/main/modules/base.py index fc5f921c3..435a14c02 100644 --- a/shpc/main/modules/base.py +++ b/shpc/main/modules/base.py @@ -15,7 +15,7 @@ import shpc.main.modules.template as templatectl import shpc.main.modules.versions as versionfile import shpc.main.modules.views as views -import shpc.main.registry as registry +import shpc.main.registry import shpc.utils as utils from shpc.logger import logger from shpc.main.client import Client as BaseClient @@ -198,7 +198,7 @@ def add(self, image, module_name=None, **kwargs): # Load config (but don't validate yet!) config = container.ContainerConfig( - registry.FilesystemResult(module_name, template), validate=False + shpc.main.registry.FilesystemResult(module_name, template), validate=False ) return self.container.add( module_name, image, config, container_yaml=dest, **kwargs @@ -236,16 +236,8 @@ def docgen(self, module_name, registry=None, out=None, branch="main"): template = self.template.load("docs.md") registry = registry or defaults.github_url github_url = "%s/blob/%s/%s/container.yaml" % (registry, branch, module_name) - registry_bare = registry.split(".com")[-1] - raw = ( - "https://gitlab.com/%s/-/raw/%s/%s/container.yaml" - if "gitlab" in registry - else "https://raw.githubusercontent.com/%s/%s/%s/container.yaml" - ) - raw_github_url = raw % ( - registry_bare, - branch, - module_name, + raw_github_url = shpc.main.registry.get_module_config_url( + registry, module_name, branch ) # Currently one doc is rendered for all containers diff --git a/shpc/main/registry/__init__.py b/shpc/main/registry/__init__.py index af4d2a983..f98185c68 100644 --- a/shpc/main/registry/__init__.py +++ b/shpc/main/registry/__init__.py @@ -14,7 +14,7 @@ from shpc.main.settings import SettingsBase from .filesystem import Filesystem, FilesystemResult -from .remote import GitHub, GitLab +from .remote import GitHub, GitLab, get_module_config_url def update_container_module(module, from_path, existing_path): diff --git a/shpc/main/registry/remote.py b/shpc/main/registry/remote.py index c06142df7..6053a079d 100644 --- a/shpc/main/registry/remote.py +++ b/shpc/main/registry/remote.py @@ -5,8 +5,8 @@ import os import re +import shutil import subprocess as sp -import sys import requests @@ -16,6 +16,23 @@ from .provider import Provider, Result +def get_module_config_url(registry, module_name, branch="main"): + """ + Get the raw address of the config (container.yaml) + """ + registry_bare = registry.split(".com")[-1] + raw = ( + "https://gitlab.com/%s/-/raw/%s/%s/container.yaml" + if "gitlab" in registry + else "https://raw.githubusercontent.com/%s/%s/%s/container.yaml" + ) + return raw % ( + registry_bare, + branch, + module_name, + ) + + class RemoteResult(Result): """ A remote result provides courtesy functions for interacting with @@ -117,6 +134,9 @@ def exists(self, name): """ Determine if a module exists in the registry. """ + name = name.split(":")[0] + if self._cache and name in self._cache: + return True dirname = self.source if self.subdir: dirname = os.path.join(dirname, self.subdir) @@ -158,7 +178,8 @@ def find(self, name): """ Find a particular entry in a registry """ - self._update_cache() + if not self._cache: + self._update_cache() if name in self._cache: return RemoteResult(name, self._cache[name]) @@ -172,12 +193,29 @@ def _update_cache(self, force=False): # Check for exposed library API on GitHub or GitLab pages response = requests.get(self.web_url) if response.status_code != 200: - sys.exit( - "Remote %s is not deploying a Registry API (%s). Open a GitHub issue to ask for help." - % (self.source, self.web_url) - ) + return self._update_clone_cache() self._cache = response.json() + def _update_clone_cache(self): + """ + Given a remote that does not expose a library.json, handle via clone. + """ + logger.warning( + "Remote %s is not deploying a Registry API, falling back to clone." + % self.source + ) + tmpdir = self.clone() + for dirname, module in self.iter_modules(): + # Minimum amount of metadata to function here + config_url = get_module_config_url(self.source, module) + self._cache[module] = { + "config": shpc.utils.read_yaml( + os.path.join(dirname, module, "container.yaml") + ), + "config_url": config_url, + } + shutil.rmtree(tmpdir) + def iter_registry(self, filter_string=None): """ Yield metadata about containers in a remote registry. diff --git a/shpc/tests/test_sync.py b/shpc/tests/test_sync.py index 1de5b74ad..7008a8e26 100644 --- a/shpc/tests/test_sync.py +++ b/shpc/tests/test_sync.py @@ -106,6 +106,8 @@ def test_sync_from_file(tmp_path): [ "https://github.com/singularityhub/shpc-registry", "https://gitlab.com/singularityhub/shpc-registry", + # This registry does not expose a web UI + "https://github.com/researchapps/shpc-test-registry", ], ) def test_remote_upgrade(tmp_path, remote): @@ -133,3 +135,30 @@ def test_remote_upgrade(tmp_path, remote): client.registry.sync(sync_registry=remote) assert list(client.registry.iter_modules()) + + +@pytest.mark.parametrize( + "remote", + [ + "https://github.com/singularityhub/shpc-registry", + "https://gitlab.com/singularityhub/shpc-registry", + # This registry does not expose a web UI + "https://github.com/researchapps/shpc-test-registry", + ], +) +def test_registry_interaction(tmp_path, remote): + """ + Test interactions with registries of different types + """ + client = init_client(str(tmp_path), "lmod", "singularity") + reg = client.registry.get_registry(remote) + + assert not reg.is_filesystem_registry + + # This will hit the underlying logic to list/show + mods = list(reg.iter_registry()) + assert mods + + # Should use the cache + assert reg.exists("vanessa/salad") + assert reg.find("vanessa/salad") is not None diff --git a/shpc/version.py b/shpc/version.py index a22ef5373..e295dd9c9 100644 --- a/shpc/version.py +++ b/shpc/version.py @@ -2,7 +2,7 @@ __copyright__ = "Copyright 2021-2022, Vanessa Sochat" __license__ = "MPL 2.0" -__version__ = "0.1.11" +__version__ = "0.1.12" AUTHOR = "Vanessa Sochat" EMAIL = "vsoch@users.noreply.github.com" NAME = "singularity-hpc"