From e1de2b0ec6584b681871d9dd529b920aac6b0a5e Mon Sep 17 00:00:00 2001 From: Florian Pinault Date: Mon, 18 Nov 2024 08:44:12 +0100 Subject: [PATCH] prepare 0.4.8 (#56) * Fix s3 download for folders (#54) * fix ssh (#55) --- src/anemoi/utils/remote/__init__.py | 6 ++++-- src/anemoi/utils/remote/s3.py | 3 +++ src/anemoi/utils/remote/ssh.py | 5 +++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/anemoi/utils/remote/__init__.py b/src/anemoi/utils/remote/__init__.py index d814119..722a685 100644 --- a/src/anemoi/utils/remote/__init__.py +++ b/src/anemoi/utils/remote/__init__.py @@ -138,6 +138,8 @@ def copy(self, source, target, **kwargs): raise NotImplementedError def get_temporary_target(self, target, pattern): + if pattern is None: + return target dirname, basename = os.path.split(target) return pattern.format(dirname=dirname, basename=basename) @@ -199,13 +201,13 @@ def __init__( target = os.path.basename(source) temporary_target = { - False: "{dirname}/{basename}", + False: None, True: "{dirname}-downloading/{basename}", "-tmp/*": "{dirname}-tmp/{basename}", "*-tmp": "{dirname}/{basename}-tmp", "tmp-*": "{dirname}/tmp-{basename}", }.get(temporary_target, temporary_target) - assert isinstance(temporary_target, str), (type(temporary_target), temporary_target) + assert temporary_target is None or isinstance(temporary_target, str), (type(temporary_target), temporary_target) self.source = source self.target = target diff --git a/src/anemoi/utils/remote/s3.py b/src/anemoi/utils/remote/s3.py index 672d182..1100f78 100644 --- a/src/anemoi/utils/remote/s3.py +++ b/src/anemoi/utils/remote/s3.py @@ -192,6 +192,9 @@ def _transfer_file(self, source, target, overwrite, resume, verbosity, threads, _, _, bucket, key = source.split("/", 3) s3 = s3_client(bucket) + if key.endswith("/"): + return 0 + try: response = s3.head_object(Bucket=bucket, Key=key) except s3.exceptions.ClientError as e: diff --git a/src/anemoi/utils/remote/ssh.py b/src/anemoi/utils/remote/ssh.py index 99c407c..423f7db 100644 --- a/src/anemoi/utils/remote/ssh.py +++ b/src/anemoi/utils/remote/ssh.py @@ -51,8 +51,9 @@ def _parse_target(self, target): def get_temporary_target(self, target, pattern): hostname, path = self._parse_target(target) - dirname, basename = os.path.split(path) - path = pattern.format(dirname=dirname, basename=basename) + if pattern is not None: + dirname, basename = os.path.split(path) + path = pattern.format(dirname=dirname, basename=basename) return f"ssh://{hostname}:{path}" def rename_target(self, target, new_target):