diff --git a/exodus_gw/aws/dynamodb.py b/exodus_gw/aws/dynamodb.py index 35c15186..e6f0b55b 100644 --- a/exodus_gw/aws/dynamodb.py +++ b/exodus_gw/aws/dynamodb.py @@ -46,14 +46,21 @@ def definitions(self): self._definitions = self.query_definitions() return self._definitions - def _aliases(self, alias_types: list[str]) -> list[tuple[str, str, list[str]]]: + def _aliases( + self, alias_types: list[str] + ) -> list[tuple[str, str, list[str]]]: out: list[tuple[str, str, list[str]]] = [] for k, v in self.definitions.items(): if k in alias_types: for alias in v: - out.append((alias["src"], alias["dest"], - alias.get("exclude_paths") or [])) + out.append( + ( + alias["src"], + alias["dest"], + alias.get("exclude_paths") or [], + ) + ) return out @@ -129,7 +136,9 @@ def aliases_for_flush(self) -> list[tuple[str, str, list[str]]]: # this alias. If we don't traverse the alias from dest => src then we # will miss the fact that /content/dist/rhel8/rhui paths should also # have cache flushed. - out = out + [(dest, src, exclusions) for (src, dest, exclusions) in out] + out = out + [ + (dest, src, exclusions) for (src, dest, exclusions) in out + ] return out diff --git a/exodus_gw/aws/util.py b/exodus_gw/aws/util.py index 9a1c36c3..d31e856d 100644 --- a/exodus_gw/aws/util.py +++ b/exodus_gw/aws/util.py @@ -161,7 +161,9 @@ def get_reader(cls, request): return cls(request) -def uri_alias(uri: str, aliases: list[tuple[str, str, list[str]]]) -> list[str]: +def uri_alias( + uri: str, aliases: list[tuple[str, str, list[str]]] +) -> list[str]: # Resolve every alias between paths within the uri (e.g. # allow RHUI paths to be aliased to non-RHUI). # @@ -222,12 +224,13 @@ def add_out(new_uri: str) -> bool: # Used to replicate old NetStorage-compatible behaviour. This will # typically match non-rpm paths, such as /images/ or /isos/ if any([exclusion in uri for exclusion in exclude_paths]): - LOG.debug("Aliasing for %s was not applied as it matches one " - "of the following exclusion paths: %s.", - uri, - ",".join(exclude_paths), - extra={"event": "publish", "success": True}, - ) + LOG.debug( + "Aliasing for %s was not applied as it matches one " + "of the following exclusion paths: %s.", + uri, + ",".join(exclude_paths), + extra={"event": "publish", "success": True}, + ) continue new_uri = uri.replace(src, dest, 1) diff --git a/exodus_gw/routers/config.py b/exodus_gw/routers/config.py index 94a345c1..2e0d1202 100644 --- a/exodus_gw/routers/config.py +++ b/exodus_gw/routers/config.py @@ -47,8 +47,8 @@ "type": "array", "items": {"type": "string"}, "description": "Paths for which alias will not be resolved, " - "treated as an unanchored regex." - } + "treated as an unanchored regex.", + }, }, }, "uniqueItems": True, @@ -143,27 +143,26 @@ def config_post( { "src": "/content/origin", "dest": "/origin", - "exclude_paths": [] + "exclude_paths": [], }, { "src": "/origin/rpm", "dest": "/origin/rpms", - "exclude_paths": ["/iso/"] + "exclude_paths": ["/iso/"], }, ], "releasever_alias": [ { "dest": "/content/dist/rhel8/8.5", "src": "/content/dist/rhel8/8", - "exclude_paths": ["/files/", "/images/", "/iso/"] + "exclude_paths": ["/files/", "/images/", "/iso/"], }, - ], "rhui_alias": [ { "dest": "/content/dist/rhel8", "src": "/content/dist/rhel8/rhui", - "exclude_paths": ["/files/", "/images/", "/iso/"] + "exclude_paths": ["/files/", "/images/", "/iso/"], }, ], } diff --git a/exodus_gw/routers/deploy.py b/exodus_gw/routers/deploy.py index d3e31ffa..d4071baf 100644 --- a/exodus_gw/routers/deploy.py +++ b/exodus_gw/routers/deploy.py @@ -59,27 +59,26 @@ def deploy_config( { "src": "/content/origin", "dest": "/origin", - "exclude_paths": [] + "exclude_paths": [], }, { "src": "/origin/rpm", "dest": "/origin/rpms", - "exclude_paths": ["/iso/"] + "exclude_paths": ["/iso/"], }, ], "releasever_alias": [ { "dest": "/content/dist/rhel8/8.5", "src": "/content/dist/rhel8/8", - "exclude_paths": ["/files/", "/images/", "/iso/"] + "exclude_paths": ["/files/", "/images/", "/iso/"], }, - ], "rhui_alias": [ { "dest": "/content/dist/rhel8", "src": "/content/dist/rhel8/rhui", - "exclude_paths": ["/files/", "/images/", "/iso/"] + "exclude_paths": ["/files/", "/images/", "/iso/"], }, ], } diff --git a/exodus_gw/schemas.py b/exodus_gw/schemas.py index bf1a6ee2..a45dbbf4 100644 --- a/exodus_gw/schemas.py +++ b/exodus_gw/schemas.py @@ -320,8 +320,9 @@ class Alias(BaseModel): ..., description="Target of the alias, relative to CDN root." ) exclude_paths: list[str] | None = Field( - [], description="Paths for which alias will not be resolved, " - "treated as an unanchored regex." + [], + description="Paths for which alias will not be resolved, " + "treated as an unanchored regex.", ) diff --git a/exodus_gw/worker/deploy.py b/exodus_gw/worker/deploy.py index 6682682a..813a08cb 100644 --- a/exodus_gw/worker/deploy.py +++ b/exodus_gw/worker/deploy.py @@ -152,8 +152,12 @@ def deploy_config( # If any original exclusion matches the uri, the uri wouldn't # have been treated as an alias, thus cache flushing would be # unnecessary. - if any([exclusion in published_path.web_uri - for exclusion in original_exclusions.get(src, [])]): + if any( + [ + exclusion in published_path.web_uri + for exclusion in original_exclusions.get(src, []) + ] + ): continue LOG.info( "Updated alias %s will flush cache for %s", diff --git a/tests/aws/test_uri_alias.py b/tests/aws/test_uri_alias.py index f25a812c..05e40348 100644 --- a/tests/aws/test_uri_alias.py +++ b/tests/aws/test_uri_alias.py @@ -145,37 +145,47 @@ def test_uri_alias_limit(caplog: pytest.LogCaptureFixture): ( "/content/dist/rhel9/9/x86_64/baseos/iso/PULP_MANIFEST", [ - ("/content/dist/rhel9/9", "/content/dist/rhel9/9.5", ["/iso/"]), + ( + "/content/dist/rhel9/9", + "/content/dist/rhel9/9.5", + ["/iso/"], + ), ], # just returns the original ["/content/dist/rhel9/9/x86_64/baseos/iso/PULP_MANIFEST"], "Aliasing for /content/dist/rhel9/9/x86_64/baseos/iso/PULP_MANIFEST " - "was not applied as it matches one of the following exclusion paths: /iso/." + "was not applied as it matches one of the following exclusion paths: /iso/.", ), ( "/some/path/with/file/in/it", [ - ("/some/path", "/another/different/path", ["/none/", "/here/"]), - ("/another/different/path", "/this/wont/alias", ["/file/"]) + ( + "/some/path", + "/another/different/path", + ["/none/", "/here/"], + ), + ("/another/different/path", "/this/wont/alias", ["/file/"]), + ], + [ + "/another/different/path/with/file/in/it", + "/some/path/with/file/in/it", ], - ["/another/different/path/with/file/in/it", - "/some/path/with/file/in/it"], "Aliasing for /another/different/path/with/file/in/it was not " - "applied as it matches one of the following exclusion paths: /file/." + "applied as it matches one of the following exclusion paths: /file/.", ), ( "/my/base/content/path/cool_iso_tool.rpm", [ ("/my/base", "/your/own", ["/iso/"]), ], - ["/your/own/content/path/cool_iso_tool.rpm", - "/my/base/content/path/cool_iso_tool.rpm"], + [ + "/your/own/content/path/cool_iso_tool.rpm", + "/my/base/content/path/cool_iso_tool.rpm", + ], "Resolved alias:\\n\\tsrc: /my/base/content/path/cool_iso_tool.rpm" - "\\n\\tdest: /your/own/content/path/cool_iso_tool.rpm" - + "\\n\\tdest: /your/own/content/path/cool_iso_tool.rpm", ), ], - ids=["basic", "transitive", "filename"], ) def test_uri_alias_exclusions(input, aliases, output, log_message, caplog): diff --git a/tests/conftest.py b/tests/conftest.py index 1f63e49f..8dd8f76f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,7 +16,6 @@ from .async_utils import BlockDetector - DEFAULT_EXCLUDE_PATHS = ["/files/", "/images/", "/iso/"] diff --git a/tests/worker/test_cdn_cache.py b/tests/worker/test_cdn_cache.py index 95bd1339..d75ec55d 100644 --- a/tests/worker/test_cdn_cache.py +++ b/tests/worker/test_cdn_cache.py @@ -212,17 +212,23 @@ def test_flush_cdn_cache_typical( { "src": "/path/one", "dest": "/path/one-dest", - "exclude_paths": ["/files/", "/images/", - "/iso/"], + "exclude_paths": [ + "/files/", + "/images/", + "/iso/", + ], }, ], "rhui_alias": [ { "src": "/path/rhui/two", "dest": "/path/two", - "exclude_paths": ["/files/", "/images/", - "/iso/"], - }, + "exclude_paths": [ + "/files/", + "/images/", + "/iso/", + ], + }, ], } ) diff --git a/tests/worker/test_deploy.py b/tests/worker/test_deploy.py index d236a972..8f7146c6 100644 --- a/tests/worker/test_deploy.py +++ b/tests/worker/test_deploy.py @@ -166,7 +166,7 @@ def test_deploy_config_with_flush( { "dest": "/content/testproduct/1.2.0", "src": "/content/testproduct/1", - "exclude_paths": ["/newExclusion/"] + "exclude_paths": ["/newExclusion/"], }, ] worker.deploy_config(updated_config, "test", NOW_UTC) @@ -222,7 +222,7 @@ def test_deploy_config_with_flush( "/content/dist/rhel/server/listing", "/content/testproduct/1/file1", "/content/testproduct/1/file2", - "/content/testproduct/1/newExclusion/file5" + "/content/testproduct/1/newExclusion/file5", ] # And actor call should have been delayed by this long.