Merge pull request #755 from amcmahon-rh/exclude_paths

Add exclusions for aliases [RHELDST-26013]

exodus_gw/aws/dynamodb.py

@@ -46,18 +46,26 @@ def definitions(self):
                     self._definitions = self.query_definitions()
         return self._definitions
 
-    def _aliases(self, alias_types: list[str]) -> list[tuple[str, str]]:
-        out: list[tuple[str, str]] = []
+    def _aliases(
+        self, alias_types: list[str]
+    ) -> list[tuple[str, str, list[str]]]:
+        out: list[tuple[str, str, list[str]]] = []
 
         for k, v in self.definitions.items():
             if k in alias_types:
                 for alias in v:
-                    out.append((alias["src"], alias["dest"]))
+                    out.append(
+                        (
+                            alias["src"],
+                            alias["dest"],
+                            alias.get("exclude_paths") or [],
+                        )
+                    )
 
         return out
 
     @property
-    def aliases_for_write(self) -> list[tuple[str, str]]:
+    def aliases_for_write(self) -> list[tuple[str, str, list[str]]]:
         # Aliases used when writing items to DynamoDB.
         #
         # Note that these aliases are traversed only in the src => dest
@@ -95,7 +103,7 @@ def aliases_for_write(self) -> list[tuple[str, str]]:
         return self._aliases(["origin_alias", "releasever_alias"])
 
     @property
-    def aliases_for_flush(self) -> list[tuple[str, str]]:
+    def aliases_for_flush(self) -> list[tuple[str, str, list[str]]]:
         # Aliases used when flushing cache.
         out = self._aliases(["origin_alias", "releasever_alias", "rhui_alias"])
 
@@ -128,7 +136,9 @@ def aliases_for_flush(self) -> list[tuple[str, str]]:
         # this alias. If we don't traverse the alias from dest => src then we
         # will miss the fact that /content/dist/rhel8/rhui paths should also
         # have cache flushed.
-        out = out + [(dest, src) for (src, dest) in out]
+        out = out + [
+            (dest, src, exclusions) for (src, dest, exclusions) in out
+        ]
 
         return out
 

exodus_gw/aws/util.py

@@ -161,7 +161,9 @@ def get_reader(cls, request):
         return cls(request)
 
 
-def uri_alias(uri: str, aliases: list[tuple[str, str]]) -> list[str]:
+def uri_alias(
+    uri: str, aliases: list[tuple[str, str, list[str]]]
+) -> list[str]:
     # Resolve every alias between paths within the uri (e.g.
     # allow RHUI paths to be aliased to non-RHUI).
     #
@@ -187,7 +189,7 @@ def uri_alias(uri: str, aliases: list[tuple[str, str]]) -> list[str]:
 def uri_alias_recurse(
     accum: list[str],
     uri: str,
-    aliases: list[tuple[str, str]],
+    aliases: list[tuple[str, str, list[str]]],
     depth=0,
     maxdepth=4,
 ):
@@ -217,8 +219,20 @@ def add_out(new_uri: str) -> bool:
         accum.insert(0, new_uri)
         return out
 
-    for src, dest in aliases:
+    for src, dest, exclude_paths in aliases:
         if uri.startswith(src + "/") or uri == src:
+            # Used to replicate old NetStorage-compatible behaviour. This will
+            # typically match non-rpm paths, such as /images/ or /isos/
+            if any([re.search(exclusion, uri) for exclusion in exclude_paths]):
+                LOG.debug(
+                    "Aliasing for %s was not applied as it matches one "
+                    "of the following exclusion paths: %s.",
+                    uri,
+                    ",".join(exclude_paths),
+                    extra={"event": "publish", "success": True},
+                )
+                continue
+
             new_uri = uri.replace(src, dest, 1)
             LOG.debug(
                 "Resolved alias:\n\tsrc: %s\n\tdest: %s",
@@ -252,8 +266,8 @@ def add_out(new_uri: str) -> bool:
                 # But this is not desired behavior, we know that every alias is intended
                 # to be resolved in the URL a maximum of once, hence this adjustment.
                 sub_aliases = [
-                    (subsrc, subdest)
-                    for (subsrc, subdest) in aliases
+                    (subsrc, subdest, exclusions)
+                    for (subsrc, subdest, exclusions) in aliases
                     if (subsrc, subdest) != (src, dest)
                 ]
 
@@ -267,7 +281,7 @@ def add_out(new_uri: str) -> bool:
 
 
 def uris_with_aliases(
-    uris: Iterable[str], aliases: list[tuple[str, str]]
+    uris: Iterable[str], aliases: list[tuple[str, str, list[str]]]
 ) -> list[str]:
     # Given a collection of uris and aliases, returns a new collection of uris
     # post alias resolution, including *both* sides of each alias when applicable.

exodus_gw/routers/config.py

@@ -43,6 +43,12 @@
                 "pattern": PATH_PATTERN,
                 "description": "Target of the alias, relative to CDN root.",
             },
+            "exclude_paths": {
+                "type": "array",
+                "items": {"type": "string"},
+                "description": "Paths for which alias will not be resolved, "
+                "treated as an unanchored regex.",
+            },
         },
     },
     "uniqueItems": True,
@@ -134,19 +140,29 @@ def config_post(
                     },
                 },
                 "origin_alias": [
-                    {"src": "/content/origin", "dest": "/origin"},
-                    {"src": "/origin/rpm", "dest": "/origin/rpms"},
+                    {
+                        "src": "/content/origin",
+                        "dest": "/origin",
+                        "exclude_paths": [],
+                    },
+                    {
+                        "src": "/origin/rpm",
+                        "dest": "/origin/rpms",
+                        "exclude_paths": ["/iso/"],
+                    },
                 ],
                 "releasever_alias": [
                     {
                         "dest": "/content/dist/rhel8/8.5",
                         "src": "/content/dist/rhel8/8",
+                        "exclude_paths": ["/files/", "/images/", "/iso/"],
                     },
                 ],
                 "rhui_alias": [
                     {
                         "dest": "/content/dist/rhel8",
                         "src": "/content/dist/rhel8/rhui",
+                        "exclude_paths": ["/files/", "/images/", "/iso/"],
                     },
                 ],
             }

exodus_gw/routers/deploy.py

@@ -56,19 +56,29 @@ def deploy_config(
                     },
                 },
                 "origin_alias": [
-                    {"src": "/content/origin", "dest": "/origin"},
-                    {"src": "/origin/rpm", "dest": "/origin/rpms"},
+                    {
+                        "src": "/content/origin",
+                        "dest": "/origin",
+                        "exclude_paths": [],
+                    },
+                    {
+                        "src": "/origin/rpm",
+                        "dest": "/origin/rpms",
+                        "exclude_paths": ["/iso/"],
+                    },
                 ],
                 "releasever_alias": [
                     {
                         "dest": "/content/dist/rhel8/8.5",
                         "src": "/content/dist/rhel8/8",
+                        "exclude_paths": ["/files/", "/images/", "/iso/"],
                     },
                 ],
                 "rhui_alias": [
                     {
                         "dest": "/content/dist/rhel8",
                         "src": "/content/dist/rhel8/rhui",
+                        "exclude_paths": ["/files/", "/images/", "/iso/"],
                     },
                 ],
             }

exodus_gw/schemas.py

@@ -319,6 +319,11 @@ class Alias(BaseModel):
     dest: str = Field(
         ..., description="Target of the alias, relative to CDN root."
     )
+    exclude_paths: list[str] | None = Field(
+        [],
+        description="Paths for which alias will not be resolved, "
+        "treated as an unanchored regex.",
+    )
 
 
 class YumVariable(Enum):

exodus_gw/worker/cache.py

@@ -36,7 +36,7 @@ def __init__(
         paths: list[str],
         settings: Settings,
         env: str,
-        aliases: list[tuple[str, str]],
+        aliases: list[tuple[str, str, list[str]]],
     ):
         self.paths = [p for p in paths if not exclude_path(p)]
         self.settings = settings

exodus_gw/worker/deploy.py

@@ -1,4 +1,5 @@
 import logging
+import re
 from typing import Any
 
 import dramatiq
@@ -93,7 +94,8 @@ def deploy_config(
     db = Session(bind=db_engine(settings))
     ddb = DynamoDB(env, settings, from_date)
 
-    original_aliases = {src: dest for (src, dest) in ddb.aliases_for_flush}
+    original_aliases = {src: dest for (src, dest, _) in ddb.aliases_for_flush}
+    original_exclusions = {src: exc for (src, _, exc) in ddb.aliases_for_flush}
 
     message = CurrentMessage.get_current_message()
     assert message
@@ -142,12 +144,22 @@ def deploy_config(
     # URLs depending on what changed in the config.
     flush_paths: set[str] = set()
 
-    for src, updated_dest in ddb.aliases_for_flush:
+    for src, updated_dest, _ in ddb.aliases_for_flush:
         if original_aliases.get(src) != updated_dest:
             for published_path in db.query(models.PublishedPath).filter(
                 models.PublishedPath.env == env,
                 models.PublishedPath.web_uri.like(f"{src}/%"),
             ):
+                # If any original exclusion matches the uri, the uri wouldn't
+                # have been treated as an alias, thus cache flushing would be
+                # unnecessary.
+                if any(
+                    [
+                        re.search(exclusion, published_path.web_uri)
+                        for exclusion in original_exclusions.get(src, [])
+                    ]
+                ):
+                    continue
                 LOG.info(
                     "Updated alias %s will flush cache for %s",
                     src,