Merge pull request #9645 from meaksh/master-extend-filters-on-reposync

Allow "reposync" to filter packages using the entire NEVRA (bsc#1234226)

python/spacewalk/rhn-conf/rhn_server_satellite.conf

@@ -43,6 +43,7 @@ default_mail_from =
 reposync_download_threads = 5
 reposync_timeout = 300
 reposync_minrate = 1000
+reposync_nevra_filter = 0
 
 # URLGrabber log level. This parameter is used by spacewalk-repo-sync to provide
 # additional logs, overriding URLGRABBER_DEBUG. It takes the form "level,filename". 

python/spacewalk/satellite_tools/repo_plugins/deb_src.py

@@ -68,6 +68,8 @@ def __setitem__(self, key, value):
         return setattr(self, key, value)
 
     def evr(self):
+        # The format is: [epoch:]upstream_version[-debian_revision].
+        # https://www.debian.org/doc/debian-policy/ch-controlfields.html#version
         evr = ""
         if self.epoch:
             # pylint: disable-next=consider-using-f-string
@@ -80,6 +82,9 @@ def evr(self):
             evr = evr + "-{}".format(self.release)
         return evr
 
+    def nevra(self):
+        return f"{self.name}_{self.evr()}_{self.arch}"
+
     def is_populated(self):
         return all(
             [
@@ -412,6 +417,14 @@ def __init__(
             except ValueError:
                 self.timeout = 300
 
+            try:
+                # extended reposync nevra filter enable
+                # this will filter packages based on full nevra
+                # instead of package name only.
+                self.nevra_filter = bool(CFG.REPOSYNC_NEVRA_FILTER)
+            except (AttributeError, ValueError):
+                self.nevra_filter = False
+
             # SUSE vendor repositories belongs to org = NULL
             # The repository cache root will be "/var/cache/rhn/reposync/REPOSITORY_LABEL/"
             root = os.path.join(CACHE_DIR, str(org or "NULL"), self.reponame)
@@ -479,7 +492,9 @@ def list_packages(self, filters, latest):
                 filters.append(("-", [p]))
 
         if filters:
-            pkglist = self._filter_packages(pkglist, filters)
+            pkglist = self._filter_packages(
+                pkglist, filters, nevra_filter=self.nevra_filter
+            )
             self.num_excluded = self.num_packages - len(pkglist)
 
         to_return = []
@@ -510,7 +525,7 @@ def _sort_packages(pkg1, pkg2):
             return -1
 
     @staticmethod
-    def _filter_packages(packages, filters):
+    def _filter_packages(packages, filters, nevra_filter=False):
         """implement include / exclude logic
         filters are: [ ('+', includelist1), ('-', excludelist1),
                        ('+', includelist2), ... ]
@@ -536,7 +551,11 @@ def _filter_packages(packages, filters):
             if sense == "+":
                 # include
                 for excluded_pkg in excluded:
-                    if reobj.match(excluded_pkg["name"]):
+                    if nevra_filter:
+                        pkg_name = excluded_pkg.nevra()
+                    else:
+                        pkg_name = excluded_pkg["name"]
+                    if reobj.match(pkg_name):
                         allmatched_include.insert(0, excluded_pkg)
                         selected.insert(0, excluded_pkg)
                 for pkg in allmatched_include:
@@ -545,10 +564,13 @@ def _filter_packages(packages, filters):
             elif sense == "-":
                 # exclude
                 for selected_pkg in selected:
-                    if reobj.match(selected_pkg["name"]):
+                    if nevra_filter:
+                        pkg_name = selected_pkg.nevra()
+                    else:
+                        pkg_name = selected_pkg["name"]
+                    if reobj.match(pkg_name):
                         allmatched_exclude.insert(0, selected_pkg)
                         excluded.insert(0, selected_pkg)
-
                 for pkg in allmatched_exclude:
                     if pkg in selected:
                         selected.remove(pkg)

python/spacewalk/satellite_tools/repo_plugins/yum_src.py

@@ -671,6 +671,13 @@ def __init__(
                 self.timeout = int(CFG.REPOSYNC_TIMEOUT)
             except ValueError:
                 self.timeout = 300
+            try:
+                # extended reposync nevra filter enable
+                # this will filter packages based on full nevra
+                # instead of package name only.
+                self.nevra_filter = bool(CFG.REPOSYNC_NEVRA_FILTER)
+            except (AttributeError, ValueError):
+                self.nevra_filter = False
 
     def _load_proxy_settings(self, url):
         # read the proxy configuration in /etc/rhn/rhn.conf
@@ -1216,11 +1223,13 @@ def _apply_filters(self, pkglist, filters):
                 filters.append(("-", [p]))
 
         if filters:
-            pkglist = self._filter_packages(pkglist, filters)
+            pkglist = self._filter_packages(
+                pkglist, filters, nevra_filter=self.nevra_filter
+            )
             pkglist = self._get_solvable_dependencies(pkglist)
 
             # Do not pull in dependencies if there're explicitly excluded
-            pkglist = self._filter_packages(pkglist, filters, True)
+            pkglist = self._filter_packages(pkglist, filters, True, self.nevra_filter)
             self.num_excluded = self.num_packages - len(pkglist)
 
         return pkglist
@@ -1233,7 +1242,7 @@ def _fix_encoding(text):
             return str(text)
 
     @staticmethod
-    def _filter_packages(packages, filters, exclude_only=False):
+    def _filter_packages(packages, filters, exclude_only=False, nevra_filter=False):
         """implement include / exclude logic
         filters are: [ ('+', includelist1), ('-', excludelist1),
                        ('+', includelist2), ... ]
@@ -1261,7 +1270,11 @@ def _filter_packages(packages, filters, exclude_only=False):
                     continue
                 # include
                 for excluded_pkg in excluded:
-                    if reobj.match(excluded_pkg.name):
+                    if nevra_filter:
+                        pkg_name = str(excluded_pkg)
+                    else:
+                        pkg_name = excluded_pkg.name
+                    if reobj.match(pkg_name):
                         allmatched_include.insert(0, excluded_pkg)
                         selected.insert(0, excluded_pkg)
                 for pkg in allmatched_include:
@@ -1270,10 +1283,13 @@ def _filter_packages(packages, filters, exclude_only=False):
             elif sense == "-":
                 # exclude
                 for selected_pkg in selected:
-                    if reobj.match(selected_pkg.name):
+                    if nevra_filter:
+                        pkg_name = str(selected_pkg)
+                    else:
+                        pkg_name = selected_pkg.name
+                    if reobj.match(pkg_name):
                         allmatched_exclude.insert(0, selected_pkg)
                         excluded.insert(0, selected_pkg)
-
                 for pkg in allmatched_exclude:
                     if pkg in selected:
                         selected.remove(pkg)

python/spacewalk/spacewalk-backend.changes.meaksh.master-extend-filters-on-reposync

@@ -0,0 +1,2 @@
+- Allow spacewalk-repo-sync filtering using NEVRA
+  instead of package name only (bsc#1234226)

python/test/unit/spacewalk/satellite_tools/test_yum_src.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# pylint: disable=missing-module-docstring
+# pylint: disable=missing-module-docstring,missing-class-docstring
 # -*- coding: utf-8 -*-
 #
 # Copyright (c) 2011 SUSE LLC
@@ -111,8 +111,107 @@ def test_list_packages_empty(self):
 
         self.assertEqual(cs.list_packages(filters=None, latest=False), [])
 
+    def test_list_packages_filters(self):
+        cs = self._make_dummy_cs()
+
+        class ChecksumMock:
+            def typestr(self):
+                pass
+
+            def hex(self):
+                pass
+
+        class SolvableMock:
+            def __init__(self, name, evr, nevra, arch):
+                self.name = name
+                self.evr = evr
+                self.nevra = nevra
+                self.arch = arch
+
+            # pylint: disable-next=unused-argument
+            def lookup_checksum(self, x):
+                return ChecksumMock()
+
+            def lookup_num(self, x):
+                pass
+
+            def lookup_location(self):
+                return ["foobar"]
+
+            def __str__(self):
+                return self.nevra
+
+        # pylint: disable-next=protected-access
+        cs._get_solvable_packages = MagicMock(
+            side_effect=lambda: [
+                SolvableMock(
+                    "pkg1",
+                    "1.2.3-xx.0.foobar",
+                    "pkg1-1.2.3-xx.0.foobar.x86_64",
+                    "x86_64",
+                ),
+                SolvableMock(
+                    "pkg1",
+                    "1.2.4-xx.0.foobar",
+                    "pkg1-1.2.4-xx.0.foobar.x86_64",
+                    "x86_64",
+                ),
+                SolvableMock("pkg2", "3.2.1-1", "pkg2-3.2.1-1.x86_64", "x86_64"),
+                SolvableMock("pkg2", "3.2.2-1", "pkg2-3.2.2-1.x86_64", "x86_64"),
+            ]
+        )
+        # pylint: disable-next=protected-access
+        cs._get_solvable_dependencies = MagicMock(side_effect=lambda x: x)
+
+        self.assertEqual(len(cs.list_packages(filters=None, latest=False)), 4)
+
+        cs.nevra_filter = False
+        self.assertEqual(
+            len(cs.list_packages(filters=[("+", ["pkg*"])], latest=False)), 4
+        )
+        self.assertEqual(
+            len(cs.list_packages(filters=[("+", ["pkg1*"])], latest=False)), 2
+        )
+        self.assertEqual(
+            len(cs.list_packages(filters=[("+", ["pkg1-1.2*"])], latest=False)), 0
+        )
+        self.assertEqual(
+            len(cs.list_packages(filters=[("+", ["pkg1"])], latest=False)), 2
+        )
+        self.assertEqual(
+            len(
+                cs.list_packages(
+                    filters=[("+", ["pkg1-1.2.3-xx.0.foobar.x86_64"])], latest=False
+                )
+            ),
+            0,
+        )
+
+        cs.nevra_filter = True
+        self.assertEqual(
+            len(cs.list_packages(filters=[("+", ["pkg*"])], latest=False)), 4
+        )
+        self.assertEqual(
+            len(cs.list_packages(filters=[("+", ["pkg1*"])], latest=False)), 2
+        )
+        self.assertEqual(
+            len(cs.list_packages(filters=[("+", ["pkg1-1.2*"])], latest=False)), 2
+        )
+        self.assertEqual(
+            len(cs.list_packages(filters=[("+", ["pkg1"])], latest=False)), 0
+        )
+        self.assertEqual(
+            len(
+                cs.list_packages(
+                    filters=[("+", ["pkg1-1.2.3-xx.0.foobar.x86_64"])], latest=False
+                )
+            ),
+            1,
+        )
+
     @unittest.skip
     def test_list_packages_with_pack(self):
+
         cs = self._make_dummy_cs()
 
         package_attrs = [

spacewalk/config/spacewalk-config.changes.meaksh.master-extend-filters-on-reposync

@@ -0,0 +1 @@
+- Add new config: reposync_nevra_filter (bsc#1234226)

spacewalk/config/var/lib/rhn/rhn-satellite-prep/etc/rhn/rhn.conf

@@ -80,3 +80,5 @@ pam_auth_service = susemanager
 # Maximum Java Heap Size (in MB)
 # taskomatic.java.maxmemory=4096
 
+# Extended reposync filters to use the entire NEVRA
+server.satellite.reposync_nevra_filter = 0