Populate optional comps.xml unit fields [RHELDST-25995]

amcmahon-rh · amcmahon-rh · commit 580367493e0b · 2024-09-30T14:03:50.000+02:00
When pushing comps.xml files, if a unit already exists in Pulp, fields
that have been removed from the comps.xml did not get overwritten. This
resulted in unexpected older data being published from Pulp.

This change extends the comps.xml parsing by giving an option to fill in
all fields in comp unit classes from pulp-rpm. When the data is uploaded
into pulp this will overwrite old unwanted data.
diff --git a/src/pubtools/pulplib/_impl/comps.py b/src/pubtools/pulplib/_impl/comps.py
@@ -5,6 +5,34 @@
 from io import StringIO
 
 
+# All optional fields in the pulp-rpm comps.xml units
+COMMON_FIELDS = [
+    "description",
+    "translated_description",
+    "display_order",
+    "translated_name",
+    "name",
+]
+UNIT_FIELD_DEFAULTS = {
+    "package_group": (
+        {"default": False, "user_visible": False},
+        [
+            "default_package_names",
+            "optional_package_names",
+            "mandatory_package_names",
+            "default",
+            "user_visible",
+            "langonly",
+            "conditional_package_names",
+        ]
+        + COMMON_FIELDS,
+    ),
+    "package_category": ({}, ["packagegroupids"] + COMMON_FIELDS),
+    "package_environment": ({}, ["group_ids", "options"] + COMMON_FIELDS),
+    "package_langpacks": ({}, ["matches"]),
+}
+
+
 class BooleanStringIO(StringIO):
     """A StringIO which coerces the output value into a boolean."""
 
@@ -356,4 +384,21 @@ def units_for_xml(io):
     modules.
     """
     parser = CompsParser()
-    return parser.parse(io)
+    return fill_unit_field_defaults(parser.parse(io))
+
+
+def fill_unit_field_defaults(units):
+    """
+    Fill in missing comp unit fields with default values.
+
+    Arguments:
+    units (dict)
+        Parsed comps.xml file.
+    """
+
+    for unit in units:
+        defaults, fields = UNIT_FIELD_DEFAULTS[unit["_content_type_id"]]
+        for field in fields:
+            unit[field] = unit.get(field, defaults.get(field))
+
+    return units
diff --git a/tests/comps/test_comps_parse.py b/tests/comps/test_comps_parse.py
@@ -2,7 +2,7 @@
 
 import os
 
-from pubtools.pulplib._impl.comps import units_for_xml
+from pubtools.pulplib._impl.comps import units_for_xml, fill_unit_field_defaults
 
 # Don't use autoformatting in this file because we use u'' string literals
 # at least until py2 support is dropped, and black wants to remove them...
@@ -38,6 +38,10 @@ def test_can_parse_units(data_path):
         },
         "translated_name": {"af": "3D-drukwerk", "bg": u"3D Печатане"},
         "user_visible": True,
+        "optional_package_names": None,
+        "langonly": None,
+        "mandatory_package_names": None,
+        "display_order": None,
     }
 
     assert units[1] == {
@@ -53,6 +57,11 @@ def test_can_parse_units(data_path):
         "default": False,
         "user_visible": True,
         "mandatory_package_names": ["abrt-desktop", "gnome-disk-utility"],
+        "conditional_package_names": None,
+        "default_package_names": None,
+        "display_order": None,
+        "langonly": None,
+        "optional_package_names": None,
     }
 
     assert units[2] == {
@@ -115,6 +124,7 @@ def test_can_parse_units(data_path):
             "es": u"Cinnamon proporciona un entorno de escritorio tradicional, con características avanzadas, fácil de usar, potente y flexible.",
         },
         "display_order": 22,
+        "description": None,
         "group_ids": ["input-methods", "multimedia"],
         "options": [{"default": False, "group": "libreoffice"}],
     }
@@ -128,3 +138,113 @@ def test_can_parse_units(data_path):
             {"install": "tkgate-%s", "name": "tkgate"},
         ],
     }
+
+
+def test_parse_populates_default_fields(data_path):
+    """units_for_xml parses comps.xml data and can populate fields with
+       default values."""
+
+    xml_path = os.path.join(data_path, "sparse-comps.xml")
+
+    with open(xml_path, "rb") as f:
+        units = units_for_xml(f)
+
+    assert len(units) == 8
+
+    assert units[0] == {"_content_type_id": "package_group", "id": "3d-printing",
+            "name": "3D Printing",
+            "translated_name": {"af": "3D-drukwerk", "bg": "3D Печатане"},
+            "user_visible": True, "default_package_names": None,
+            "optional_package_names": None, "mandatory_package_names": None,
+            "default": False, "langonly": None,
+            "conditional_package_names": None, "description": None,
+            "translated_description": None, "display_order": None}
+    assert units[1] == {"_content_type_id": "package_group", "id": "admin-tools",
+            "name": "Administration Tools",
+            "translated_name": {"af": "Administrasienutsgoed",
+                                "am": "የአስተዳደሩ መሣሪያዎች"},
+            "description": "This group is a collection of graphical administration tools for the system, such as for managing user accounts and configuring system hardware.",
+            "translated_description": {
+                "sr": "Ова група је скуп графичких системских административних алатки, нпр. за управљање корисничким налозима и подешавање хардвера у систему.",
+                "sr@Latn": "Ova grupa je skup grafičkih sistemskih administrativnih alatki, npr. za upravljanje korisničkim nalozima i podešavanje hardvera u sistemu."},
+            "mandatory_package_names": ["abrt-desktop", "gnome-disk-utility"],
+            "default_package_names": None, "optional_package_names": None,
+            "default": False, "user_visible": False, "langonly": None,
+            "conditional_package_names": None, "display_order": None}
+    assert units[2] == {"_content_type_id": "package_category",
+            "id": "kde-desktop-environment", "name": "KDE Desktop",
+            "translated_name": {"af": "KDE-werkskerm", "as": "KDE ডেস্কটপ"},
+            "packagegroupids": ["kde-office", "kde-telepathy"],
+            "description": None, "translated_description": None,
+            "display_order": None}
+    assert units[3] == {
+        "_content_type_id": "package_category",
+        "id": "xfce-desktop-environment",
+        "name": "Xfce Desktop",
+        "translated_name": {"uk": u"Графічне середовище Xfce", "zh_CN": u"Xfce 桌面环境"},
+        "description": "A lightweight desktop environment that works well on low end machines.",
+        "translated_description": {
+            "as": u"এটা লঘুভাৰৰ ডেষ্কট'প পৰিবেশ যি নিম্ন বিন্যাসৰ যন্ত্ৰত ভালকৈ কাম কৰি ।",
+            "ast": u"Un entornu d'escritoriu llixeru que furrula bien en máquines pequeñes.",
+        },
+        "display_order": 15,
+        "packagegroupids": ["xfce-apps", "xfce-desktop"],
+    }
+    assert units[4] == {"_content_type_id": "package_environment",
+            "id": "basic-desktop-environment", "name": "Basic Desktop",
+            "description": "X Window System with a choice of window manager.",
+            "translated_description": {
+                "af": "X Window-stelsel met ’n keuse van vensterbestuurder.",
+                "bg": "X Window система с избор на мениджър на прозорци."},
+            "display_order": None,
+            "group_ids": ["networkmanager-submodules", "standard"],
+            "options": None, "translated_name": None}
+
+    assert units[5] == {"_content_type_id": "package_environment",
+            "id": "cinnamon-desktop-environment", "name": "Cinnamon Desktop",
+            "translated_name": {"en_GB": "Cinnamon Desktop",
+                                "fr": "Bureau Cinnamon"},
+            "translated_description": {
+                "ca": "Cinnamon proporciona un escriptori amb un disseny tradicional, funcionalitats avançades, facilitat d'ús, potent i flexible.",
+                "es": "Cinnamon proporciona un entorno de escritorio tradicional, con características avanzadas, fácil de usar, potente y flexible."},
+            "display_order": 22, "group_ids": ["input-methods", "multimedia"],
+            "options": [{"group": "libreoffice", "default": False}],
+            "description": None}
+
+    assert units[6] == {"_content_type_id": "package_langpacks",
+            "matches": [{"install": "stardict-dic-%s", "name": "stardict"},
+                        {"install": "tagainijisho-dic-%s",
+                         "name": "tagainijisho-common"},
+                        {"install": "tesseract-langpack-%s",
+                         "name": "tesseract"},
+                        {"install": "tkgate-%s", "name": "tkgate"}]}
+
+    assert units[7] ==  {"_content_type_id": "package_langpacks", "matches": None}
+
+
+
+def test_fill_unit_field_defaults():
+    """Missing fields are filled in with expected values."""
+    units = [{"_content_type_id": "package_group"},
+             {"_content_type_id": "package_category"},
+             {"_content_type_id": "package_environment"},
+             {"_content_type_id": "package_langpacks"}]
+    expected = [{"_content_type_id": "package_group", "default_package_names": None,
+            "optional_package_names": None, "mandatory_package_names": None,
+            "default": False, "user_visible": False, "langonly": None,
+            "conditional_package_names": None, "description": None,
+            "translated_description": None, "display_order": None,
+            "translated_name": None, "name": None},
+           {"_content_type_id": "package_category", "packagegroupids": None,
+            "description": None, "translated_description": None,
+            "display_order": None,
+            "translated_name": None, "name": None},
+           {"_content_type_id": "package_environment", "group_ids": None,
+            "options": None, "description": None,
+            "translated_description": None,
+            "display_order": None, "translated_name": None, "name": None},
+           {"_content_type_id": "package_langpacks", "matches": None}]
+
+    units = fill_unit_field_defaults(units)
+
+    assert units == expected
diff --git a/tests/data/sparse-comps.xml b/tests/data/sparse-comps.xml
@@ -0,0 +1,102 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE comps
+  PUBLIC '-//Red Hat, Inc.//DTD Comps info//EN'
+  'comps.dtd'>
+
+<!--
+  A comps file for testing.
+
+  This was originally copied from sample-comps.xml, but trimmed down to
+  test field population
+-->
+
+<comps>
+
+
+  <group>
+    <id>3d-printing</id>
+    <name>3D Printing</name>
+    <name xml:lang="af">3D-drukwerk</name>
+    <name xml:lang="bg">3D Печатане</name>
+    <uservisible>true</uservisible>
+  </group>
+  <group>
+    <id>admin-tools</id>
+    <name>Administration Tools</name>
+    <name xml:lang="af">Administrasienutsgoed</name>
+    <name xml:lang="am">የአስተዳደሩ መሣሪያዎች</name>
+    <description>This group is a collection of graphical administration tools for the system, such as for managing user accounts and configuring system hardware.</description>
+    <description xml:lang="sr">Ова група је скуп графичких системских административних алатки, нпр. за управљање корисничким налозима и подешавање хардвера у систему.</description>
+    <description xml:lang="sr@Latn">Ova grupa je skup grafičkih sistemskih administrativnih alatki, npr. za upravljanje korisničkim nalozima i podešavanje hardvera u sistemu.</description>
+    <packagelist>
+      <packagereq type="mandatory">abrt-desktop</packagereq>
+      <packagereq type="mandatory">gnome-disk-utility</packagereq>
+    </packagelist>
+  </group>
+
+
+  <category>
+    <id>kde-desktop-environment</id>
+    <name>KDE Desktop</name>
+    <name xml:lang="af">KDE-werkskerm</name>
+    <name xml:lang="as">KDE ডেস্কটপ</name>
+    <grouplist>
+      <groupid>kde-office</groupid>
+      <groupid>kde-telepathy</groupid>
+    </grouplist>
+  </category>
+  <category>
+    <id>xfce-desktop-environment</id>
+    <name>Xfce Desktop</name>
+    <name xml:lang="uk">Графічне середовище Xfce</name>
+    <name xml:lang="zh_CN">Xfce 桌面环境</name>
+    <description>A lightweight desktop environment that works well on low end machines.</description>
+    <description xml:lang="as">এটা লঘুভাৰৰ ডেষ্কট'প পৰিবেশ যি নিম্ন বিন্যাসৰ যন্ত্ৰত ভালকৈ কাম কৰি ।</description>
+    <description xml:lang="ast">Un entornu d'escritoriu llixeru que furrula bien en máquines pequeñes.</description>
+    <display_order>15</display_order>
+    <grouplist>
+      <groupid>xfce-apps</groupid>
+      <groupid>xfce-desktop</groupid>
+    </grouplist>
+  </category>
+
+
+  <environment>
+    <id>basic-desktop-environment</id>
+    <name>Basic Desktop</name>
+    <description>X Window System with a choice of window manager.</description>
+    <description xml:lang="af">X Window-stelsel met ’n keuse van vensterbestuurder.</description>
+    <description xml:lang="bg">X Window система с избор на мениджър на прозорци.</description>
+    <display_order><!-- tweaked: try empty display order --></display_order>
+    <grouplist>
+      <groupid>networkmanager-submodules</groupid>
+      <groupid>standard</groupid>
+    </grouplist>
+  </environment>
+  <environment>
+    <id>cinnamon-desktop-environment</id>
+    <name>Cinnamon Desktop</name>
+    <name xml:lang="en_GB">Cinnamon Desktop</name>
+    <name xml:lang="fr">Bureau Cinnamon</name>
+    <description xml:lang="ca">Cinnamon proporciona un escriptori amb un disseny tradicional, funcionalitats avançades, facilitat d'ús, potent i flexible.</description>
+    <description xml:lang="es">Cinnamon proporciona un entorno de escritorio tradicional, con características avanzadas, fácil de usar, potente y flexible.</description>
+    <display_order>22</display_order>
+    <grouplist>
+      <groupid>input-methods</groupid>
+      <groupid>multimedia</groupid>
+    </grouplist>
+    <optionlist>
+      <groupid>libreoffice</groupid>
+    </optionlist>
+  </environment>
+
+  <langpacks>
+    <match install="stardict-dic-%s" name="stardict"/>
+    <match install="tagainijisho-dic-%s" name="tagainijisho-common"/>
+    <match install="tesseract-langpack-%s" name="tesseract"/>
+    <match install="tkgate-%s" name="tkgate"/>
+  </langpacks>
+  <langpacks>
+    <!-- An empty langpack doesn't really make much sense hey-->
+  </langpacks>
+</comps>
diff --git a/tests/repository/test_upload_comps.py b/tests/repository/test_upload_comps.py
@@ -136,37 +136,50 @@ def test_upload_comps_xml(client, requests_mocker):
     imported_units.sort(key=lambda u: u["_content_type_id"])
 
     # Did we import exactly the expected units?
+
     assert imported_units == [
         {
             "_content_type_id": "package_category",
+            "description": None,
+            "display_order": 10,
             "id": "kde-desktop-environment",
             "name": "KDE Desktop",
-            "display_order": 10,
             "packagegroupids": ["kde-office", "kde-telepathy"],
             "repo_id": "repo1",
+            "translated_description": None,
+            "translated_name": None,
         },
         {
             "_content_type_id": "package_environment",
-            "id": "basic-desktop-environment",
-            "name": "Basic Desktop",
-            "translated_name": {"af": "Basiese werkskerm"},
             "description": "X Window System with a choice of window manager.",
+            "display_order": None,
             "group_ids": ["networkmanager-submodules", "standard"],
+            "id": "basic-desktop-environment",
+            "name": "Basic Desktop",
             "options": [
-                {"group": "xmonad", "default": True},
-                {"group": "xmonad-mate", "default": False},
+                {"default": True, "group": "xmonad"},
+                {"default": False, "group": "xmonad-mate"},
             ],
             "repo_id": "repo1",
+            "translated_description": None,
+            "translated_name": {"af": "Basiese werkskerm"},
         },
         {
             "_content_type_id": "package_group",
-            "id": "3d-printing",
-            "name": "3D Printing",
-            "translated_name": {"af": "3D-drukwerk"},
-            "description": "3D printing software",
+            "conditional_package_names": None,
+            "default": False,
             "default_package_names": ["admesh"],
+            "description": "3D printing software",
+            "display_order": None,
+            "id": "3d-printing",
+            "langonly": None,
             "mandatory_package_names": ["blender"],
+            "name": "3D Printing",
+            "optional_package_names": None,
             "repo_id": "repo1",
+            "translated_description": None,
+            "translated_name": {"af": "3D-drukwerk"},
+            "user_visible": False,
         },
         {
             "_content_type_id": "package_langpacks",
