Merge remote-tracking branch 'origin/master' into deps/pip-compile

Author: web-flow

CHANGELOG.md

@@ -9,6 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - n/a
 
+## [2.44.0] - 2024-04-08
+
+### Added
+
+- Introduced `VMIPushItem.marketplace_name`
+- Added support for ART/rhcos koji builds
+
 ## [2.43.1] - 2024-02-29
 
 ### Fixed
@@ -489,7 +496,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Initial stable release of project
 
-[Unreleased]: https://github.com/release-engineering/pushsource/compare/v2.43.1...HEAD
+[Unreleased]: https://github.com/release-engineering/pushsource/compare/v2.44.0...HEAD
+[2.44.0]: https://github.com/release-engineering/pushsource/compare/v2.43.1...v2.44.0
 [2.43.1]: https://github.com/release-engineering/pushsource/compare/v2.43.0...v2.43.1
 [2.43.0]: https://github.com/release-engineering/pushsource/compare/v2.42.0...v2.43.0
 [2.42.0]: https://github.com/release-engineering/pushsource/compare/v2.41.0...v2.42.0

setup.py

@@ -21,7 +21,7 @@ def get_requirements():
 
 setup(
     name="pushsource",
-    version="2.43.1",
+    version="2.44.0",
     packages=find_packages("src"),
     package_dir={"": "src"},
     include_package_data=True,

src/pushsource/_impl/backend/koji_source.py

@@ -2,6 +2,7 @@
 import threading
 import logging
 from functools import partial
+import json
 
 from queue import Queue, Empty
 from threading import Thread
@@ -538,8 +539,10 @@ def _push_items_from_vmi_build(self, nvr, meta):
             raise ValueError(message)
 
         build_id = meta["id"]
-
         archives = self._get_archives(build_id)
+        for archive in archives:
+            if archive["filename"] == "meta.json":
+                return self._push_items_from_rhcos_build(nvr, meta)
 
         # Supported types of VMI archives
         ami_types = ["raw", "raw-xz"]
@@ -559,9 +562,7 @@ def _push_items_from_vmi_build(self, nvr, meta):
             if elem.get("btype") == "image"
             and elem.get("type_name") in cloud_types.keys()
         ]
-
         out = []
-
         # Generate the respective PushItem type for each archive
         for archive in vmi_archives:
             path = self._pathinfo.typedir(meta, archive["btype"])
@@ -621,6 +622,100 @@ def _push_items_from_vmi_build(self, nvr, meta):
 
         return out
 
+    def _push_items_from_rhcos_build(self, nvr, meta, archives=None):
+
+        LOG.debug("Looking for core os assembler build on %s, %s", nvr, meta)
+
+        build_id = meta["id"]
+        archives = archives or self._get_archives(build_id)
+
+        for archive in archives:
+            if archive.get("filename") == "meta.json":
+                path = self._pathinfo.typedir(meta, archive["btype"])
+                item_src = os.path.join(path, archive["filename"])
+                with open(item_src) as archive_file:
+                    rhcos_meta_data = json.loads(archive_file.read())
+
+        extra = meta.get("extra") or {}
+        image = extra.get("typeinfo").get("image") or {}
+
+        arch = image.get("arch")
+
+        boot_mode = image.get("boot_mode")
+        if boot_mode is not None:
+            boot_mode = BootMode(boot_mode)
+
+        rhcos_container_config = (
+            rhcos_meta_data.get("coreos-assembler.container-config-git") or {}
+        )
+        branch = rhcos_container_config.get("branch") or ""
+        version = branch.split("-")[-1] or ""
+
+        # Try to get the resping version from release, if available
+        respin = try_int(meta["release"])
+        respin = 0 if not isinstance(respin, int) else respin
+
+        vms_with_custom_meta_data = []
+        # add aws ami
+        for ami in rhcos_meta_data["amis"]:
+            vms_with_custom_meta_data.append(
+                {
+                    "marketplace_name": "aws",
+                    "push_item_class": AmiPushItem,
+                    "custom_meta_data": {
+                        "region": ami.get("name"),
+                        "src": ami.get("hvm"),
+                    },
+                }
+            )
+
+        # add vm for azure
+        vms_with_custom_meta_data.append(
+            {
+                "marketplace_name": "azure",
+                "push_item_class": VHDPushItem,
+                "custom_meta_data": {
+                    "src": rhcos_meta_data["azure"]["url"],
+                },
+            }
+        )
+
+        out = []
+        for vm_item in vms_with_custom_meta_data:
+            klass = vm_item.get("push_item_class")
+            rel_klass = klass._RELEASE_TYPE
+            release = rel_klass(
+                arch=arch,
+                date=(meta.get("completion_time") or "").split(" ")[0],
+                product=meta.get("name").split("-")[0].upper(),
+                version=version,
+                respin=respin,
+            )
+            images = rhcos_meta_data.get("images") or {}
+            vm_target = images.get(vm_item["marketplace_name"]) or {}
+            sha256sum = vm_target.get("sha256")
+
+            out.append(
+                klass(
+                    name=rhcos_meta_data.get("name"),
+                    description=rhcos_meta_data.get("summary"),
+                    **vm_item["custom_meta_data"],
+                    boot_mode=boot_mode,
+                    build=nvr,
+                    build_info=KojiBuildInfo(
+                        id=int(build_id),
+                        name=meta["name"],
+                        version=version,
+                        release=meta["release"],
+                    ),
+                    sha256sum=sha256sum,
+                    release=release,
+                    marketplace_name=vm_item["marketplace_name"]
+                )
+            )
+
+        return out
+
     def _get_operator_item(self, nvr, meta, archives, container_items):
         extra = meta.get("extra") or {}
         typeinfo = extra.get("typeinfo") or {}

src/pushsource/_impl/model/vms.py

@@ -81,6 +81,9 @@ class VMIPushItem(PushItem):
     
     It's used by the property `marketplace_title` to format it as the marketplace title."""
 
+    marketplace_name = attr.ib(type=str, default=None, validator=optional_str)
+    """Name of the marketplace where the Image is expected to be shipped."""
+
     @property
     def marketplace_title(self) -> str:
         """The marketplace title which is used for some certain layered products.

src/pushsource/_impl/schema/staged-schema.yaml

@@ -124,6 +124,7 @@ definitions:
             marketplace_entity_type:
                 $ref: "#/definitions/optional_string"
 
+
             scanning_port:
                 type:
                     - number
@@ -137,6 +138,8 @@ definitions:
                 $ref: "#/definitions/optional_string"
             marketplace_title_template:
                 $ref: "#/definitions/optional_string"
+            marketplace_name:
+                $ref: "#/definitions/optional_string"
             security_groups:
                 type:
                     - array

tests/baseline/cases/staged-simple-ami-bc.yml

@@ -24,6 +24,7 @@ items:
     ena_support: true
     image_id: null
     marketplace_entity_type: AMIProduct
+    marketplace_name: null
     marketplace_title_template: null
     md5sum: null
     name: fake-image.raw

tests/baseline/cases/staged-simple-ami-bootmode.yml

@@ -20,6 +20,7 @@ items:
     ena_support: true
     image_id: null
     marketplace_entity_type: null
+    marketplace_name: null
     marketplace_title_template: null
     md5sum: null
     name: fake-image.raw

tests/baseline/cases/staged-simple-ami-uefi.yml

@@ -20,6 +20,7 @@ items:
     ena_support: true
     image_id: null
     marketplace_entity_type: null
+    marketplace_name: null
     marketplace_title_template: null
     md5sum: null
     name: fake-image.raw

tests/baseline/cases/staged-simple-ami.yml

@@ -20,6 +20,7 @@ items:
     ena_support: true
     image_id: null
     marketplace_entity_type: null
+    marketplace_name: null
     marketplace_title_template: null
     md5sum: null
     name: fake-image.raw

tests/koji/test_koji_vmi.py

@@ -238,3 +238,151 @@ def test_koji_vmi_compound_product_name(fake_koji, koji_dir):
         signing_key=None,
         release=rel_obj,
     )
+
+
+@mark.parametrize("boot_mode", [None, "legacy", "uefi", "hybrid"])
+def test_coreos_assembler_image(boot_mode, fake_koji, koji_dir):
+    boot_mode_enum = BootMode(boot_mode) if boot_mode else None
+
+    archives = []
+    images = {
+        "meta.json": "json",
+        "commitmeta.json": "json",
+        "coreos-assembler-git.tar.gz": "tar",
+    }
+    for file_name, type_name in images.items():
+        archives.append(
+            {
+                "btype": "image",
+                "btype_id": 4,
+                "build_id": 123456,
+                "buildroot_id": 1234567,
+                "checksum": "dde60621880aa996c42e356a687744ef",
+                "checksum_type": 0,
+                "compression_type": None,
+                "filename": file_name,
+                "id": 234792,
+                "metadata_only": False,
+                "size": 16077,
+                "type_description": "JSON data",
+                "type_extensions": "json",
+                "type_id": 49,
+                "type_name": type_name,
+            }
+        )
+
+    name = "rhcos"
+    version = "4.11"
+    release = "1"
+    nvr = f"{name}-{version}-{release}"
+
+    source = Source.get(
+        f"koji:https://koji.example.com/?vmi_build={nvr}", basedir=koji_dir
+    )
+    assert not fake_koji.last_url
+
+    fake_koji.build_data[1234] = {
+        "id": 1234,
+        "name": name,
+        "version": version,
+        "release": release,
+        "nvr": nvr,
+        "completion_time": "2022-12-20 12:30:25",
+        "extra": {"typeinfo": {"image": {"arch": "x86_64", "boot_mode": boot_mode}}},
+    }
+
+    fake_koji.build_data[nvr] = fake_koji.build_data[1234]
+
+    fake_koji.insert_archives(archives=archives, build_nvr=nvr)
+
+    meta_data_directory = os.path.join(
+        koji_dir, "packages", name, version, release, "images"
+    )
+    os.makedirs(meta_data_directory)
+    meta_data_path = os.path.join(meta_data_directory, "meta.json")
+
+    meta_data = """{    
+    "name": "rhcos",
+    "summary": "OpenShift 4",
+    "coreos-assembler.container-config-git":{
+            "branch": "release-4.11"
+    },
+    "images": {
+        "azure": {
+            "path": "rhcos-4.11-0-azure.x86_64.vhd.gz",
+            "sha256": "2cd817331af29093e2eaa025139ebddd6008c193970b06b35afbbdbebae0ce3e",
+            "dest": ["https://example.windows.net/imagebucket/rhcos-4.11-0-azure.x86_64.vhd"]},
+  "aws": {
+        "path": "rhcos-4.11-0-aws.x86_64.vmdk.gz",
+        "sha256": "4ef7806152bd89ce44326ff746c4f1883ad543885e980bce59821df2d946ea4c"
+        }},
+ "azure": {
+        "image": "rhcos-4.11-0-azure.x86_64.vhd",
+        "url": "https://example.windows.net/imagebucket/rhcos-4.11-0-azure.x86_64.vhd"},
+ "amis": [
+    {"name": "us-gov-west-1",
+   "hvm": "ami-01"
+   },
+    {"name": "us-east-1",
+   "hvm": "ami-02"
+   }]}"""
+
+    with open(meta_data_path, "w") as rhcos_metadata:
+        rhcos_metadata.write(meta_data)
+    items = list(source)
+
+    vms_with_custom_meta_data = [
+        {
+            "marketplace_name": "aws",
+            "push_item_class": AmiPushItem,
+            "custom_meta_data": {"region": "us-gov-west-1", "src": "ami-01"},
+            "sha256": "4ef7806152bd89ce44326ff746c4f1883ad543885e980bce59821df2d946ea4c",
+        },
+        {
+            "marketplace_name": "aws",
+            "push_item_class": AmiPushItem,
+            "custom_meta_data": {"region": "us-east-1", "src": "ami-02"},
+            "sha256": "4ef7806152bd89ce44326ff746c4f1883ad543885e980bce59821df2d946ea4c",
+        },
+        {
+            "marketplace_name": "azure",
+            "push_item_class": VHDPushItem,
+            "custom_meta_data": {
+                "src": "https://example.windows.net/imagebucket/rhcos-4.11-0-azure.x86_64.vhd"
+            },
+            "sha256": "2cd817331af29093e2eaa025139ebddd6008c193970b06b35afbbdbebae0ce3e",
+        },
+    ]
+
+    out = []
+    for vm_item in vms_with_custom_meta_data:
+        klass = vm_item.get("push_item_class")
+        rel_klass = klass._RELEASE_TYPE
+        release = rel_klass(
+            arch="x86_64",
+            date="2022-12-20",
+            product="RHCOS",
+            version="4.11",
+            respin=1,
+        )
+
+        out.append(
+            klass(
+                name="rhcos",
+                description="OpenShift 4",
+                **vm_item["custom_meta_data"],
+                boot_mode=boot_mode_enum,
+                build="rhcos-4.11-1",
+                build_info=KojiBuildInfo(
+                    id=1234,
+                    name="rhcos",
+                    version="4.11",
+                    release="1",
+                ),
+                sha256sum=vm_item.get("sha256"),
+                release=release,
+                marketplace_name=vm_item["marketplace_name"],
+            )
+        )
+    items = sorted(items, key=lambda metadata: metadata.name)
+    assert items == out