From 77e9468360761a771f2fe6c19ebc4535a3137598 Mon Sep 17 00:00:00 2001 From: Ashwini Kumar Date: Wed, 28 Feb 2024 19:08:08 +0530 Subject: [PATCH] adding support for art build --- src/pushsource/_impl/backend/koji_source.py | 104 +++++++++++++++-- tests/koji/test_koji_vmi.py | 122 ++++++++++++++++++++ 2 files changed, 219 insertions(+), 7 deletions(-) diff --git a/src/pushsource/_impl/backend/koji_source.py b/src/pushsource/_impl/backend/koji_source.py index 7414a881..eed06876 100644 --- a/src/pushsource/_impl/backend/koji_source.py +++ b/src/pushsource/_impl/backend/koji_source.py @@ -2,6 +2,7 @@ import threading import logging from functools import partial +import json from queue import Queue, Empty from threading import Thread @@ -519,7 +520,7 @@ def _push_items_from_vmi_build(self, nvr, meta): message = "Virtual machine image build not found in koji: %s" % nvr LOG.debug(message) raise ValueError(message) - + # VMI has the {'typeinfo': {'image':{}}} on extra extra = meta.get("extra") or {} typeinfo = extra.get("typeinfo") or {} @@ -536,10 +537,12 @@ def _push_items_from_vmi_build(self, nvr, meta): message = "Build %s not recognized as a virtual machine image build" % nvr LOG.debug(message) raise ValueError(message) - + build_id = meta["id"] - archives = self._get_archives(build_id) + for archive in archives: + if archive["filename"] == "meta.json": + return self._push_items_from_rhcos_build(nvr, meta) # Supported types of VMI archives ami_types = ["raw", "raw-xz"] @@ -548,10 +551,10 @@ def _push_items_from_vmi_build(self, nvr, meta): for k in ami_types: cloud_types.setdefault(k, AmiPushItem) - + for k in azure_types: cloud_types.setdefault(k, VHDPushItem) - + # Collect all known VM image archives vmi_archives = [ elem @@ -559,9 +562,7 @@ def _push_items_from_vmi_build(self, nvr, meta): if elem.get("btype") == "image" and elem.get("type_name") in cloud_types.keys() ] - out = [] - # Generate the respective PushItem type for each archive for archive in vmi_archives: path = self._pathinfo.typedir(meta, archive["btype"]) @@ -620,6 +621,95 @@ def _push_items_from_vmi_build(self, nvr, meta): ) return out + + def _push_items_from_rhcos_build(self, nvr, meta): + + LOG.debug("Looking for core os assembler build on %s, %s", nvr, meta) + + build_id = meta["id"] + archives = self._get_archives(build_id) + + for archive in archives: + if archive.get("filename") == "meta.json": + path = self._pathinfo.typedir(meta, archive["btype"]) + item_src = os.path.join(path, archive["filename"]) + with open(item_src) as archive_file: + rhcos_meta_data = json.loads(archive_file.read()) + + extra = meta.get("extra") or {} + image = extra.get("typeinfo").get("image") or {} + + arch = image.get("arch") + boot_mode = image.get("boot_mode") + + if boot_mode is not None: + boot_mode = BootMode(boot_mode) + + # Try to get the resping version from release, if available + respin = try_int(meta["release"]) + respin = 0 if not isinstance(respin, int) else respin + + vms_with_custom_meta_data = [] + # add aws ami + for ami in rhcos_meta_data["amis"]: + vms_with_custom_meta_data.append( + { + "target": "aws", + "push_item_class": AmiPushItem, + "custom_meta_data": { + "region": ami.get("name"), + "src": ami.get("hvm"), + + } + + } + ) + + # add vm for azure + vms_with_custom_meta_data.append( + { + "target": "azure", + "push_item_class": VHDPushItem, + "custom_meta_data": { + "src": rhcos_meta_data["azure"]["url"], + + } + + } + ) + + out = [] + for vm_item in vms_with_custom_meta_data: + klass = vm_item.get("push_item_class") + rel_klass = klass._RELEASE_TYPE + release = rel_klass( + arch=arch, + date=(meta.get("completion_time") or "").split(" ")[0], + product=meta.get("name").split("-")[0].upper(), + version=meta["version"], + respin=respin, + ) + + out.append( + klass( + name=rhcos_meta_data.get("name"), + description=rhcos_meta_data.get("summary"), + **vm_item["custom_meta_data"], + boot_mode=boot_mode, + build=nvr, + build_info=KojiBuildInfo( + id=int(build_id), + name=meta["name"], + version=meta["version"], + release=meta["release"], + ), + sha256sum=rhcos_meta_data.get("images").get( + vm_item["target"]).get("sha256"), + release=release, + ) + ) + + return out def _get_operator_item(self, nvr, meta, archives, container_items): extra = meta.get("extra") or {} diff --git a/tests/koji/test_koji_vmi.py b/tests/koji/test_koji_vmi.py index 166f1df6..94cf4734 100644 --- a/tests/koji/test_koji_vmi.py +++ b/tests/koji/test_koji_vmi.py @@ -1,6 +1,7 @@ import os from pytest import raises, mark +from unittest.mock import patch, mock_open from pushsource import ( Source, @@ -238,3 +239,124 @@ def test_koji_vmi_compound_product_name(fake_koji, koji_dir): signing_key=None, release=rel_obj, ) + +def test_coreos_assembler_image(fake_koji, koji_dir): + archives = [] + images = { + "meta.json": "json", + "commitmeta.json": "json", + "coreos-assembler-git.tar.gz": "tar" + + } + for file_name, type_name in images.items(): + archives.append( + {'btype': 'image', + 'btype_id': 4, + 'build_id': 123456, + 'buildroot_id': 1234567, + 'checksum': 'dde60621880aa996c42e356a687744ef', + 'checksum_type': 0, + 'compression_type': None, + 'extra': {'image': {'arch': 'x86_64'}}, + 'filename': file_name, + 'id': 234792, + 'metadata_only': False, + 'size': 16077, + 'type_description': 'JSON data', + 'type_extensions': 'json', + 'type_id': 49, + 'type_name': type_name + } + ) + + name = "rhcos" + version = "414.92" + release = "1" + nvr = f"{name}-{version}-{release}" + + source = Source.get( + f"koji:https://koji.example.com/?vmi_build={nvr}", basedir=koji_dir + ) + assert not fake_koji.last_url + + fake_koji.build_data[1234] = { + "id": 1234, + "name": name, + "version": version, + "release": release, + "nvr": nvr, + "completion_time": "2022-12-20 12:30:25", + "extra": {"typeinfo": {"image": {"arch": "x86_64"}}}} + + fake_koji.build_data[nvr] = fake_koji.build_data[1234] + + fake_koji.insert_archives(archives=archives, build_nvr=nvr) + + path = "/path/to/rhcos/metadata" + meta_data = """{ + "name": "rhcos", + "summary": "OpenShift 4", + "images": { + "azure": { + "path": "rhcos-414.92-0-azure.x86_64.vhd.gz", + "sha256": "2cd817331af29093e2eaa025139ebddd6008c193970b06b35afbbdbebae0ce3e", + "dest": ["https://example.windows.net/imagebucket/rhcos-414.92-0-azure.x86_64.vhd"]}, + "aws": { + "path": "rhcos-414.92-0-aws.x86_64.vmdk.gz", + "sha256": "4ef7806152bd89ce44326ff746c4f1883ad543885e980bce59821df2d946ea4c" + }}, + "azure": { + "image": "rhcos-414.92-0-azure.x86_64.vhd", + "url": "https://example.windows.net/imagebucket/rhcos-414.92-0-azure.x86_64.vhd"}, + "amis": [ + {"name": "us-gov-west-1", + "hvm": "ami-01" + }, + {"name": "us-east-1", + "hvm": "ami-02" + }]}""" + + patch.object(os.path, "join", return_value=path) + with patch("builtins.open", mock_open(read_data=meta_data)) as mocked_open: + items = list(source) + + vms_with_custom_meta_data = [ + {'target': 'aws', 'push_item_class': AmiPushItem, 'custom_meta_data': {'region': 'us-gov-west-1', + 'src': 'ami-01'}, "sha256": "4ef7806152bd89ce44326ff746c4f1883ad543885e980bce59821df2d946ea4c"}, + {'target': 'aws', 'push_item_class': AmiPushItem, 'custom_meta_data': {'region': 'us-east-1', + 'src': 'ami-02'}, "sha256": "4ef7806152bd89ce44326ff746c4f1883ad543885e980bce59821df2d946ea4c"}, + {'target': 'azure', 'push_item_class': VHDPushItem, 'custom_meta_data': { + 'src': 'https://example.windows.net/imagebucket/rhcos-414.92-0-azure.x86_64.vhd'}, "sha256": "2cd817331af29093e2eaa025139ebddd6008c193970b06b35afbbdbebae0ce3e"} + ] + + out = [] + for vm_item in vms_with_custom_meta_data: + klass = vm_item.get("push_item_class") + rel_klass = klass._RELEASE_TYPE + release = rel_klass( + arch="x86_64", + date="2022-12-20", + product="RHCOS", + version="414.92", + respin=1, + ) + + out.append( + klass( + name="rhcos", + description="OpenShift 4", + **vm_item["custom_meta_data"], + boot_mode=None, + build="rhcos-414.92-1", + build_info=KojiBuildInfo( + id=1234, + name="rhcos", + version="414.92", + release="1", + ), + sha256sum=vm_item.get("sha256"), + release=release, + ) + ) + items = sorted(items, key=lambda metadata: metadata.name) + assert items == out