From bf618543a81304c32f8d8f3f52ccf1e16237bd31 Mon Sep 17 00:00:00 2001 From: Jordan Reidy Date: Wed, 23 Oct 2024 11:02:32 -0400 Subject: [PATCH 1/6] Adds staged file structure for cloud images. --- docs/sources/staged.rst | 14 ++ .../_impl/backend/staged/staged_cloud.py | 124 +++++++++++++ .../_impl/backend/staged/staged_source.py | 8 +- tests/baseline/cases/staged-simple-cloud.yml | 170 ++++++++++++++++++ .../build1/CLOUD_IMAGES/resources.yaml | 14 ++ .../build1/CLOUD_IMAGES/sample1.raw.xz | 0 .../build1/CLOUD_IMAGES/sample2.raw.xz | 0 .../build2/CLOUD_IMAGES/resources.yaml | 14 ++ .../build2/CLOUD_IMAGES/sample1.raw.xz | 0 .../build2/CLOUD_IMAGES/sample2.raw.xz | 0 10 files changed, 343 insertions(+), 1 deletion(-) create mode 100644 src/pushsource/_impl/backend/staged/staged_cloud.py create mode 100644 tests/baseline/cases/staged-simple-cloud.yml create mode 100644 tests/staged/data/simple_cloud/build1/CLOUD_IMAGES/resources.yaml create mode 100644 tests/staged/data/simple_cloud/build1/CLOUD_IMAGES/sample1.raw.xz create mode 100644 tests/staged/data/simple_cloud/build1/CLOUD_IMAGES/sample2.raw.xz create mode 100644 tests/staged/data/simple_cloud/build2/CLOUD_IMAGES/resources.yaml create mode 100644 tests/staged/data/simple_cloud/build2/CLOUD_IMAGES/sample1.raw.xz create mode 100644 tests/staged/data/simple_cloud/build2/CLOUD_IMAGES/sample2.raw.xz diff --git a/docs/sources/staged.rst b/docs/sources/staged.rst index 168e44bf..c6bb62a2 100644 --- a/docs/sources/staged.rst +++ b/docs/sources/staged.rst @@ -55,6 +55,7 @@ Here is a brief overview of the structure of a staging directory: root/destination/MODULEMD/* root/destination/RPMS/*.rpm root/destination/AWS_IMAGES/* + root/destination/CLOUD_IMAGES/* root/destination/RAW/* The staging directory consists of: @@ -196,6 +197,19 @@ Files in this directory must have metadata included in ``staged.yaml``. Will yield instances of :class:`~pushsource.AmiPushItem`. +root/destination/CLOUD_IMAGES/\* +.............................. + +The ``CLOUD_IMAGES`` should contain the VMI(s) plus a ``resources.yaml``. + +The ``resources.yaml`` contains all the information needed for the +images in that folder. + +Files in this directory should not include the ``staged.yaml``. + +Will yield instances of either :class:`~pushsource.AmiPushItem` or +:class:`~pushsource.VHDPushItem`. + root/destination/RAW/\* ....................... diff --git a/src/pushsource/_impl/backend/staged/staged_cloud.py b/src/pushsource/_impl/backend/staged/staged_cloud.py new file mode 100644 index 00000000..4a72ae7e --- /dev/null +++ b/src/pushsource/_impl/backend/staged/staged_cloud.py @@ -0,0 +1,124 @@ +import logging +import yaml +import os + +from .staged_base import StagedBaseMixin, handles_type +from ...model import VHDPushItem, AmiPushItem, AmiRelease, BootMode, KojiBuildInfo + +LOG = logging.getLogger("pushsource") + + +class StagedCloudMixin(StagedBaseMixin): + def __build_ami_push_item(self, resources, src, origin, name): + build_resources = resources.get("build") + build_info = KojiBuildInfo( + name=build_resources.get("name"), + version=build_resources.get("version"), + release=build_resources.get("respin") + ) + image_kwargs = { + "name": name, + "src": src, + "build_info": build_info, + "origin": origin, + } + + image_kwargs.update( + { + "boot_mode": ( + BootMode(resources.get("boot_mode")) + if resources.get("boot_mode") + else None + ) + } + ) + + release_required = ["product", "date", "arch", "respin"] + if all(x in build_resources for x in release_required): + release_attrs = [ + "product", + "date", + "arch", + "respin", + "version", + "base_product", + "base_version", + "variant", + "type", + ] + release_kwargs = {} + for key in release_attrs: + release_kwargs[key] = build_resources.get(key) + + image_kwargs["release"] = AmiRelease(**release_kwargs) + + image_attrs = [ + "dest", + "type", + "region", + "virtualization", + "volume", + "root_device", + "description", + "sriov_net_support", + "ena_support", + "uefi_support", + "public_image", + "release_notes", + "usage_instructions", + "recommended_instance_type", + "marketplace_entity_type", + "image_id", + "scanning_port", + "user_name", + "version_title", + "security_groups", + "access_endpoint_url", + ] + + for key in image_attrs: + if key in resources: + image_kwargs[key] = resources.get(key) + + return AmiPushItem(**image_kwargs) + + def __build_azure_push_item(self, resources, src, origin, name): + build_resources = resources.get("build") + build_info = KojiBuildInfo( + name=build_resources.get("name"), + version=build_resources.get("version"), + release=build_resources.get("respin") + ) + image_kwargs = { + "name": name, + "src": src, + "build_info": build_info, + "origin": origin, + } + return VHDPushItem(**image_kwargs) + + @handles_type("CLOUD_IMAGES") + def __cloud_push_item(self, leafdir, metadata, entry): + with open(entry.path, "rt") as fh: + raw = yaml.safe_load(fh) + if not raw: + return + + out = [] + image_type = raw.get("type") + images_info = raw.get("images") + + for image in images_info: + image_relative_path = os.path.join(leafdir.path, image.get("path")) + if image_type == "AMI": + out.append(self.__build_ami_push_item(raw, + image_relative_path, + leafdir.topdir, + image.get("path"))) + elif image_type == "VHD": + out.append(self.__build_azure_push_item(raw, + image_relative_path, + leafdir.topdir, + image.get("path"))) + + return out \ No newline at end of file diff --git a/src/pushsource/_impl/backend/staged/staged_source.py b/src/pushsource/_impl/backend/staged/staged_source.py index 300bfdfe..baed3493 100644 --- a/src/pushsource/_impl/backend/staged/staged_source.py +++ b/src/pushsource/_impl/backend/staged/staged_source.py @@ -16,6 +16,7 @@ from .staged_utils import StagingMetadata, StagingLeafDir from .staged_ami import StagedAmiMixin +from .staged_cloud import StagedCloudMixin from .staged_files import StagedFilesMixin from .staged_errata import StagedErrataMixin from .staged_compsxml import StagedCompsXmlMixin @@ -33,6 +34,7 @@ class StagedSource( Source, StagedAmiMixin, + StagedCloudMixin, StagedFilesMixin, StagedErrataMixin, StagedCompsXmlMixin, @@ -179,7 +181,11 @@ def _push_items_for_topdir(self, topdir): ) for f in completed_fs: for pushitem in f.result(): - yield pushitem + if isinstance(pushitem, list): + for p in pushitem: + yield p + else: + yield pushitem Source.register_backend("staged", StagedSource) diff --git a/tests/baseline/cases/staged-simple-cloud.yml b/tests/baseline/cases/staged-simple-cloud.yml new file mode 100644 index 00000000..1949d736 --- /dev/null +++ b/tests/baseline/cases/staged-simple-cloud.yml @@ -0,0 +1,170 @@ +# A pushsource library testcase. +# +# This file was generated from a template. +# To regenerate, run test_baseline.py with PUSHSOURCE_UPDATE_BASELINE=1. + +# URL of Source to test. +url: "staged:{{ src_dir }}/tests/staged/data/simple_cloud" + +# Push items generated from above. +items: +- AmiPushItem: + access_endpoint_url: null + billing_codes: null + boot_mode: hybrid + build: null + build_info: + id: null + name: rhel-ec2 + release: '1' + version: '9.4' + description: build1 sample + dest: [] + ena_support: null + image_id: null + marketplace_entity_type: null + marketplace_name: null + marketplace_title_template: null + md5sum: null + name: something1.raw.xz + origin: {{ src_dir }}/tests/staged/data/simple_cloud + public_image: null + recommended_instance_type: null + region: null + release: null + release_notes: null + root_device: null + scanning_port: null + security_groups: [] + sha256sum: null + signing_key: null + src: {{ src_dir }}/tests/staged/data/simple_cloud/build1/CLOUD_IMAGES/something1.raw.xz + sriov_net_support: null + state: PENDING + type: AMI + uefi_support: null + usage_instructions: null + user_name: null + version_title: null + virtualization: null + volume: null +- AmiPushItem: + access_endpoint_url: null + billing_codes: null + boot_mode: hybrid + build: null + build_info: + id: null + name: rhel-ec2 + release: '1' + version: '9.4' + description: build1 sample + dest: [] + ena_support: null + image_id: null + marketplace_entity_type: null + marketplace_name: null + marketplace_title_template: null + md5sum: null + name: something2.raw.xz + origin: {{ src_dir }}/tests/staged/data/simple_cloud + public_image: null + recommended_instance_type: null + region: null + release: null + release_notes: null + root_device: null + scanning_port: null + security_groups: [] + sha256sum: null + signing_key: null + src: {{ src_dir }}/tests/staged/data/simple_cloud/build1/CLOUD_IMAGES/something2.raw.xz + sriov_net_support: null + state: PENDING + type: AMI + uefi_support: null + usage_instructions: null + user_name: null + version_title: null + virtualization: null + volume: null +- AmiPushItem: + access_endpoint_url: null + billing_codes: null + boot_mode: hybrid + build: null + build_info: + id: null + name: rhel-ec2 + release: '1' + version: '9.4' + description: build2 sample + dest: [] + ena_support: null + image_id: null + marketplace_entity_type: null + marketplace_name: null + marketplace_title_template: null + md5sum: null + name: something1.raw.xz + origin: {{ src_dir }}/tests/staged/data/simple_cloud + public_image: null + recommended_instance_type: null + region: null + release: null + release_notes: null + root_device: null + scanning_port: null + security_groups: [] + sha256sum: null + signing_key: null + src: {{ src_dir }}/tests/staged/data/simple_cloud/build2/CLOUD_IMAGES/something1.raw.xz + sriov_net_support: null + state: PENDING + type: AMI + uefi_support: null + usage_instructions: null + user_name: null + version_title: null + virtualization: null + volume: null +- AmiPushItem: + access_endpoint_url: null + billing_codes: null + boot_mode: hybrid + build: null + build_info: + id: null + name: rhel-ec2 + release: '1' + version: '9.4' + description: build2 sample + dest: [] + ena_support: null + image_id: null + marketplace_entity_type: null + marketplace_name: null + marketplace_title_template: null + md5sum: null + name: something2.raw.xz + origin: {{ src_dir }}/tests/staged/data/simple_cloud + public_image: null + recommended_instance_type: null + region: null + release: null + release_notes: null + root_device: null + scanning_port: null + security_groups: [] + sha256sum: null + signing_key: null + src: {{ src_dir }}/tests/staged/data/simple_cloud/build2/CLOUD_IMAGES/something2.raw.xz + sriov_net_support: null + state: PENDING + type: AMI + uefi_support: null + usage_instructions: null + user_name: null + version_title: null + virtualization: null + volume: null diff --git a/tests/staged/data/simple_cloud/build1/CLOUD_IMAGES/resources.yaml b/tests/staged/data/simple_cloud/build1/CLOUD_IMAGES/resources.yaml new file mode 100644 index 00000000..f7f8398c --- /dev/null +++ b/tests/staged/data/simple_cloud/build1/CLOUD_IMAGES/resources.yaml @@ -0,0 +1,14 @@ +api: v1 +resource: CloudImage +images: + - path: something1.raw.xz + architecture: x86_64 + - path: something2.raw.xz + architecture: arm64 +build: + name: rhel-ec2 + version: "9.4" + respin: "1" +description: "build1 sample" +boot_mode: hybrid +type: AMI diff --git a/tests/staged/data/simple_cloud/build1/CLOUD_IMAGES/sample1.raw.xz b/tests/staged/data/simple_cloud/build1/CLOUD_IMAGES/sample1.raw.xz new file mode 100644 index 00000000..e69de29b diff --git a/tests/staged/data/simple_cloud/build1/CLOUD_IMAGES/sample2.raw.xz b/tests/staged/data/simple_cloud/build1/CLOUD_IMAGES/sample2.raw.xz new file mode 100644 index 00000000..e69de29b diff --git a/tests/staged/data/simple_cloud/build2/CLOUD_IMAGES/resources.yaml b/tests/staged/data/simple_cloud/build2/CLOUD_IMAGES/resources.yaml new file mode 100644 index 00000000..f66cc5e0 --- /dev/null +++ b/tests/staged/data/simple_cloud/build2/CLOUD_IMAGES/resources.yaml @@ -0,0 +1,14 @@ +api: v1 +resource: CloudImage +images: + - path: something1.raw.xz + architecture: x86_64 + - path: something2.raw.xz + architecture: arm64 +build: + name: rhel-ec2 + version: "9.4" + respin: "1" +description: "build2 sample" +boot_mode: hybrid +type: AMI \ No newline at end of file diff --git a/tests/staged/data/simple_cloud/build2/CLOUD_IMAGES/sample1.raw.xz b/tests/staged/data/simple_cloud/build2/CLOUD_IMAGES/sample1.raw.xz new file mode 100644 index 00000000..e69de29b diff --git a/tests/staged/data/simple_cloud/build2/CLOUD_IMAGES/sample2.raw.xz b/tests/staged/data/simple_cloud/build2/CLOUD_IMAGES/sample2.raw.xz new file mode 100644 index 00000000..e69de29b From a0651986f3f764f411d4b0a474b32bfcfcece910 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 23 Oct 2024 16:35:19 +0000 Subject: [PATCH 2/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/schema/cloud.rst | 19 ++ docs/sources/staged.rst | 9 +- .../_impl/backend/staged/staged_cloud.py | 75 +++-- src/pushsource/_impl/schema/cloud-schema.yaml | 258 ++++++++++++++++++ tests/baseline/cases/staged-simple-cloud.yml | 93 ++++--- .../CLOUD_IMAGES/bad_path/badpath1.raw.xz} | 0 .../CLOUD_IMAGES/bad_path/resources.yaml | 12 + .../CLOUD_IMAGES/build1}/resources.yaml | 2 +- .../CLOUD_IMAGES/build1/sample1.vhd.xz} | 0 .../CLOUD_IMAGES/build1/sample2.vhd.xz} | 0 .../CLOUD_IMAGES/build2}/resources.yaml | 0 .../CLOUD_IMAGES/build2/sample1.raw.xz} | 0 .../CLOUD_IMAGES/build2/sample2.raw.xz | 0 .../CLOUD_IMAGES/build3/resources.yaml | 22 ++ .../CLOUD_IMAGES/build3/sample1.raw.xz | 0 .../empty_resource/resources.yaml | 0 .../no_resources_file/sample1.raw.xz | 0 .../starmap/CLOUD_IMAGES/staged.yaml | 0 18 files changed, 422 insertions(+), 68 deletions(-) create mode 100644 docs/schema/cloud.rst create mode 100644 src/pushsource/_impl/schema/cloud-schema.yaml rename tests/staged/data/simple_cloud/{build1/CLOUD_IMAGES/sample1.raw.xz => starmap/CLOUD_IMAGES/bad_path/badpath1.raw.xz} (100%) create mode 100644 tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/bad_path/resources.yaml rename tests/staged/data/simple_cloud/{build1/CLOUD_IMAGES => starmap/CLOUD_IMAGES/build1}/resources.yaml (96%) rename tests/staged/data/simple_cloud/{build1/CLOUD_IMAGES/sample2.raw.xz => starmap/CLOUD_IMAGES/build1/sample1.vhd.xz} (100%) rename tests/staged/data/simple_cloud/{build2/CLOUD_IMAGES/sample1.raw.xz => starmap/CLOUD_IMAGES/build1/sample2.vhd.xz} (100%) rename tests/staged/data/simple_cloud/{build2/CLOUD_IMAGES => starmap/CLOUD_IMAGES/build2}/resources.yaml (100%) rename tests/staged/data/simple_cloud/{build2/CLOUD_IMAGES/sample2.raw.xz => starmap/CLOUD_IMAGES/build2/sample1.raw.xz} (100%) create mode 100644 tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2/sample2.raw.xz create mode 100644 tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build3/resources.yaml create mode 100644 tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build3/sample1.raw.xz create mode 100644 tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/empty_resource/resources.yaml create mode 100644 tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/no_resources_file/sample1.raw.xz create mode 100644 tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/staged.yaml diff --git a/docs/schema/cloud.rst b/docs/schema/cloud.rst new file mode 100644 index 00000000..6981e242 --- /dev/null +++ b/docs/schema/cloud.rst @@ -0,0 +1,19 @@ +.. _cloud_schema: + +Schema: cloud +============== + +This document shows the schema of cloud build YAML files +supported by this library, in `JSON schema`_ format. +These files may be used within the ``CLOUD_IMAGES`` directory within +a :ref:`staging_structure`. + +The latest version of this schema is available in raw form at +https://release-engineering.github.io/pushsource/cloud-schema.yaml. + + +.. include:: ../../src/pushsource/_impl/schema/cloud-schema.yaml + :code: yaml + + +.. _JSON schema: https://json-schema.org/ diff --git a/docs/sources/staged.rst b/docs/sources/staged.rst index c6bb62a2..67b54484 100644 --- a/docs/sources/staged.rst +++ b/docs/sources/staged.rst @@ -198,14 +198,17 @@ Files in this directory must have metadata included in ``staged.yaml``. Will yield instances of :class:`~pushsource.AmiPushItem`. root/destination/CLOUD_IMAGES/\* -.............................. +................................ -The ``CLOUD_IMAGES`` should contain the VMI(s) plus a ``resources.yaml``. +The ``CLOUD_IMAGES`` directory should contain the VMI(s) plus a ``resources.yaml``. The ``resources.yaml`` contains all the information needed for the images in that folder. -Files in this directory should not include the ``staged.yaml``. +:ref:`cloud_schema` provides a complete reference of the fields which can be set by this +file. + +A blank ``staged.yaml`` must be included in the root directory. Will yield instances of either :class:`~pushsource.AmiPushItem` or :class:`~pushsource.VHDPushItem`. diff --git a/src/pushsource/_impl/backend/staged/staged_cloud.py b/src/pushsource/_impl/backend/staged/staged_cloud.py index 4a72ae7e..143fcb87 100644 --- a/src/pushsource/_impl/backend/staged/staged_cloud.py +++ b/src/pushsource/_impl/backend/staged/staged_cloud.py @@ -1,6 +1,6 @@ import logging -import yaml import os +import yaml from .staged_base import StagedBaseMixin, handles_type from ...model import VHDPushItem, AmiPushItem, AmiRelease, BootMode, KojiBuildInfo @@ -11,10 +11,11 @@ class StagedCloudMixin(StagedBaseMixin): def __build_ami_push_item(self, resources, src, origin, name): build_resources = resources.get("build") + release_resources = resources.get("release", {}) build_info = KojiBuildInfo( name=build_resources.get("name"), version=build_resources.get("version"), - release=build_resources.get("respin") + release=build_resources.get("respin"), ) image_kwargs = { "name": name, @@ -34,7 +35,7 @@ def __build_ami_push_item(self, resources, src, origin, name): ) release_required = ["product", "date", "arch", "respin"] - if all(x in build_resources for x in release_required): + if all(x in release_resources.keys() for x in release_required): release_attrs = [ "product", "date", @@ -48,7 +49,7 @@ def __build_ami_push_item(self, resources, src, origin, name): ] release_kwargs = {} for key in release_attrs: - release_kwargs[key] = build_resources.get(key) + release_kwargs[key] = release_resources.get(key) image_kwargs["release"] = AmiRelease(**release_kwargs) @@ -81,44 +82,60 @@ def __build_ami_push_item(self, resources, src, origin, name): image_kwargs[key] = resources.get(key) return AmiPushItem(**image_kwargs) - + def __build_azure_push_item(self, resources, src, origin, name): build_resources = resources.get("build") build_info = KojiBuildInfo( name=build_resources.get("name"), version=build_resources.get("version"), - release=build_resources.get("respin") + release=build_resources.get("respin"), ) image_kwargs = { "name": name, "src": src, + "description": resources.get("description"), "build_info": build_info, "origin": origin, } return VHDPushItem(**image_kwargs) - - @handles_type("CLOUD_IMAGES") - def __cloud_push_item(self, leafdir, metadata, entry): - with open(entry.path, "rt") as fh: - raw = yaml.safe_load(fh) - if not raw: - return - + + def __process_builds(self, build_dirs): out = [] - image_type = raw.get("type") - images_info = raw.get("images") + for bd in build_dirs: + yaml_path = os.path.join(bd, "resources.yaml") + try: + with open(yaml_path, "rt") as fh: + raw = yaml.safe_load(fh) + except FileNotFoundError as e: + LOG.warning("No resources.yaml file found at %s (ignored)", yaml_path) + continue + if not raw: + continue + image_type = raw.get("type", "") + images_info = raw.get("images", []) - for image in images_info: - image_relative_path = os.path.join(leafdir.path, image.get("path")) - if image_type == "AMI": - out.append(self.__build_ami_push_item(raw, - image_relative_path, - leafdir.topdir, - image.get("path"))) - elif image_type == "VHD": - out.append(self.__build_azure_push_item(raw, - image_relative_path, - leafdir.topdir, - image.get("path"))) + for image in images_info: + if "/" in image.get("path"): + LOG.warning("Unexpected '/' in %s (ignored)", image_relative_path) + continue + image_relative_path = os.path.join(bd, image.get("path")) + if image_type == "AMI": + out.append( + self.__build_ami_push_item( + raw, image_relative_path, bd, image.get("path") + ) + ) + elif image_type == "VHD": + out.append( + self.__build_azure_push_item( + raw, image_relative_path, bd, image.get("path") + ) + ) + return out + + @handles_type("CLOUD_IMAGES") + def __cloud_push_item(self, leafdir, _, entry): + builds = [ f.path for f in os.scandir(leafdir.path) if f.is_dir() ] + out = self.__process_builds(builds) - return out \ No newline at end of file + return out diff --git a/src/pushsource/_impl/schema/cloud-schema.yaml b/src/pushsource/_impl/schema/cloud-schema.yaml new file mode 100644 index 00000000..78355abd --- /dev/null +++ b/src/pushsource/_impl/schema/cloud-schema.yaml @@ -0,0 +1,258 @@ +--- +# Schema for metadata file "resources.yaml" accompanying a +# cloud staged source. +# +# For general information about staging directories, see: +# https://release-engineering.github.io/pushsource/sources/staged.html +# + +$schema: http://json-schema.org/draft-07/schema# +$id: http://release-engineering.github.io/pushsource/cloud-schema.yaml + + +############################################################################### +# Subschemas +definitions: + ami_release: + # Release metadata for an AMI. + type: object + properties: + product: + type: string + minLength: 1 + version: + $ref: "#/definitions/optional_string" + base_product: + $ref: "#/definitions/optional_string" + base_version: + $ref: "#/definitions/optional_string" + variant: + $ref: "#/definitions/optional_string" + arch: + type: string + enum: + - arm64 + - x86_64 + respin: + type: integer + type: + $ref: "#/definitions/optional_string" + enum: ['ga', 'beta', null] + date: + type: string + minLength: 1 + pattern: "^[0-9]{8}$" + required: + - product + - date + - arch + - respin + + optional_string: + # A field which may hold a non-empty string, or null. + type: + - string + - "null" + minLength: 1 + + ami_billing_codes: + # Billing codes for an AMI + type: + - object + - "null" + properties: + name: + type: string + codes: + type: array + items: + type: string + required: + - name + - codes + + image: + type: object + properties: + path: + type: string + architecture: + type: string + +type: object +properties: + api: + type: string + enum: ["v1"] + resource: + type: string + enum: ["CloudImage"] + images: + type: array + items: + $ref: "#/definitions/image" + build: + type: object + properties: + name: string + version: string + respin: string + description: + type: string + boot_mode: + enum: [ 'hybrid', 'uefi', 'legacy', null ] + type: + enum: ['AMI', 'VHD', null] + release: + $ref: "#/definitions/ami_release" + + region: + # AWS region to which this AMI should be pushed. + type: string + minLength: 1 + + type: + # Billing type for the image. + type: string + enum: + - hourly + - access + - marketplace + + virtualization: + # Virtualization type. + type: string + enum: + - hvm + + volume: + type: string + enum: + - standard + - gp2 + - gp3 + - io1 + - io2 + - st1 + - sc1 + + root_device: + type: string + minLength: 1 + + description: + $ref: "#/definitions/optional_string" + + sriov_net_support: + type: + - string + - "null" + enum: + - simple + - null + + ena_support: + type: + - boolean + - "null" + + uefi_support: + type: + - boolean + - "null" + + billing_codes: + $ref: "#/definitions/ami_billing_codes" + + boot_mode: + enum: [ 'hybrid', 'uefi', 'legacy', null ] + + public_image: + type: + - boolean + - "null" + release_notes: + $ref: "#/definitions/optional_string" + + usage_instructions: + $ref: "#/definitions/optional_string" + + recommended_instance_type: + $ref: "#/definitions/optional_string" + + marketplace_entity_type: + $ref: "#/definitions/optional_string" + + scanning_port: + type: + - number + - "null" + minimum: 0 + maximum: 65536 + + user_name: + $ref: "#/definitions/optional_string" + version_title: + $ref: "#/definitions/optional_string" + marketplace_title_template: + $ref: "#/definitions/optional_string" + marketplace_name: + $ref: "#/definitions/optional_string" + security_groups: + type: + - array + - "null" + items: + type: object + properties: + ip_protocol: + type: string + ip_ranges: + type: array + items: + type: string + if: + properties: + ip_protocol: + enum: ["icmp", "icmpv6"] + then: + properties: + from_port: + type: number + minimum: -1 + maximum: 255 + to_port: + type: number + minimum: -1 + maximum: 255 + else: + properties: + from_port: + type: number + minimum: 0 + maximum: 65536 + to_port: + type: number + minimum: 0 + maximum: 65536 + + access_endpoint_url: + type: + - object + - "null" + properties: + port: + type: number + minimum: 0 + maximum: 65536 + protocol: + enum: ["http", "https"] + +required: +- api +- resource +- images +- build +- description + +additionalProperties: false diff --git a/tests/baseline/cases/staged-simple-cloud.yml b/tests/baseline/cases/staged-simple-cloud.yml index 1949d736..d3b6e343 100644 --- a/tests/baseline/cases/staged-simple-cloud.yml +++ b/tests/baseline/cases/staged-simple-cloud.yml @@ -18,7 +18,7 @@ items: name: rhel-ec2 release: '1' version: '9.4' - description: build1 sample + description: build2 sample dest: [] ena_support: null image_id: null @@ -27,7 +27,7 @@ items: marketplace_title_template: null md5sum: null name: something1.raw.xz - origin: {{ src_dir }}/tests/staged/data/simple_cloud + origin: {{ src_dir }}/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2 public_image: null recommended_instance_type: null region: null @@ -38,7 +38,7 @@ items: security_groups: [] sha256sum: null signing_key: null - src: {{ src_dir }}/tests/staged/data/simple_cloud/build1/CLOUD_IMAGES/something1.raw.xz + src: {{ src_dir }}/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2/something1.raw.xz sriov_net_support: null state: PENDING type: AMI @@ -58,7 +58,7 @@ items: name: rhel-ec2 release: '1' version: '9.4' - description: build1 sample + description: build2 sample dest: [] ena_support: null image_id: null @@ -67,7 +67,7 @@ items: marketplace_title_template: null md5sum: null name: something2.raw.xz - origin: {{ src_dir }}/tests/staged/data/simple_cloud + origin: {{ src_dir }}/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2 public_image: null recommended_instance_type: null region: null @@ -78,7 +78,7 @@ items: security_groups: [] sha256sum: null signing_key: null - src: {{ src_dir }}/tests/staged/data/simple_cloud/build1/CLOUD_IMAGES/something2.raw.xz + src: {{ src_dir }}/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2/something2.raw.xz sriov_net_support: null state: PENDING type: AMI @@ -98,7 +98,7 @@ items: name: rhel-ec2 release: '1' version: '9.4' - description: build2 sample + description: build3 sample dest: [] ena_support: null image_id: null @@ -107,18 +107,27 @@ items: marketplace_title_template: null md5sum: null name: something1.raw.xz - origin: {{ src_dir }}/tests/staged/data/simple_cloud + origin: {{ src_dir }}/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build3 public_image: null recommended_instance_type: null region: null - release: null + release: + arch: x86_64 + base_product: null + base_version: null + date: 2020-05-11 + product: Fake-Product + respin: 1 + type: ga + variant: Fake-Variant + version: Fake-Version release_notes: null root_device: null scanning_port: null security_groups: [] sha256sum: null signing_key: null - src: {{ src_dir }}/tests/staged/data/simple_cloud/build2/CLOUD_IMAGES/something1.raw.xz + src: {{ src_dir }}/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build3/something1.raw.xz sriov_net_support: null state: PENDING type: AMI @@ -128,43 +137,57 @@ items: version_title: null virtualization: null volume: null -- AmiPushItem: - access_endpoint_url: null - billing_codes: null - boot_mode: hybrid +- VHDPushItem: + boot_mode: null build: null build_info: id: null name: rhel-ec2 release: '1' version: '9.4' - description: build2 sample + description: build1 sample dest: [] - ena_support: null - image_id: null - marketplace_entity_type: null + disk_version: null + generation: V2 + legacy_sku_id: null + marketplace_name: null + marketplace_title_template: null + md5sum: null + name: something1.raw.xz + origin: {{ src_dir }}/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build1 + recommended_sizes: [] + release: null + sas_uri: null + sha256sum: null + signing_key: null + sku_id: null + src: {{ src_dir }}/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build1/something1.raw.xz + state: PENDING + support_legacy: false +- VHDPushItem: + boot_mode: null + build: null + build_info: + id: null + name: rhel-ec2 + release: '1' + version: '9.4' + description: build1 sample + dest: [] + disk_version: null + generation: V2 + legacy_sku_id: null marketplace_name: null marketplace_title_template: null md5sum: null name: something2.raw.xz - origin: {{ src_dir }}/tests/staged/data/simple_cloud - public_image: null - recommended_instance_type: null - region: null + origin: {{ src_dir }}/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build1 + recommended_sizes: [] release: null - release_notes: null - root_device: null - scanning_port: null - security_groups: [] + sas_uri: null sha256sum: null signing_key: null - src: {{ src_dir }}/tests/staged/data/simple_cloud/build2/CLOUD_IMAGES/something2.raw.xz - sriov_net_support: null + sku_id: null + src: {{ src_dir }}/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build1/something2.raw.xz state: PENDING - type: AMI - uefi_support: null - usage_instructions: null - user_name: null - version_title: null - virtualization: null - volume: null + support_legacy: false diff --git a/tests/staged/data/simple_cloud/build1/CLOUD_IMAGES/sample1.raw.xz b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/bad_path/badpath1.raw.xz similarity index 100% rename from tests/staged/data/simple_cloud/build1/CLOUD_IMAGES/sample1.raw.xz rename to tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/bad_path/badpath1.raw.xz diff --git a/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/bad_path/resources.yaml b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/bad_path/resources.yaml new file mode 100644 index 00000000..1559593a --- /dev/null +++ b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/bad_path/resources.yaml @@ -0,0 +1,12 @@ +api: v1 +resource: CloudImage +images: + - path: "/etc/home/badpath1.raw.xz" + architecture: x86_64 +build: + name: rhel-ec2 + version: "9.4" + respin: "1" +description: "build1 sample" +boot_mode: hybrid +type: AMI diff --git a/tests/staged/data/simple_cloud/build1/CLOUD_IMAGES/resources.yaml b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build1/resources.yaml similarity index 96% rename from tests/staged/data/simple_cloud/build1/CLOUD_IMAGES/resources.yaml rename to tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build1/resources.yaml index f7f8398c..3e0e6992 100644 --- a/tests/staged/data/simple_cloud/build1/CLOUD_IMAGES/resources.yaml +++ b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build1/resources.yaml @@ -11,4 +11,4 @@ build: respin: "1" description: "build1 sample" boot_mode: hybrid -type: AMI +type: VHD diff --git a/tests/staged/data/simple_cloud/build1/CLOUD_IMAGES/sample2.raw.xz b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build1/sample1.vhd.xz similarity index 100% rename from tests/staged/data/simple_cloud/build1/CLOUD_IMAGES/sample2.raw.xz rename to tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build1/sample1.vhd.xz diff --git a/tests/staged/data/simple_cloud/build2/CLOUD_IMAGES/sample1.raw.xz b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build1/sample2.vhd.xz similarity index 100% rename from tests/staged/data/simple_cloud/build2/CLOUD_IMAGES/sample1.raw.xz rename to tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build1/sample2.vhd.xz diff --git a/tests/staged/data/simple_cloud/build2/CLOUD_IMAGES/resources.yaml b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2/resources.yaml similarity index 100% rename from tests/staged/data/simple_cloud/build2/CLOUD_IMAGES/resources.yaml rename to tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2/resources.yaml diff --git a/tests/staged/data/simple_cloud/build2/CLOUD_IMAGES/sample2.raw.xz b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2/sample1.raw.xz similarity index 100% rename from tests/staged/data/simple_cloud/build2/CLOUD_IMAGES/sample2.raw.xz rename to tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2/sample1.raw.xz diff --git a/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2/sample2.raw.xz b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2/sample2.raw.xz new file mode 100644 index 00000000..e69de29b diff --git a/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build3/resources.yaml b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build3/resources.yaml new file mode 100644 index 00000000..85c417e9 --- /dev/null +++ b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build3/resources.yaml @@ -0,0 +1,22 @@ +api: v1 +resource: CloudImage +images: + - path: something1.raw.xz + architecture: x86_64 +build: + name: rhel-ec2 + version: "9.4" + respin: "1" +release: + arch: x86_64 + base_product: null + base_version: null + date: "20200511" + product: Fake-Product + respin: 1 + type: ga + variant: Fake-Variant + version: Fake-Version +description: "build3 sample" +boot_mode: hybrid +type: AMI \ No newline at end of file diff --git a/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build3/sample1.raw.xz b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build3/sample1.raw.xz new file mode 100644 index 00000000..e69de29b diff --git a/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/empty_resource/resources.yaml b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/empty_resource/resources.yaml new file mode 100644 index 00000000..e69de29b diff --git a/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/no_resources_file/sample1.raw.xz b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/no_resources_file/sample1.raw.xz new file mode 100644 index 00000000..e69de29b diff --git a/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/staged.yaml b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/staged.yaml new file mode 100644 index 00000000..e69de29b From 3c1a735e6b616a4639d2fe9bb4b5827e60f48092 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 29 Oct 2024 16:54:53 +0000 Subject: [PATCH 3/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/sources/staged.rst | 5 +- .../_impl/backend/staged/staged_base.py | 14 ++-- .../_impl/backend/staged/staged_cloud.py | 77 ++++++++++--------- tests/baseline/cases/staged-simple-cloud.yml | 15 ++-- .../starmap/CLOUD_IMAGES/staged.yaml | 0 5 files changed, 58 insertions(+), 53 deletions(-) delete mode 100644 tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/staged.yaml diff --git a/docs/sources/staged.rst b/docs/sources/staged.rst index 67b54484..adfdf15d 100644 --- a/docs/sources/staged.rst +++ b/docs/sources/staged.rst @@ -200,7 +200,8 @@ Will yield instances of :class:`~pushsource.AmiPushItem`. root/destination/CLOUD_IMAGES/\* ................................ -The ``CLOUD_IMAGES`` directory should contain the VMI(s) plus a ``resources.yaml``. +Each directory within ``CLOUD_IMAGES`` should contain one or more VMI(s) plus a +``resources.yaml`` . The ``resources.yaml`` contains all the information needed for the images in that folder. @@ -208,8 +209,6 @@ images in that folder. :ref:`cloud_schema` provides a complete reference of the fields which can be set by this file. -A blank ``staged.yaml`` must be included in the root directory. - Will yield instances of either :class:`~pushsource.AmiPushItem` or :class:`~pushsource.VHDPushItem`. diff --git a/src/pushsource/_impl/backend/staged/staged_base.py b/src/pushsource/_impl/backend/staged/staged_base.py index 3e824ebe..b0f679d9 100644 --- a/src/pushsource/_impl/backend/staged/staged_base.py +++ b/src/pushsource/_impl/backend/staged/staged_base.py @@ -10,11 +10,12 @@ class TypeHandler(object): # Decorator for handling specific file directories (e.g. "FILES", "ISOS" etc) HANDLERS = {} - def __init__(self, type_name): + def __init__(self, type_name, accepts = lambda entry: entry.is_file()): self.type_name = type_name + self.accepts = accepts def __call__(self, fn): - TypeHandler.HANDLERS[self.type_name] = fn + TypeHandler.HANDLERS[self.type_name] = (fn, self.accepts) return fn @@ -29,19 +30,18 @@ def __init__(self, *args, **kwargs): super(StagedBaseMixin, self).__init__(*args, **kwargs) self._FILE_TYPES = self._FILE_TYPES.copy() for typename in TypeHandler.HANDLERS: - fn = TypeHandler.HANDLERS[typename] + fn, accepts = TypeHandler.HANDLERS[typename] bound_fn = partial(fn, self) self._FILE_TYPES[typename] = partial( - self.__mixin_push_items, delegate=bound_fn - ) + self.__mixin_push_items, delegate=bound_fn, accepts=accepts) - def __mixin_push_items(self, leafdir, metadata, delegate): + def __mixin_push_items(self, leafdir, metadata, delegate, accepts): out = [] LOG.debug("Looking for files in %s", leafdir) for entry in scandir(leafdir.path): - if entry.is_file(): + if accepts(entry): item = delegate(leafdir, metadata, entry) if item: out.append(item) diff --git a/src/pushsource/_impl/backend/staged/staged_cloud.py b/src/pushsource/_impl/backend/staged/staged_cloud.py index 143fcb87..54e5ef85 100644 --- a/src/pushsource/_impl/backend/staged/staged_cloud.py +++ b/src/pushsource/_impl/backend/staged/staged_cloud.py @@ -9,9 +9,10 @@ class StagedCloudMixin(StagedBaseMixin): - def __build_ami_push_item(self, resources, src, origin, name): + def __build_ami_push_item(self, resources, origin, name, dest): build_resources = resources.get("build") - release_resources = resources.get("release", {}) + release_resources = resources.get("release") or {} + src = os.path.join(origin, name) build_info = KojiBuildInfo( name=build_resources.get("name"), version=build_resources.get("version"), @@ -22,6 +23,7 @@ def __build_ami_push_item(self, resources, src, origin, name): "src": src, "build_info": build_info, "origin": origin, + "dest": [dest], } image_kwargs.update( @@ -54,7 +56,6 @@ def __build_ami_push_item(self, resources, src, origin, name): image_kwargs["release"] = AmiRelease(**release_kwargs) image_attrs = [ - "dest", "type", "region", "virtualization", @@ -83,8 +84,9 @@ def __build_ami_push_item(self, resources, src, origin, name): return AmiPushItem(**image_kwargs) - def __build_azure_push_item(self, resources, src, origin, name): + def __build_azure_push_item(self, resources, origin, name, dest): build_resources = resources.get("build") + src = os.path.join(origin, name) build_info = KojiBuildInfo( name=build_resources.get("name"), version=build_resources.get("version"), @@ -96,46 +98,45 @@ def __build_azure_push_item(self, resources, src, origin, name): "description": resources.get("description"), "build_info": build_info, "origin": origin, + "dest": [dest], } return VHDPushItem(**image_kwargs) - - def __process_builds(self, build_dirs): - out = [] - for bd in build_dirs: - yaml_path = os.path.join(bd, "resources.yaml") - try: - with open(yaml_path, "rt") as fh: - raw = yaml.safe_load(fh) - except FileNotFoundError as e: - LOG.warning("No resources.yaml file found at %s (ignored)", yaml_path) - continue - if not raw: - continue - image_type = raw.get("type", "") - images_info = raw.get("images", []) - for image in images_info: - if "/" in image.get("path"): - LOG.warning("Unexpected '/' in %s (ignored)", image_relative_path) - continue - image_relative_path = os.path.join(bd, image.get("path")) - if image_type == "AMI": - out.append( - self.__build_ami_push_item( - raw, image_relative_path, bd, image.get("path") - ) + def __process_builds(self, current_dir, leafdir): + yaml_path = os.path.join(current_dir, "resources.yaml") + try: + with open(yaml_path, "rt") as fh: + raw = yaml.safe_load(fh) + except FileNotFoundError: + LOG.warning("No resources.yaml file found at %s (ignored)", yaml_path) + return + if not raw: + LOG.warning("Resources.yaml file at %s is empty (ignored)", yaml_path) + return + image_type = raw.get("type") or "" + images_info = raw.get("images") or [] + out = [] + for image in images_info: + if "/" in image.get("path"): + LOG.warning("Unexpected '/' in %s (ignored)", image.get("path")) + return + if image_type == "AMI": + out.append( + self.__build_ami_push_item( + raw, current_dir, image.get("path"), leafdir.dest ) - elif image_type == "VHD": - out.append( - self.__build_azure_push_item( - raw, image_relative_path, bd, image.get("path") - ) + ) + elif image_type == "VHD": + out.append( + self.__build_azure_push_item( + raw, current_dir, image.get("path"), leafdir.dest ) + ) return out - @handles_type("CLOUD_IMAGES") - def __cloud_push_item(self, leafdir, _, entry): - builds = [ f.path for f in os.scandir(leafdir.path) if f.is_dir() ] - out = self.__process_builds(builds) + @handles_type("CLOUD_IMAGES", + accepts=lambda entry: entry.is_dir() and os.path.exists(os.path.join(entry.path, "resources.yaml"))) + def __cloud_push_item(self, leafdir, metadata, entry): + out = self.__process_builds(entry.path, leafdir) return out diff --git a/tests/baseline/cases/staged-simple-cloud.yml b/tests/baseline/cases/staged-simple-cloud.yml index d3b6e343..a3c4c62f 100644 --- a/tests/baseline/cases/staged-simple-cloud.yml +++ b/tests/baseline/cases/staged-simple-cloud.yml @@ -19,7 +19,8 @@ items: release: '1' version: '9.4' description: build2 sample - dest: [] + dest: + - starmap ena_support: null image_id: null marketplace_entity_type: null @@ -59,7 +60,8 @@ items: release: '1' version: '9.4' description: build2 sample - dest: [] + dest: + - starmap ena_support: null image_id: null marketplace_entity_type: null @@ -99,7 +101,8 @@ items: release: '1' version: '9.4' description: build3 sample - dest: [] + dest: + - starmap ena_support: null image_id: null marketplace_entity_type: null @@ -146,7 +149,8 @@ items: release: '1' version: '9.4' description: build1 sample - dest: [] + dest: + - starmap disk_version: null generation: V2 legacy_sku_id: null @@ -173,7 +177,8 @@ items: release: '1' version: '9.4' description: build1 sample - dest: [] + dest: + - starmap disk_version: null generation: V2 legacy_sku_id: null diff --git a/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/staged.yaml b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/staged.yaml deleted file mode 100644 index e69de29b..00000000 From 1cad79b7b4a7da701779933f00432cc992ff9c59 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 30 Oct 2024 14:36:20 +0000 Subject: [PATCH 4/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../_impl/backend/staged/staged_base.py | 5 +-- .../_impl/backend/staged/staged_cloud.py | 36 +++++++++---------- tests/baseline/cases/staged-simple-cloud.yml | 10 +++--- .../CLOUD_IMAGES/build1/resources.yaml | 2 ++ .../CLOUD_IMAGES/build2/resources.yaml | 2 ++ .../CLOUD_IMAGES/build3/resources.yaml | 1 + 6 files changed, 30 insertions(+), 26 deletions(-) diff --git a/src/pushsource/_impl/backend/staged/staged_base.py b/src/pushsource/_impl/backend/staged/staged_base.py index b0f679d9..1ec752a8 100644 --- a/src/pushsource/_impl/backend/staged/staged_base.py +++ b/src/pushsource/_impl/backend/staged/staged_base.py @@ -10,7 +10,7 @@ class TypeHandler(object): # Decorator for handling specific file directories (e.g. "FILES", "ISOS" etc) HANDLERS = {} - def __init__(self, type_name, accepts = lambda entry: entry.is_file()): + def __init__(self, type_name, accepts=lambda entry: entry.is_file()): self.type_name = type_name self.accepts = accepts @@ -33,7 +33,8 @@ def __init__(self, *args, **kwargs): fn, accepts = TypeHandler.HANDLERS[typename] bound_fn = partial(fn, self) self._FILE_TYPES[typename] = partial( - self.__mixin_push_items, delegate=bound_fn, accepts=accepts) + self.__mixin_push_items, delegate=bound_fn, accepts=accepts + ) def __mixin_push_items(self, leafdir, metadata, delegate, accepts): out = [] diff --git a/src/pushsource/_impl/backend/staged/staged_cloud.py b/src/pushsource/_impl/backend/staged/staged_cloud.py index 54e5ef85..22234f4f 100644 --- a/src/pushsource/_impl/backend/staged/staged_cloud.py +++ b/src/pushsource/_impl/backend/staged/staged_cloud.py @@ -9,9 +9,10 @@ class StagedCloudMixin(StagedBaseMixin): - def __build_ami_push_item(self, resources, origin, name, dest): + def __build_ami_push_item(self, resources, origin, image, dest): build_resources = resources.get("build") release_resources = resources.get("release") or {} + name = image.get("path") src = os.path.join(origin, name) build_info = KojiBuildInfo( name=build_resources.get("name"), @@ -24,6 +25,7 @@ def __build_ami_push_item(self, resources, origin, name, dest): "build_info": build_info, "origin": origin, "dest": [dest], + "sha256sum": image.get("sha256sum"), } image_kwargs.update( @@ -84,8 +86,9 @@ def __build_ami_push_item(self, resources, origin, name, dest): return AmiPushItem(**image_kwargs) - def __build_azure_push_item(self, resources, origin, name, dest): + def __build_azure_push_item(self, resources, origin, image, dest): build_resources = resources.get("build") + name = image.get("path") src = os.path.join(origin, name) build_info = KojiBuildInfo( name=build_resources.get("name"), @@ -99,17 +102,19 @@ def __build_azure_push_item(self, resources, origin, name, dest): "build_info": build_info, "origin": origin, "dest": [dest], + "sha256sum": image.get("sha256sum"), } return VHDPushItem(**image_kwargs) - def __process_builds(self, current_dir, leafdir): - yaml_path = os.path.join(current_dir, "resources.yaml") - try: - with open(yaml_path, "rt") as fh: - raw = yaml.safe_load(fh) - except FileNotFoundError: - LOG.warning("No resources.yaml file found at %s (ignored)", yaml_path) - return + @handles_type( + "CLOUD_IMAGES", + accepts=lambda entry: entry.is_dir() + and os.path.exists(os.path.join(entry.path, "resources.yaml")), + ) + def __cloud_push_item(self, leafdir, _, entry): + yaml_path = os.path.join(entry.path, "resources.yaml") + with open(yaml_path, "rt") as fh: + raw = yaml.safe_load(fh) if not raw: LOG.warning("Resources.yaml file at %s is empty (ignored)", yaml_path) return @@ -123,20 +128,13 @@ def __process_builds(self, current_dir, leafdir): if image_type == "AMI": out.append( self.__build_ami_push_item( - raw, current_dir, image.get("path"), leafdir.dest + raw, entry.path, image, leafdir.dest ) ) elif image_type == "VHD": out.append( self.__build_azure_push_item( - raw, current_dir, image.get("path"), leafdir.dest + raw, entry.path, image, leafdir.dest ) ) return out - - @handles_type("CLOUD_IMAGES", - accepts=lambda entry: entry.is_dir() and os.path.exists(os.path.join(entry.path, "resources.yaml"))) - def __cloud_push_item(self, leafdir, metadata, entry): - out = self.__process_builds(entry.path, leafdir) - - return out diff --git a/tests/baseline/cases/staged-simple-cloud.yml b/tests/baseline/cases/staged-simple-cloud.yml index a3c4c62f..2530ce66 100644 --- a/tests/baseline/cases/staged-simple-cloud.yml +++ b/tests/baseline/cases/staged-simple-cloud.yml @@ -37,7 +37,7 @@ items: root_device: null scanning_port: null security_groups: [] - sha256sum: null + sha256sum: 04db0c39efb31518ff79bf98d1c27256d46cdc72b967a5b2094a6efec3166df2 signing_key: null src: {{ src_dir }}/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2/something1.raw.xz sriov_net_support: null @@ -78,7 +78,7 @@ items: root_device: null scanning_port: null security_groups: [] - sha256sum: null + sha256sum: 04db0c39efb31518ff79bf98d1c27256d46cdc72b967a5b2094a6efec3166df2 signing_key: null src: {{ src_dir }}/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2/something2.raw.xz sriov_net_support: null @@ -128,7 +128,7 @@ items: root_device: null scanning_port: null security_groups: [] - sha256sum: null + sha256sum: 04db0c39efb31518ff79bf98d1c27256d46cdc72b967a5b2094a6efec3166df2 signing_key: null src: {{ src_dir }}/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build3/something1.raw.xz sriov_net_support: null @@ -162,7 +162,7 @@ items: recommended_sizes: [] release: null sas_uri: null - sha256sum: null + sha256sum: 04db0c39efb31518ff79bf98d1c27256d46cdc72b967a5b2094a6efec3166df2 signing_key: null sku_id: null src: {{ src_dir }}/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build1/something1.raw.xz @@ -190,7 +190,7 @@ items: recommended_sizes: [] release: null sas_uri: null - sha256sum: null + sha256sum: 04db0c39efb31518ff79bf98d1c27256d46cdc72b967a5b2094a6efec3166df2 signing_key: null sku_id: null src: {{ src_dir }}/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build1/something2.raw.xz diff --git a/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build1/resources.yaml b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build1/resources.yaml index 3e0e6992..a4ae91bd 100644 --- a/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build1/resources.yaml +++ b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build1/resources.yaml @@ -3,8 +3,10 @@ resource: CloudImage images: - path: something1.raw.xz architecture: x86_64 + sha256sum: 04db0c39efb31518ff79bf98d1c27256d46cdc72b967a5b2094a6efec3166df2 - path: something2.raw.xz architecture: arm64 + sha256sum: 04db0c39efb31518ff79bf98d1c27256d46cdc72b967a5b2094a6efec3166df2 build: name: rhel-ec2 version: "9.4" diff --git a/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2/resources.yaml b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2/resources.yaml index f66cc5e0..c0c761df 100644 --- a/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2/resources.yaml +++ b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2/resources.yaml @@ -3,8 +3,10 @@ resource: CloudImage images: - path: something1.raw.xz architecture: x86_64 + sha256sum: 04db0c39efb31518ff79bf98d1c27256d46cdc72b967a5b2094a6efec3166df2 - path: something2.raw.xz architecture: arm64 + sha256sum: 04db0c39efb31518ff79bf98d1c27256d46cdc72b967a5b2094a6efec3166df2 build: name: rhel-ec2 version: "9.4" diff --git a/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build3/resources.yaml b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build3/resources.yaml index 85c417e9..407d6203 100644 --- a/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build3/resources.yaml +++ b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build3/resources.yaml @@ -3,6 +3,7 @@ resource: CloudImage images: - path: something1.raw.xz architecture: x86_64 + sha256sum: 04db0c39efb31518ff79bf98d1c27256d46cdc72b967a5b2094a6efec3166df2 build: name: rhel-ec2 version: "9.4" From 7e54c0042fc8d41635ce1fe501da20639914969b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 30 Oct 2024 19:41:07 +0000 Subject: [PATCH 5/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../_impl/backend/staged/staged_cloud.py | 65 ++++++++++++------- tests/baseline/cases/staged-simple-cloud.yml | 48 ++++++++++++-- 2 files changed, 82 insertions(+), 31 deletions(-) diff --git a/src/pushsource/_impl/backend/staged/staged_cloud.py b/src/pushsource/_impl/backend/staged/staged_cloud.py index 22234f4f..5ec5c3f4 100644 --- a/src/pushsource/_impl/backend/staged/staged_cloud.py +++ b/src/pushsource/_impl/backend/staged/staged_cloud.py @@ -2,13 +2,24 @@ import os import yaml +from datetime import datetime, timezone + from .staged_base import StagedBaseMixin, handles_type -from ...model import VHDPushItem, AmiPushItem, AmiRelease, BootMode, KojiBuildInfo +from ...model import VHDPushItem, VMIRelease, AmiPushItem, AmiRelease, BootMode, KojiBuildInfo LOG = logging.getLogger("pushsource") class StagedCloudMixin(StagedBaseMixin): + def __get_product_name(self, base_name): + splitted_name = base_name.split("-") + if len(splitted_name) > 1: + product = "-".join(splitted_name[:-1]) + else: + product = splitted_name[0] + return product + + def __build_ami_push_item(self, resources, origin, image, dest): build_resources = resources.get("build") release_resources = resources.get("release") or {} @@ -38,24 +49,23 @@ def __build_ami_push_item(self, resources, origin, image, dest): } ) - release_required = ["product", "date", "arch", "respin"] - if all(x in release_resources.keys() for x in release_required): - release_attrs = [ - "product", - "date", - "arch", - "respin", - "version", - "base_product", - "base_version", - "variant", - "type", - ] - release_kwargs = {} - for key in release_attrs: - release_kwargs[key] = release_resources.get(key) - - image_kwargs["release"] = AmiRelease(**release_kwargs) + release_kwargs = { + "product": self.__get_product_name(build_resources.get("name")), + "date": datetime.now(timezone.utc).strftime("%Y%m%d"), + "arch": image.get("architecture"), + "respin": int(build_resources.get("respin")) or 0 + } + release_attrs = [ + "version", + "base_product", + "base_version", + "variant", + "type", + ] + for key in release_attrs: + release_kwargs[key] = release_resources.get(key) + + image_kwargs["release"] = AmiRelease(**release_kwargs) image_attrs = [ "type", @@ -95,6 +105,14 @@ def __build_azure_push_item(self, resources, origin, image, dest): version=build_resources.get("version"), release=build_resources.get("respin"), ) + + release_kwargs = { + "product": self.__get_product_name(build_resources.get("name")), + "date": datetime.now(timezone.utc).strftime("%Y%m%d"), + "arch": image.get("architecture"), + "respin": int(build_resources.get("respin")) or 0 + } + image_kwargs = { "name": name, "src": src, @@ -103,6 +121,7 @@ def __build_azure_push_item(self, resources, origin, image, dest): "origin": origin, "dest": [dest], "sha256sum": image.get("sha256sum"), + "release": VMIRelease(**release_kwargs) } return VHDPushItem(**image_kwargs) @@ -127,14 +146,10 @@ def __cloud_push_item(self, leafdir, _, entry): return if image_type == "AMI": out.append( - self.__build_ami_push_item( - raw, entry.path, image, leafdir.dest - ) + self.__build_ami_push_item(raw, entry.path, image, leafdir.dest) ) elif image_type == "VHD": out.append( - self.__build_azure_push_item( - raw, entry.path, image, leafdir.dest - ) + self.__build_azure_push_item(raw, entry.path, image, leafdir.dest) ) return out diff --git a/tests/baseline/cases/staged-simple-cloud.yml b/tests/baseline/cases/staged-simple-cloud.yml index 2530ce66..338c1034 100644 --- a/tests/baseline/cases/staged-simple-cloud.yml +++ b/tests/baseline/cases/staged-simple-cloud.yml @@ -32,7 +32,16 @@ items: public_image: null recommended_instance_type: null region: null - release: null + release: + arch: x86_64 + base_product: null + base_version: null + date: 2024-10-31 + product: rhel + respin: 1 + type: null + variant: null + version: null release_notes: null root_device: null scanning_port: null @@ -73,7 +82,16 @@ items: public_image: null recommended_instance_type: null region: null - release: null + release: + arch: arm64 + base_product: null + base_version: null + date: 2024-10-31 + product: rhel + respin: 1 + type: null + variant: null + version: null release_notes: null root_device: null scanning_port: null @@ -118,8 +136,8 @@ items: arch: x86_64 base_product: null base_version: null - date: 2020-05-11 - product: Fake-Product + date: 2024-10-31 + product: rhel respin: 1 type: ga variant: Fake-Variant @@ -160,7 +178,16 @@ items: name: something1.raw.xz origin: {{ src_dir }}/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build1 recommended_sizes: [] - release: null + release: + arch: x86_64 + base_product: null + base_version: null + date: 2024-10-31 + product: rhel + respin: 1 + type: null + variant: null + version: null sas_uri: null sha256sum: 04db0c39efb31518ff79bf98d1c27256d46cdc72b967a5b2094a6efec3166df2 signing_key: null @@ -188,7 +215,16 @@ items: name: something2.raw.xz origin: {{ src_dir }}/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build1 recommended_sizes: [] - release: null + release: + arch: arm64 + base_product: null + base_version: null + date: 2024-10-31 + product: rhel + respin: 1 + type: null + variant: null + version: null sas_uri: null sha256sum: 04db0c39efb31518ff79bf98d1c27256d46cdc72b967a5b2094a6efec3166df2 signing_key: null From eeb56c0af621fae4e6d0330a5cd1bc0633c54d96 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 31 Oct 2024 12:52:21 +0000 Subject: [PATCH 6/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../_impl/backend/staged/staged_cloud.py | 16 +++++++++++----- tests/baseline/cases/staged-simple-cloud.yml | 4 ++-- .../starmap/CLOUD_IMAGES/build2/resources.yaml | 2 +- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/pushsource/_impl/backend/staged/staged_cloud.py b/src/pushsource/_impl/backend/staged/staged_cloud.py index 5ec5c3f4..e3b6ef81 100644 --- a/src/pushsource/_impl/backend/staged/staged_cloud.py +++ b/src/pushsource/_impl/backend/staged/staged_cloud.py @@ -5,7 +5,14 @@ from datetime import datetime, timezone from .staged_base import StagedBaseMixin, handles_type -from ...model import VHDPushItem, VMIRelease, AmiPushItem, AmiRelease, BootMode, KojiBuildInfo +from ...model import ( + VHDPushItem, + VMIRelease, + AmiPushItem, + AmiRelease, + BootMode, + KojiBuildInfo, +) LOG = logging.getLogger("pushsource") @@ -19,7 +26,6 @@ def __get_product_name(self, base_name): product = splitted_name[0] return product - def __build_ami_push_item(self, resources, origin, image, dest): build_resources = resources.get("build") release_resources = resources.get("release") or {} @@ -53,7 +59,7 @@ def __build_ami_push_item(self, resources, origin, image, dest): "product": self.__get_product_name(build_resources.get("name")), "date": datetime.now(timezone.utc).strftime("%Y%m%d"), "arch": image.get("architecture"), - "respin": int(build_resources.get("respin")) or 0 + "respin": int(build_resources.get("respin")) or 0, } release_attrs = [ "version", @@ -110,7 +116,7 @@ def __build_azure_push_item(self, resources, origin, image, dest): "product": self.__get_product_name(build_resources.get("name")), "date": datetime.now(timezone.utc).strftime("%Y%m%d"), "arch": image.get("architecture"), - "respin": int(build_resources.get("respin")) or 0 + "respin": int(build_resources.get("respin")) or 0, } image_kwargs = { @@ -121,7 +127,7 @@ def __build_azure_push_item(self, resources, origin, image, dest): "origin": origin, "dest": [dest], "sha256sum": image.get("sha256sum"), - "release": VMIRelease(**release_kwargs) + "release": VMIRelease(**release_kwargs), } return VHDPushItem(**image_kwargs) diff --git a/tests/baseline/cases/staged-simple-cloud.yml b/tests/baseline/cases/staged-simple-cloud.yml index 338c1034..7078f8fa 100644 --- a/tests/baseline/cases/staged-simple-cloud.yml +++ b/tests/baseline/cases/staged-simple-cloud.yml @@ -15,7 +15,7 @@ items: build: null build_info: id: null - name: rhel-ec2 + name: rhel release: '1' version: '9.4' description: build2 sample @@ -65,7 +65,7 @@ items: build: null build_info: id: null - name: rhel-ec2 + name: rhel release: '1' version: '9.4' description: build2 sample diff --git a/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2/resources.yaml b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2/resources.yaml index c0c761df..180f3156 100644 --- a/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2/resources.yaml +++ b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2/resources.yaml @@ -8,7 +8,7 @@ images: architecture: arm64 sha256sum: 04db0c39efb31518ff79bf98d1c27256d46cdc72b967a5b2094a6efec3166df2 build: - name: rhel-ec2 + name: rhel version: "9.4" respin: "1" description: "build2 sample"