From a0651986f3f764f411d4b0a474b32bfcfcece910 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 23 Oct 2024 16:35:19 +0000 Subject: [PATCH] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/schema/cloud.rst | 19 ++ docs/sources/staged.rst | 9 +- .../_impl/backend/staged/staged_cloud.py | 75 +++-- src/pushsource/_impl/schema/cloud-schema.yaml | 258 ++++++++++++++++++ tests/baseline/cases/staged-simple-cloud.yml | 93 ++++--- .../CLOUD_IMAGES/bad_path/badpath1.raw.xz} | 0 .../CLOUD_IMAGES/bad_path/resources.yaml | 12 + .../CLOUD_IMAGES/build1}/resources.yaml | 2 +- .../CLOUD_IMAGES/build1/sample1.vhd.xz} | 0 .../CLOUD_IMAGES/build1/sample2.vhd.xz} | 0 .../CLOUD_IMAGES/build2}/resources.yaml | 0 .../CLOUD_IMAGES/build2/sample1.raw.xz} | 0 .../CLOUD_IMAGES/build2/sample2.raw.xz | 0 .../CLOUD_IMAGES/build3/resources.yaml | 22 ++ .../CLOUD_IMAGES/build3/sample1.raw.xz | 0 .../empty_resource/resources.yaml | 0 .../no_resources_file/sample1.raw.xz | 0 .../starmap/CLOUD_IMAGES/staged.yaml | 0 18 files changed, 422 insertions(+), 68 deletions(-) create mode 100644 docs/schema/cloud.rst create mode 100644 src/pushsource/_impl/schema/cloud-schema.yaml rename tests/staged/data/simple_cloud/{build1/CLOUD_IMAGES/sample1.raw.xz => starmap/CLOUD_IMAGES/bad_path/badpath1.raw.xz} (100%) create mode 100644 tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/bad_path/resources.yaml rename tests/staged/data/simple_cloud/{build1/CLOUD_IMAGES => starmap/CLOUD_IMAGES/build1}/resources.yaml (96%) rename tests/staged/data/simple_cloud/{build1/CLOUD_IMAGES/sample2.raw.xz => starmap/CLOUD_IMAGES/build1/sample1.vhd.xz} (100%) rename tests/staged/data/simple_cloud/{build2/CLOUD_IMAGES/sample1.raw.xz => starmap/CLOUD_IMAGES/build1/sample2.vhd.xz} (100%) rename tests/staged/data/simple_cloud/{build2/CLOUD_IMAGES => starmap/CLOUD_IMAGES/build2}/resources.yaml (100%) rename tests/staged/data/simple_cloud/{build2/CLOUD_IMAGES/sample2.raw.xz => starmap/CLOUD_IMAGES/build2/sample1.raw.xz} (100%) create mode 100644 tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2/sample2.raw.xz create mode 100644 tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build3/resources.yaml create mode 100644 tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build3/sample1.raw.xz create mode 100644 tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/empty_resource/resources.yaml create mode 100644 tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/no_resources_file/sample1.raw.xz create mode 100644 tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/staged.yaml diff --git a/docs/schema/cloud.rst b/docs/schema/cloud.rst new file mode 100644 index 00000000..6981e242 --- /dev/null +++ b/docs/schema/cloud.rst @@ -0,0 +1,19 @@ +.. _cloud_schema: + +Schema: cloud +============== + +This document shows the schema of cloud build YAML files +supported by this library, in `JSON schema`_ format. +These files may be used within the ``CLOUD_IMAGES`` directory within +a :ref:`staging_structure`. + +The latest version of this schema is available in raw form at +https://release-engineering.github.io/pushsource/cloud-schema.yaml. + + +.. include:: ../../src/pushsource/_impl/schema/cloud-schema.yaml + :code: yaml + + +.. _JSON schema: https://json-schema.org/ diff --git a/docs/sources/staged.rst b/docs/sources/staged.rst index c6bb62a2..67b54484 100644 --- a/docs/sources/staged.rst +++ b/docs/sources/staged.rst @@ -198,14 +198,17 @@ Files in this directory must have metadata included in ``staged.yaml``. Will yield instances of :class:`~pushsource.AmiPushItem`. root/destination/CLOUD_IMAGES/\* -.............................. +................................ -The ``CLOUD_IMAGES`` should contain the VMI(s) plus a ``resources.yaml``. +The ``CLOUD_IMAGES`` directory should contain the VMI(s) plus a ``resources.yaml``. The ``resources.yaml`` contains all the information needed for the images in that folder. -Files in this directory should not include the ``staged.yaml``. +:ref:`cloud_schema` provides a complete reference of the fields which can be set by this +file. + +A blank ``staged.yaml`` must be included in the root directory. Will yield instances of either :class:`~pushsource.AmiPushItem` or :class:`~pushsource.VHDPushItem`. diff --git a/src/pushsource/_impl/backend/staged/staged_cloud.py b/src/pushsource/_impl/backend/staged/staged_cloud.py index 4a72ae7e..143fcb87 100644 --- a/src/pushsource/_impl/backend/staged/staged_cloud.py +++ b/src/pushsource/_impl/backend/staged/staged_cloud.py @@ -1,6 +1,6 @@ import logging -import yaml import os +import yaml from .staged_base import StagedBaseMixin, handles_type from ...model import VHDPushItem, AmiPushItem, AmiRelease, BootMode, KojiBuildInfo @@ -11,10 +11,11 @@ class StagedCloudMixin(StagedBaseMixin): def __build_ami_push_item(self, resources, src, origin, name): build_resources = resources.get("build") + release_resources = resources.get("release", {}) build_info = KojiBuildInfo( name=build_resources.get("name"), version=build_resources.get("version"), - release=build_resources.get("respin") + release=build_resources.get("respin"), ) image_kwargs = { "name": name, @@ -34,7 +35,7 @@ def __build_ami_push_item(self, resources, src, origin, name): ) release_required = ["product", "date", "arch", "respin"] - if all(x in build_resources for x in release_required): + if all(x in release_resources.keys() for x in release_required): release_attrs = [ "product", "date", @@ -48,7 +49,7 @@ def __build_ami_push_item(self, resources, src, origin, name): ] release_kwargs = {} for key in release_attrs: - release_kwargs[key] = build_resources.get(key) + release_kwargs[key] = release_resources.get(key) image_kwargs["release"] = AmiRelease(**release_kwargs) @@ -81,44 +82,60 @@ def __build_ami_push_item(self, resources, src, origin, name): image_kwargs[key] = resources.get(key) return AmiPushItem(**image_kwargs) - + def __build_azure_push_item(self, resources, src, origin, name): build_resources = resources.get("build") build_info = KojiBuildInfo( name=build_resources.get("name"), version=build_resources.get("version"), - release=build_resources.get("respin") + release=build_resources.get("respin"), ) image_kwargs = { "name": name, "src": src, + "description": resources.get("description"), "build_info": build_info, "origin": origin, } return VHDPushItem(**image_kwargs) - - @handles_type("CLOUD_IMAGES") - def __cloud_push_item(self, leafdir, metadata, entry): - with open(entry.path, "rt") as fh: - raw = yaml.safe_load(fh) - if not raw: - return - + + def __process_builds(self, build_dirs): out = [] - image_type = raw.get("type") - images_info = raw.get("images") + for bd in build_dirs: + yaml_path = os.path.join(bd, "resources.yaml") + try: + with open(yaml_path, "rt") as fh: + raw = yaml.safe_load(fh) + except FileNotFoundError as e: + LOG.warning("No resources.yaml file found at %s (ignored)", yaml_path) + continue + if not raw: + continue + image_type = raw.get("type", "") + images_info = raw.get("images", []) - for image in images_info: - image_relative_path = os.path.join(leafdir.path, image.get("path")) - if image_type == "AMI": - out.append(self.__build_ami_push_item(raw, - image_relative_path, - leafdir.topdir, - image.get("path"))) - elif image_type == "VHD": - out.append(self.__build_azure_push_item(raw, - image_relative_path, - leafdir.topdir, - image.get("path"))) + for image in images_info: + if "/" in image.get("path"): + LOG.warning("Unexpected '/' in %s (ignored)", image_relative_path) + continue + image_relative_path = os.path.join(bd, image.get("path")) + if image_type == "AMI": + out.append( + self.__build_ami_push_item( + raw, image_relative_path, bd, image.get("path") + ) + ) + elif image_type == "VHD": + out.append( + self.__build_azure_push_item( + raw, image_relative_path, bd, image.get("path") + ) + ) + return out + + @handles_type("CLOUD_IMAGES") + def __cloud_push_item(self, leafdir, _, entry): + builds = [ f.path for f in os.scandir(leafdir.path) if f.is_dir() ] + out = self.__process_builds(builds) - return out \ No newline at end of file + return out diff --git a/src/pushsource/_impl/schema/cloud-schema.yaml b/src/pushsource/_impl/schema/cloud-schema.yaml new file mode 100644 index 00000000..78355abd --- /dev/null +++ b/src/pushsource/_impl/schema/cloud-schema.yaml @@ -0,0 +1,258 @@ +--- +# Schema for metadata file "resources.yaml" accompanying a +# cloud staged source. +# +# For general information about staging directories, see: +# https://release-engineering.github.io/pushsource/sources/staged.html +# + +$schema: http://json-schema.org/draft-07/schema# +$id: http://release-engineering.github.io/pushsource/cloud-schema.yaml + + +############################################################################### +# Subschemas +definitions: + ami_release: + # Release metadata for an AMI. + type: object + properties: + product: + type: string + minLength: 1 + version: + $ref: "#/definitions/optional_string" + base_product: + $ref: "#/definitions/optional_string" + base_version: + $ref: "#/definitions/optional_string" + variant: + $ref: "#/definitions/optional_string" + arch: + type: string + enum: + - arm64 + - x86_64 + respin: + type: integer + type: + $ref: "#/definitions/optional_string" + enum: ['ga', 'beta', null] + date: + type: string + minLength: 1 + pattern: "^[0-9]{8}$" + required: + - product + - date + - arch + - respin + + optional_string: + # A field which may hold a non-empty string, or null. + type: + - string + - "null" + minLength: 1 + + ami_billing_codes: + # Billing codes for an AMI + type: + - object + - "null" + properties: + name: + type: string + codes: + type: array + items: + type: string + required: + - name + - codes + + image: + type: object + properties: + path: + type: string + architecture: + type: string + +type: object +properties: + api: + type: string + enum: ["v1"] + resource: + type: string + enum: ["CloudImage"] + images: + type: array + items: + $ref: "#/definitions/image" + build: + type: object + properties: + name: string + version: string + respin: string + description: + type: string + boot_mode: + enum: [ 'hybrid', 'uefi', 'legacy', null ] + type: + enum: ['AMI', 'VHD', null] + release: + $ref: "#/definitions/ami_release" + + region: + # AWS region to which this AMI should be pushed. + type: string + minLength: 1 + + type: + # Billing type for the image. + type: string + enum: + - hourly + - access + - marketplace + + virtualization: + # Virtualization type. + type: string + enum: + - hvm + + volume: + type: string + enum: + - standard + - gp2 + - gp3 + - io1 + - io2 + - st1 + - sc1 + + root_device: + type: string + minLength: 1 + + description: + $ref: "#/definitions/optional_string" + + sriov_net_support: + type: + - string + - "null" + enum: + - simple + - null + + ena_support: + type: + - boolean + - "null" + + uefi_support: + type: + - boolean + - "null" + + billing_codes: + $ref: "#/definitions/ami_billing_codes" + + boot_mode: + enum: [ 'hybrid', 'uefi', 'legacy', null ] + + public_image: + type: + - boolean + - "null" + release_notes: + $ref: "#/definitions/optional_string" + + usage_instructions: + $ref: "#/definitions/optional_string" + + recommended_instance_type: + $ref: "#/definitions/optional_string" + + marketplace_entity_type: + $ref: "#/definitions/optional_string" + + scanning_port: + type: + - number + - "null" + minimum: 0 + maximum: 65536 + + user_name: + $ref: "#/definitions/optional_string" + version_title: + $ref: "#/definitions/optional_string" + marketplace_title_template: + $ref: "#/definitions/optional_string" + marketplace_name: + $ref: "#/definitions/optional_string" + security_groups: + type: + - array + - "null" + items: + type: object + properties: + ip_protocol: + type: string + ip_ranges: + type: array + items: + type: string + if: + properties: + ip_protocol: + enum: ["icmp", "icmpv6"] + then: + properties: + from_port: + type: number + minimum: -1 + maximum: 255 + to_port: + type: number + minimum: -1 + maximum: 255 + else: + properties: + from_port: + type: number + minimum: 0 + maximum: 65536 + to_port: + type: number + minimum: 0 + maximum: 65536 + + access_endpoint_url: + type: + - object + - "null" + properties: + port: + type: number + minimum: 0 + maximum: 65536 + protocol: + enum: ["http", "https"] + +required: +- api +- resource +- images +- build +- description + +additionalProperties: false diff --git a/tests/baseline/cases/staged-simple-cloud.yml b/tests/baseline/cases/staged-simple-cloud.yml index 1949d736..d3b6e343 100644 --- a/tests/baseline/cases/staged-simple-cloud.yml +++ b/tests/baseline/cases/staged-simple-cloud.yml @@ -18,7 +18,7 @@ items: name: rhel-ec2 release: '1' version: '9.4' - description: build1 sample + description: build2 sample dest: [] ena_support: null image_id: null @@ -27,7 +27,7 @@ items: marketplace_title_template: null md5sum: null name: something1.raw.xz - origin: {{ src_dir }}/tests/staged/data/simple_cloud + origin: {{ src_dir }}/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2 public_image: null recommended_instance_type: null region: null @@ -38,7 +38,7 @@ items: security_groups: [] sha256sum: null signing_key: null - src: {{ src_dir }}/tests/staged/data/simple_cloud/build1/CLOUD_IMAGES/something1.raw.xz + src: {{ src_dir }}/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2/something1.raw.xz sriov_net_support: null state: PENDING type: AMI @@ -58,7 +58,7 @@ items: name: rhel-ec2 release: '1' version: '9.4' - description: build1 sample + description: build2 sample dest: [] ena_support: null image_id: null @@ -67,7 +67,7 @@ items: marketplace_title_template: null md5sum: null name: something2.raw.xz - origin: {{ src_dir }}/tests/staged/data/simple_cloud + origin: {{ src_dir }}/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2 public_image: null recommended_instance_type: null region: null @@ -78,7 +78,7 @@ items: security_groups: [] sha256sum: null signing_key: null - src: {{ src_dir }}/tests/staged/data/simple_cloud/build1/CLOUD_IMAGES/something2.raw.xz + src: {{ src_dir }}/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2/something2.raw.xz sriov_net_support: null state: PENDING type: AMI @@ -98,7 +98,7 @@ items: name: rhel-ec2 release: '1' version: '9.4' - description: build2 sample + description: build3 sample dest: [] ena_support: null image_id: null @@ -107,18 +107,27 @@ items: marketplace_title_template: null md5sum: null name: something1.raw.xz - origin: {{ src_dir }}/tests/staged/data/simple_cloud + origin: {{ src_dir }}/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build3 public_image: null recommended_instance_type: null region: null - release: null + release: + arch: x86_64 + base_product: null + base_version: null + date: 2020-05-11 + product: Fake-Product + respin: 1 + type: ga + variant: Fake-Variant + version: Fake-Version release_notes: null root_device: null scanning_port: null security_groups: [] sha256sum: null signing_key: null - src: {{ src_dir }}/tests/staged/data/simple_cloud/build2/CLOUD_IMAGES/something1.raw.xz + src: {{ src_dir }}/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build3/something1.raw.xz sriov_net_support: null state: PENDING type: AMI @@ -128,43 +137,57 @@ items: version_title: null virtualization: null volume: null -- AmiPushItem: - access_endpoint_url: null - billing_codes: null - boot_mode: hybrid +- VHDPushItem: + boot_mode: null build: null build_info: id: null name: rhel-ec2 release: '1' version: '9.4' - description: build2 sample + description: build1 sample dest: [] - ena_support: null - image_id: null - marketplace_entity_type: null + disk_version: null + generation: V2 + legacy_sku_id: null + marketplace_name: null + marketplace_title_template: null + md5sum: null + name: something1.raw.xz + origin: {{ src_dir }}/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build1 + recommended_sizes: [] + release: null + sas_uri: null + sha256sum: null + signing_key: null + sku_id: null + src: {{ src_dir }}/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build1/something1.raw.xz + state: PENDING + support_legacy: false +- VHDPushItem: + boot_mode: null + build: null + build_info: + id: null + name: rhel-ec2 + release: '1' + version: '9.4' + description: build1 sample + dest: [] + disk_version: null + generation: V2 + legacy_sku_id: null marketplace_name: null marketplace_title_template: null md5sum: null name: something2.raw.xz - origin: {{ src_dir }}/tests/staged/data/simple_cloud - public_image: null - recommended_instance_type: null - region: null + origin: {{ src_dir }}/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build1 + recommended_sizes: [] release: null - release_notes: null - root_device: null - scanning_port: null - security_groups: [] + sas_uri: null sha256sum: null signing_key: null - src: {{ src_dir }}/tests/staged/data/simple_cloud/build2/CLOUD_IMAGES/something2.raw.xz - sriov_net_support: null + sku_id: null + src: {{ src_dir }}/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build1/something2.raw.xz state: PENDING - type: AMI - uefi_support: null - usage_instructions: null - user_name: null - version_title: null - virtualization: null - volume: null + support_legacy: false diff --git a/tests/staged/data/simple_cloud/build1/CLOUD_IMAGES/sample1.raw.xz b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/bad_path/badpath1.raw.xz similarity index 100% rename from tests/staged/data/simple_cloud/build1/CLOUD_IMAGES/sample1.raw.xz rename to tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/bad_path/badpath1.raw.xz diff --git a/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/bad_path/resources.yaml b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/bad_path/resources.yaml new file mode 100644 index 00000000..1559593a --- /dev/null +++ b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/bad_path/resources.yaml @@ -0,0 +1,12 @@ +api: v1 +resource: CloudImage +images: + - path: "/etc/home/badpath1.raw.xz" + architecture: x86_64 +build: + name: rhel-ec2 + version: "9.4" + respin: "1" +description: "build1 sample" +boot_mode: hybrid +type: AMI diff --git a/tests/staged/data/simple_cloud/build1/CLOUD_IMAGES/resources.yaml b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build1/resources.yaml similarity index 96% rename from tests/staged/data/simple_cloud/build1/CLOUD_IMAGES/resources.yaml rename to tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build1/resources.yaml index f7f8398c..3e0e6992 100644 --- a/tests/staged/data/simple_cloud/build1/CLOUD_IMAGES/resources.yaml +++ b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build1/resources.yaml @@ -11,4 +11,4 @@ build: respin: "1" description: "build1 sample" boot_mode: hybrid -type: AMI +type: VHD diff --git a/tests/staged/data/simple_cloud/build1/CLOUD_IMAGES/sample2.raw.xz b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build1/sample1.vhd.xz similarity index 100% rename from tests/staged/data/simple_cloud/build1/CLOUD_IMAGES/sample2.raw.xz rename to tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build1/sample1.vhd.xz diff --git a/tests/staged/data/simple_cloud/build2/CLOUD_IMAGES/sample1.raw.xz b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build1/sample2.vhd.xz similarity index 100% rename from tests/staged/data/simple_cloud/build2/CLOUD_IMAGES/sample1.raw.xz rename to tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build1/sample2.vhd.xz diff --git a/tests/staged/data/simple_cloud/build2/CLOUD_IMAGES/resources.yaml b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2/resources.yaml similarity index 100% rename from tests/staged/data/simple_cloud/build2/CLOUD_IMAGES/resources.yaml rename to tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2/resources.yaml diff --git a/tests/staged/data/simple_cloud/build2/CLOUD_IMAGES/sample2.raw.xz b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2/sample1.raw.xz similarity index 100% rename from tests/staged/data/simple_cloud/build2/CLOUD_IMAGES/sample2.raw.xz rename to tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2/sample1.raw.xz diff --git a/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2/sample2.raw.xz b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build2/sample2.raw.xz new file mode 100644 index 00000000..e69de29b diff --git a/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build3/resources.yaml b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build3/resources.yaml new file mode 100644 index 00000000..85c417e9 --- /dev/null +++ b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build3/resources.yaml @@ -0,0 +1,22 @@ +api: v1 +resource: CloudImage +images: + - path: something1.raw.xz + architecture: x86_64 +build: + name: rhel-ec2 + version: "9.4" + respin: "1" +release: + arch: x86_64 + base_product: null + base_version: null + date: "20200511" + product: Fake-Product + respin: 1 + type: ga + variant: Fake-Variant + version: Fake-Version +description: "build3 sample" +boot_mode: hybrid +type: AMI \ No newline at end of file diff --git a/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build3/sample1.raw.xz b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/build3/sample1.raw.xz new file mode 100644 index 00000000..e69de29b diff --git a/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/empty_resource/resources.yaml b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/empty_resource/resources.yaml new file mode 100644 index 00000000..e69de29b diff --git a/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/no_resources_file/sample1.raw.xz b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/no_resources_file/sample1.raw.xz new file mode 100644 index 00000000..e69de29b diff --git a/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/staged.yaml b/tests/staged/data/simple_cloud/starmap/CLOUD_IMAGES/staged.yaml new file mode 100644 index 00000000..e69de29b