Merge pull request #627 from jajreidy/add-staged-cloud

Adds staged file structure for cloud images.

docs/schema/cloud.rst

@@ -0,0 +1,19 @@
+.. _cloud_schema:
+
+Schema: cloud
+==============
+
+This document shows the schema of cloud build YAML files
+supported by this library, in `JSON schema`_ format.
+These files may be used within the ``CLOUD_IMAGES`` directory within
+a :ref:`staging_structure`.
+
+The latest version of this schema is available in raw form at
+https://release-engineering.github.io/pushsource/cloud-schema.yaml.
+
+
+.. include:: ../../src/pushsource/_impl/schema/cloud-schema.yaml
+    :code: yaml
+
+
+.. _JSON schema: https://json-schema.org/

docs/sources/staged.rst

@@ -55,6 +55,7 @@ Here is a brief overview of the structure of a staging directory:
   root/destination/MODULEMD/*
   root/destination/RPMS/*.rpm
   root/destination/AWS_IMAGES/*
+  root/destination/CLOUD_IMAGES/*
   root/destination/RAW/*
 
 The staging directory consists of:
@@ -196,6 +197,21 @@ Files in this directory must have metadata included in ``staged.yaml``.
 
 Will yield instances of :class:`~pushsource.AmiPushItem`.
 
+root/destination/CLOUD_IMAGES/\*
+................................
+
+Each directory within ``CLOUD_IMAGES`` should contain one or more VMI(s) plus a
+``resources.yaml`` .
+
+The ``resources.yaml`` contains all the information needed for the
+images in that folder.
+
+:ref:`cloud_schema` provides a complete reference of the fields which can be set by this
+file.
+
+Will yield instances of either :class:`~pushsource.AmiPushItem` or
+:class:`~pushsource.VHDPushItem`.
+
 root/destination/RAW/\*
 .......................
 

src/pushsource/_impl/backend/staged/staged_base.py

@@ -10,11 +10,12 @@ class TypeHandler(object):
     # Decorator for handling specific file directories (e.g. "FILES", "ISOS" etc)
     HANDLERS = {}
 
-    def __init__(self, type_name):
+    def __init__(self, type_name, accepts=lambda entry: entry.is_file()):
         self.type_name = type_name
+        self.accepts = accepts
 
     def __call__(self, fn):
-        TypeHandler.HANDLERS[self.type_name] = fn
+        TypeHandler.HANDLERS[self.type_name] = (fn, self.accepts)
         return fn
 
 
@@ -29,19 +30,19 @@ def __init__(self, *args, **kwargs):
         super(StagedBaseMixin, self).__init__(*args, **kwargs)
         self._FILE_TYPES = self._FILE_TYPES.copy()
         for typename in TypeHandler.HANDLERS:
-            fn = TypeHandler.HANDLERS[typename]
+            fn, accepts = TypeHandler.HANDLERS[typename]
             bound_fn = partial(fn, self)
             self._FILE_TYPES[typename] = partial(
-                self.__mixin_push_items, delegate=bound_fn
+                self.__mixin_push_items, delegate=bound_fn, accepts=accepts
             )
 
-    def __mixin_push_items(self, leafdir, metadata, delegate):
+    def __mixin_push_items(self, leafdir, metadata, delegate, accepts):
         out = []
 
         LOG.debug("Looking for files in %s", leafdir)
 
         for entry in scandir(leafdir.path):
-            if entry.is_file():
+            if accepts(entry):
                 item = delegate(leafdir, metadata, entry)
                 if item:
                     out.append(item)

src/pushsource/_impl/backend/staged/staged_cloud.py

@@ -0,0 +1,161 @@
+import logging
+import os
+import yaml
+
+from datetime import datetime, timezone
+
+from .staged_base import StagedBaseMixin, handles_type
+from ...model import (
+    VHDPushItem,
+    VMIRelease,
+    AmiPushItem,
+    AmiRelease,
+    BootMode,
+    KojiBuildInfo,
+)
+
+LOG = logging.getLogger("pushsource")
+
+
+class StagedCloudMixin(StagedBaseMixin):
+    def __get_product_name(self, base_name):
+        splitted_name = base_name.split("-")
+        if len(splitted_name) > 1:
+            product = "-".join(splitted_name[:-1])
+        else:
+            product = splitted_name[0]
+        return product
+
+    def __build_ami_push_item(self, resources, origin, image, dest):
+        build_resources = resources.get("build")
+        release_resources = resources.get("release") or {}
+        name = image.get("path")
+        src = os.path.join(origin, name)
+        build_info = KojiBuildInfo(
+            name=build_resources.get("name"),
+            version=build_resources.get("version"),
+            release=build_resources.get("respin"),
+        )
+        image_kwargs = {
+            "name": name,
+            "src": src,
+            "build_info": build_info,
+            "origin": origin,
+            "dest": [dest],
+            "sha256sum": image.get("sha256sum"),
+        }
+
+        image_kwargs.update(
+            {
+                "boot_mode": (
+                    BootMode(resources.get("boot_mode"))
+                    if resources.get("boot_mode")
+                    else None
+                )
+            }
+        )
+
+        release_kwargs = {
+            "product": self.__get_product_name(build_resources.get("name")),
+            "date": datetime.now(timezone.utc).strftime("%Y%m%d"),
+            "arch": image.get("architecture"),
+            "respin": int(build_resources.get("respin")) or 0,
+        }
+        release_attrs = [
+            "version",
+            "base_product",
+            "base_version",
+            "variant",
+            "type",
+        ]
+        for key in release_attrs:
+            release_kwargs[key] = release_resources.get(key)
+
+        image_kwargs["release"] = AmiRelease(**release_kwargs)
+
+        image_attrs = [
+            "type",
+            "region",
+            "virtualization",
+            "volume",
+            "root_device",
+            "description",
+            "sriov_net_support",
+            "ena_support",
+            "uefi_support",
+            "public_image",
+            "release_notes",
+            "usage_instructions",
+            "recommended_instance_type",
+            "marketplace_entity_type",
+            "image_id",
+            "scanning_port",
+            "user_name",
+            "version_title",
+            "security_groups",
+            "access_endpoint_url",
+        ]
+
+        for key in image_attrs:
+            if key in resources:
+                image_kwargs[key] = resources.get(key)
+
+        return AmiPushItem(**image_kwargs)
+
+    def __build_azure_push_item(self, resources, origin, image, dest):
+        build_resources = resources.get("build")
+        name = image.get("path")
+        src = os.path.join(origin, name)
+        build_info = KojiBuildInfo(
+            name=build_resources.get("name"),
+            version=build_resources.get("version"),
+            release=build_resources.get("respin"),
+        )
+
+        release_kwargs = {
+            "product": self.__get_product_name(build_resources.get("name")),
+            "date": datetime.now(timezone.utc).strftime("%Y%m%d"),
+            "arch": image.get("architecture"),
+            "respin": int(build_resources.get("respin")) or 0,
+        }
+
+        image_kwargs = {
+            "name": name,
+            "src": src,
+            "description": resources.get("description"),
+            "build_info": build_info,
+            "origin": origin,
+            "dest": [dest],
+            "sha256sum": image.get("sha256sum"),
+            "release": VMIRelease(**release_kwargs),
+        }
+        return VHDPushItem(**image_kwargs)
+
+    @handles_type(
+        "CLOUD_IMAGES",
+        accepts=lambda entry: entry.is_dir()
+        and os.path.exists(os.path.join(entry.path, "resources.yaml")),
+    )
+    def __cloud_push_item(self, leafdir, _, entry):
+        yaml_path = os.path.join(entry.path, "resources.yaml")
+        with open(yaml_path, "rt") as fh:
+            raw = yaml.safe_load(fh)
+        if not raw:
+            LOG.warning("Resources.yaml file at %s is empty (ignored)", yaml_path)
+            return
+        image_type = raw.get("type") or ""
+        images_info = raw.get("images") or []
+        out = []
+        for image in images_info:
+            if "/" in image.get("path"):
+                LOG.warning("Unexpected '/' in %s (ignored)", image.get("path"))
+                return
+            if image_type == "AMI":
+                out.append(
+                    self.__build_ami_push_item(raw, entry.path, image, leafdir.dest)
+                )
+            elif image_type == "VHD":
+                out.append(
+                    self.__build_azure_push_item(raw, entry.path, image, leafdir.dest)
+                )
+        return out

src/pushsource/_impl/backend/staged/staged_source.py

@@ -16,6 +16,7 @@
 
 from .staged_utils import StagingMetadata, StagingLeafDir
 from .staged_ami import StagedAmiMixin
+from .staged_cloud import StagedCloudMixin
 from .staged_files import StagedFilesMixin
 from .staged_errata import StagedErrataMixin
 from .staged_compsxml import StagedCompsXmlMixin
@@ -33,6 +34,7 @@
 class StagedSource(
     Source,
     StagedAmiMixin,
+    StagedCloudMixin,
     StagedFilesMixin,
     StagedErrataMixin,
     StagedCompsXmlMixin,
@@ -179,7 +181,11 @@ def _push_items_for_topdir(self, topdir):
         )
         for f in completed_fs:
             for pushitem in f.result():
-                yield pushitem
+                if isinstance(pushitem, list):
+                    for p in pushitem:
+                        yield p
+                else:
+                    yield pushitem
 
 
 Source.register_backend("staged", StagedSource)