Adds support for clouds.json to Pub Source

src/pushsource/_impl/backend/pub_source/pub_client.py

@@ -43,19 +43,25 @@ def _session(self):
             self._tls.session = requests.Session()
         return self._tls.session
 
-    def _do_request(self, **kwargs):
-        return self._session.request(**kwargs)
+    def _do_request(self, url, **kwargs):
+        kwargs["url"] = url.format(filename="images.json")
+        resp = self._session.request(**kwargs)
+        # if 404 lets try clouds.json
+        if resp.status_code == 404:
+            kwargs["url"] = url.format(filename="clouds.json")
+            resp = self._session.request(**kwargs)
+        return resp
 
     def get_ami_json_f(self, task_id):
         """
         Returns Future[dict|list] holding json obj with AMI push items returned from Pub for given task id.
         """
         endpoint = "pub/task"
-        url_ending = "log/images.json"
+        url_ending = "log/{filename}"
         params = {"format": "raw"}
         url = os.path.join(self._url, endpoint, str(task_id), url_ending)
 
-        LOG.info("Requesting Pub service for %s of task: %s", url_ending, str(task_id))
+        LOG.info("Fetching AMI details from Pub task: %s", str(task_id))
         ft = self._executor.submit(
             self._do_request, method="GET", url=url, params=params
         )

src/pushsource/_impl/model/ami.py

@@ -224,8 +224,8 @@ def _from_data(cls, data):
             "origin": data.get("origin") or None,
             # ami push item fields
             "release": AmiRelease._from_data(data.get("release") or {}),
-            "type": data["type"],
-            "region": data["region"],
+            "type": data.get("type") or None,
+            "region": data.get("region") or None,
             "virtualization": data["virtualization"],
             "volume": data["volume"],
             "root_device": data["root_device"],
@@ -236,8 +236,10 @@ def _from_data(cls, data):
             "boot_mode": (
                 BootMode(data.get("boot_mode")) if data.get("boot_mode") else None
             ),
-            "billing_codes": AmiBillingCodes._from_data(
-                data.get("billing_codes") or {}
+            "billing_codes": (
+                AmiBillingCodes._from_data(data.get("billing_codes"))
+                if data.get("billing_codes")
+                else None
             ),
             "image_id": data.get("ami") or None,
             "public_image": data.get("public_image"),

tests/pub/data/123456/clouds.json

@@ -0,0 +1,139 @@
+[
+  {
+    "boot_mode": null,
+    "build": "rhel-sap-ec2-8.8-2116",
+    "build_info": {
+      "id": 3176889,
+      "name": "rhel-sap-ec2",
+      "release": "2116",
+      "version": "8.8"
+    },
+    "description": "Provided by Red Hat, Inc.",
+    "dest": [
+      "test-dest"
+    ],
+    "ena_support": true,
+    "image_id": "ami-test",
+    "marketplace_entity_type": "AmiProduct",
+    "md5sum": "0ef2038e599967f7074b6c52e5b2afae",
+    "name": "rhel-sap-ec2-8.8-2116.x86_64.raw.xz",
+    "recommended_instance_type": "t2.micro",
+    "release": {
+      "arch": "x86_64",
+      "base_product": null,
+      "base_version": null,
+      "date": "20240717",
+      "product": "RHEL-SAP",
+      "respin": 2116,
+      "type": null,
+      "variant": null,
+      "version": "8.8"
+    },
+    "release_notes": "Fake release notes",
+    "root_device": "/dev/sda1",
+    "scanning_port": 22,
+    "security_groups": [
+      {
+        "from_port": 22,
+        "ip_protocol": "tcp",
+        "ip_ranges": [
+          "0.0.0.0/0"
+        ],
+        "to_port": 22
+      }
+    ],
+    "src": "/test/path/packages/rhel-sap-ec2/8.8/2116/images/rhel-sap-ec2-8.8-2116.x86_64.raw.xz",
+    "sriov_net_support": "simple",
+    "starmap_query": {
+      "clouds": {
+        "aws-emea": [
+          {
+            "architecture": "x86_64",
+            "destination": "test-dest",
+            "id": null,
+            "meta": {
+              "description": "Provided by Red Hat Limited.",
+              "ena_support": true,
+              "marketplace_entity_type": "AmiProduct",
+              "recommended_instance_type": "t2.micro",
+              "release_notes": "Fake release notes",
+              "root_device": "/dev/sda1",
+              "scanning_port": 22,
+              "security_groups": [
+                {
+                  "from_port": 22,
+                  "ip_protocol": "tcp",
+                  "ip_ranges": [
+                    "0.0.0.0/0"
+                  ],
+                  "to_port": 22
+                }
+              ],
+              "sharing_accounts": [
+                "tester"
+              ],
+              "sriov_net_support": "simple",
+              "usage_instructions": "Fake use instructions",
+              "user_name": "Fake-Username",
+              "virtualization": "hvm",
+              "volume": "gp2"
+            },
+            "overwrite": false,
+            "provider": null,
+            "restrict_major": null,
+            "restrict_minor": null,
+            "restrict_version": true,
+            "tags": {}
+          }
+        ],
+        "aws-na": [
+          {
+            "architecture": "x86_64",
+            "destination": "test-dest2",
+            "id": null,
+            "meta": {
+              "description": "Provided by Red Hat, Inc.",
+              "ena_support": true,
+              "marketplace_entity_type": "AmiProduct",
+              "recommended_instance_type": "t2.micro",
+              "release_notes": "Fake release notes",
+              "root_device": "/dev/sda1",
+              "scanning_port": 22,
+              "security_groups": [
+                {
+                  "from_port": 22,
+                  "ip_protocol": "tcp",
+                  "ip_ranges": [
+                    "0.0.0.0/0"
+                  ],
+                  "to_port": 22
+                }
+              ],
+              "sharing_accounts": [
+                "tester"
+              ],
+              "sriov_net_support": "simple",
+              "usage_instructions": "Fake use instructions",
+              "user_name": "Fake-Username",
+              "virtualization": "hvm",
+              "volume": "gp2"
+            },
+            "overwrite": false,
+            "provider": null,
+            "restrict_major": null,
+            "restrict_minor": null,
+            "restrict_version": true,
+            "tags": {}
+          }
+        ]
+      },
+      "name": "rhel-sap-ec2",
+      "workflow": "stratosphere"
+    },
+    "state": "PUSHED",
+    "usage_instructions": "Fake use instructions",
+    "user_name": "Fake-Username",
+    "virtualization": "hvm",
+    "volume": "gp2"
+  }
+]