Merge pull request #10 from release-engineering/azure_verify_submission_state

Azure: Better handle "publish_preview" and "publish_live"

Author: JAVGan

cloudpub/ms_azure/service.py

@@ -7,7 +7,7 @@
 from requests import HTTPError
 from tenacity import retry
 from tenacity.retry import retry_if_result
-from tenacity.stop import stop_after_delay
+from tenacity.stop import stop_after_attempt, stop_after_delay
 from tenacity.wait import wait_chain, wait_fixed
 
 from cloudpub.common import BaseService
@@ -511,6 +511,11 @@ def _is_submission_in_preview(self, current: ProductSubmission) -> bool:
             return current.id != live.id  # If they're the same then state == live
         return True  # when no live it means it's in preview
 
+    @retry(
+        wait=wait_fixed(wait=60),
+        stop=stop_after_attempt(3),
+        reraise=True,
+    )
     def _publish_preview(self, product: Product, product_name: str) -> None:
         """
         Submit the product to 'preview'  if it's not already in this state.
@@ -536,8 +541,23 @@ def _publish_preview(self, product: Product, product_name: str) -> None:
             log.info(
                 "Submitting the product \"%s (%s)\" to \"preview\"." % (product_name, product.id)
             )
-            self.submit_to_status(product_id=product.id, status='preview')
+            res = self.submit_to_status(product_id=product.id, status='preview')
+
+            if res.job_result != 'succeeded' or not self.get_submission_state(
+                product.id, state="preview"
+            ):
+                errors = "\n".join(res.errors)
+                failure_msg = (
+                    f"Failed to submit the product {product.id} to preview. "
+                    f"Status: {res.job_result} Errors: {errors}"
+                )
+                raise RuntimeError(failure_msg)
 
+    @retry(
+        wait=wait_fixed(wait=60),
+        stop=stop_after_attempt(3),
+        reraise=True,
+    )
     def _publish_live(self, product: Product, product_name: str) -> None:
         """
         Submit the product to 'live' after going through Azure Marketplace Validation.
@@ -551,7 +571,15 @@ def _publish_live(self, product: Product, product_name: str) -> None:
         # Note: the offer can only go `live` after successfully being changed to `preview`
         # which takes up to 4 days.
         log.info("Submitting the product \"%s (%s)\" to \"live\"." % (product_name, product.id))
-        self.submit_to_status(product_id=product.id, status='live')
+        res = self.submit_to_status(product_id=product.id, status='live')
+
+        if res.job_result != 'succeeded' or not self.get_submission_state(product.id, state="live"):
+            errors = "\n".join(res.errors)
+            failure_msg = (
+                f"Failed to submit the product {product.id} to live. "
+                f"Status: {res.job_result} Errors: {errors}"
+            )
+            raise RuntimeError(failure_msg)
 
     def publish(self, metadata: AzurePublishingMetadata) -> None:
         """

tests/ms_azure/test_service.py

@@ -833,6 +833,134 @@ def test_ensure_can_publish_raises(
         with pytest.raises(RuntimeError, match=err):
             azure_service.ensure_can_publish("ffffffff-ffff-ffff-ffff-ffffffffffff")
 
+    @mock.patch("cloudpub.ms_azure.AzureService.get_submission_state")
+    @mock.patch("cloudpub.ms_azure.AzureService.submit_to_status")
+    @mock.patch("cloudpub.ms_azure.AzureService._is_submission_in_preview")
+    def test_publish_preview_success_on_retry(
+        self,
+        mock_is_sbpreview: mock.MagicMock,
+        mock_subst: mock.MagicMock,
+        mock_getsubst: mock.MagicMock,
+        product_obj: Product,
+        azure_service: AzureService,
+    ) -> None:
+        # Prepare mocks
+        mock_config_res = mock.MagicMock()
+        mock_config_res.job_result = "succeeded"
+        mock_is_sbpreview.return_value = False
+        mock_subst.side_effect = [mock_config_res for _ in range(3)]
+        mock_getsubst.side_effect = [
+            None,  # Fail on 1st call
+            None,
+            mock.MagicMock(),  # Success on 3rd call
+        ]
+        # Remove the retry sleep
+        azure_service._publish_preview.retry.sleep = mock.Mock()  # type: ignore
+
+        # Test
+        azure_service._publish_preview(product_obj, "test-product")
+
+        mock_subst.assert_has_calls(
+            [mock.call(product_id=product_obj.id, status="preview") for _ in range(3)]
+        )
+        mock_getsubst.assert_has_calls(
+            [mock.call(product_obj.id, state="preview") for _ in range(3)]
+        )
+
+    @mock.patch("cloudpub.ms_azure.AzureService.get_submission_state")
+    @mock.patch("cloudpub.ms_azure.AzureService.submit_to_status")
+    @mock.patch("cloudpub.ms_azure.AzureService._is_submission_in_preview")
+    def test_publish_preview_fail_on_retry(
+        self,
+        mock_is_sbpreview: mock.MagicMock,
+        mock_subst: mock.MagicMock,
+        mock_getsubst: mock.MagicMock,
+        product_obj: Product,
+        azure_service: AzureService,
+    ) -> None:
+        # Prepare mocks
+        err_resp = ConfigureStatus.from_json(
+            {
+                "jobId": "1",
+                "jobStatus": "completed",
+                "jobResult": "failed",
+                "errors": ["failure1", "failure2"],
+            }
+        )
+        mock_is_sbpreview.return_value = False
+        mock_subst.side_effect = [err_resp for _ in range(3)]
+        mock_getsubst.side_effect = [None for _ in range(3)]
+        # Remove the retry sleep
+        azure_service._publish_preview.retry.sleep = mock.Mock()  # type: ignore
+        expected_err = (
+            f"Failed to submit the product {product_obj.id} to preview. "
+            "Status: failed Errors: failure1\nfailure2"
+        )
+
+        # Test
+        with pytest.raises(RuntimeError, match=expected_err):
+            azure_service._publish_preview(product_obj, "test-product")
+
+    @mock.patch("cloudpub.ms_azure.AzureService.get_submission_state")
+    @mock.patch("cloudpub.ms_azure.AzureService.submit_to_status")
+    def test_publish_live_success_on_retry(
+        self,
+        mock_subst: mock.MagicMock,
+        mock_getsubst: mock.MagicMock,
+        product_obj: Product,
+        azure_service: AzureService,
+    ) -> None:
+        # Prepare mocks
+        mock_config_res = mock.MagicMock()
+        mock_config_res.job_result = "succeeded"
+        mock_subst.side_effect = [mock_config_res for _ in range(3)]
+        mock_getsubst.side_effect = [
+            None,  # Fail on 1st call
+            None,
+            mock.MagicMock(),  # Success on 3rd call
+        ]
+        # Remove the retry sleep
+        azure_service._publish_live.retry.sleep = mock.Mock()  # type: ignore
+
+        # Test
+        azure_service._publish_live(product_obj, "test-product")
+
+        mock_subst.assert_has_calls(
+            [mock.call(product_id=product_obj.id, status="live") for _ in range(3)]
+        )
+        mock_getsubst.assert_has_calls([mock.call(product_obj.id, state="live") for _ in range(3)])
+
+    @mock.patch("cloudpub.ms_azure.AzureService.get_submission_state")
+    @mock.patch("cloudpub.ms_azure.AzureService.submit_to_status")
+    def test_publish_live_fail_on_retry(
+        self,
+        mock_subst: mock.MagicMock,
+        mock_getsubst: mock.MagicMock,
+        product_obj: Product,
+        azure_service: AzureService,
+    ) -> None:
+        # Prepare mocks
+        err_resp = ConfigureStatus.from_json(
+            {
+                "jobId": "1",
+                "jobStatus": "completed",
+                "jobResult": "failed",
+                "errors": ["failure1", "failure2"],
+            }
+        )
+        mock_subst.side_effect = [err_resp for _ in range(3)]
+        mock_getsubst.side_effect = [None for _ in range(3)]
+        # Remove the retry sleep
+        azure_service._publish_live.retry.sleep = mock.Mock()  # type: ignore
+        expected_err = (
+            f"Failed to submit the product {product_obj.id} to live. "
+            "Status: failed Errors: failure1\nfailure2"
+        )
+
+        # Test
+        with pytest.raises(RuntimeError, match=expected_err):
+            azure_service._publish_live(product_obj, "test-product")
+
     @mock.patch("cloudpub.ms_azure.AzureService.configure")
     @mock.patch("cloudpub.ms_azure.AzureService.submit_to_status")
     @mock.patch("cloudpub.ms_azure.service.update_skus")
@@ -1184,6 +1312,7 @@ def test_is_submission_in_preview(
             mock_substt.assert_called_once_with(current.product_id, "live")
 
     @mock.patch("cloudpub.ms_azure.AzureService.ensure_can_publish")
+    @mock.patch("cloudpub.ms_azure.AzureService.get_submission_state")
     @mock.patch("cloudpub.ms_azure.AzureService.diff_offer")
     @mock.patch("cloudpub.ms_azure.AzureService.configure")
     @mock.patch("cloudpub.ms_azure.AzureService.submit_to_status")
@@ -1202,6 +1331,7 @@ def test_publish_live(
         mock_submit: mock.MagicMock,
         mock_configure: mock.MagicMock,
         mock_diff_offer: mock.MagicMock,
+        mock_getsubst: mock.MagicMock,
         mock_ensure_publish: mock.MagicMock,
         product_obj: Product,
         plan_summary_obj: PlanSummary,
@@ -1221,6 +1351,11 @@ def test_publish_live(
             [technical_config_obj],
             [submission_obj],
         ]
+        mock_getsubst.side_effect = ["preview", "live"]
+        mock_res_preview = mock.MagicMock()
+        mock_res_live = mock.MagicMock()
+        mock_res_preview.job_result = mock_res_live.job_result = "succeeded"
+        mock_submit.side_effect = [mock_res_preview, mock_res_live]
         mock_is_sas.return_value = False
         expected_source = VMImageSource(
             source_type="sasUri",
@@ -1237,6 +1372,7 @@ def test_publish_live(
         technical_config_obj.disk_versions = [disk_version_obj]
         technical_config_obj.disk_versions = [disk_version_obj]
 
+        # Test
         azure_service.publish(metadata_azure_obj)
         mock_getprpl_name.assert_called_once_with("example-product", "plan-1")
         filter_calls = [