diff --git a/src/pubtools/_marketplacesvm/task.py b/src/pubtools/_marketplacesvm/task.py index 6371f3e..aca6db2 100644 --- a/src/pubtools/_marketplacesvm/task.py +++ b/src/pubtools/_marketplacesvm/task.py @@ -13,7 +13,7 @@ from .utils import BuildIdBorg LOG = logging.getLogger("pubtools.marketplacesvm") -LOG_FORMAT = "%(asctime)s [%(levelname)-8s] %(message)s" +LOG_FORMAT = "%(asctime)s [%(levelname)-8s][%(threadName)s] %(message)s" RUN_RESULT = namedtuple('RUN_RESULT', ['success', 'skipped', 'collected_result']) diff --git a/src/pubtools/_marketplacesvm/tasks/push/command.py b/src/pubtools/_marketplacesvm/tasks/push/command.py index d0a898d..746213b 100644 --- a/src/pubtools/_marketplacesvm/tasks/push/command.py +++ b/src/pubtools/_marketplacesvm/tasks/push/command.py @@ -276,11 +276,10 @@ async def push_function(mapped_item, marketplace, starmap_query) -> Dict[str, An # Associate image with Product/Offer/Plan and publish only if it's not a pre-push if pi.state != State.UPLOADFAILED and not self.args.pre_push: - pi = self._publish( - marketplace, - pi, - pre_push=False, - ) + kwargs = {"marketplace": marketplace, "push_item": pi, "pre_push": False} + coro = asyncio.to_thread(self._publish, **kwargs) + task = asyncio.create_task(coro) + pi = await task elif pi.state != State.UPLOADFAILED and self.args.pre_push: # Set the state as PUSHED when the operation is nochannel pi = evolve(pi, state=State.PUSHED) diff --git a/tests/logs/push/test_push/test_push_item_fail_publish.txt b/tests/logs/push/test_push/test_push_item_fail_publish.txt index 13bb0c5..de2b9a7 100644 --- a/tests/logs/push/test_push/test_push_item_fail_publish.txt +++ b/tests/logs/push/test_push/test_push_item_fail_publish.txt @@ -1,8 +1,6 @@ -[ INFO] Loading items from koji:https://fakekoji.com?vmi_build=ami_build,azure_build +[ INFO] Loading items from koji:https://fakekoji.com?vmi_build=ami_build [ INFO] Retrieving the mappings for ami_pushitem from https://starmap-example.com -[ INFO] starmap query returned for ami_pushitem : {"name": "test-build", "version": "7.0", "query_response": {"meta": null, "name": "sample-product", "billing_code_config": {}, "cloud": "aws", "workflow": "stratosphere", "mappings": {"aws-na": {"meta": {}, "destinations": [{"meta": {"tag1": "aws-na-value1", "tag2": "aws-na-value2"}, "id": null, "architecture": "x86_64", "destination": "ffffffff-ffff-ffff-ffff-ffffffffffff", "overwrite": true, "restrict_version": false, "restrict_major": null, "restrict_minor": null, "ami_version_template": "{major}.{minor}.{patch}", "provider": null, "tags": {"key1": "value1", "key2": "value2"}}], "provider": null}, "aws-emea": {"meta": {}, "destinations": [{"meta": {"tag1": "aws-emea-value1", "tag2": "aws-emea-value2"}, "id": null, "architecture": "x86_64", "destination": "00000000-0000-0000-0000-000000000000", "overwrite": true, "restrict_version": false, "restrict_major": null, "restrict_minor": null, "ami_version_template": null, "provider": null, "tags": {"key3": "value3", "key4": "value4"}}], "provider": null}}}} -[ INFO] Retrieving the mappings for vhd_pushitem from https://starmap-example.com -[ INFO] starmap query returned for vhd_pushitem : {"name": "test-build", "version": "7.0", "query_response": {"meta": null, "name": "sample-product", "billing_code_config": {}, "cloud": "azure", "workflow": "stratosphere", "mappings": {"azure-na": {"meta": {}, "destinations": [{"meta": {"tag1": "value1", "tag2": "value2"}, "id": null, "architecture": "x86_64", "destination": "destination_offer_main/plan1", "overwrite": true, "restrict_version": false, "restrict_major": null, "restrict_minor": null, "ami_version_template": null, "provider": null, "tags": {"key1": "value1", "key2": "value2"}}, {"meta": {"tag3": "value3", "tag4": "value5"}, "id": null, "architecture": "x86_64", "destination": "destination_offer_main/plan2", "overwrite": false, "restrict_version": false, "restrict_major": null, "restrict_minor": null, "ami_version_template": null, "provider": null, "tags": null}, {"meta": {}, "id": null, "architecture": "x86_64", "destination": "destination_offer_main/plan3", "overwrite": false, "restrict_version": false, "restrict_major": null, "restrict_minor": null, "ami_version_template": null, "provider": null, "tags": null}], "provider": null}}}} +[ INFO] starmap query returned for ami_pushitem : {"name": "test-build", "version": "7.0", "query_response": {"meta": null, "name": "sample-product", "billing_code_config": {}, "cloud": "aws", "workflow": "stratosphere", "mappings": {"aws-na": {"meta": {}, "destinations": [{"meta": {"tag1": "aws-na-value1", "tag2": "aws-na-value2"}, "id": null, "architecture": "x86_64", "destination": "ffffffff-ffff-ffff-ffff-ffffffffffff", "overwrite": true, "restrict_version": false, "restrict_major": null, "restrict_minor": null, "ami_version_template": "{major}.{minor}.{patch}", "provider": null, "tags": {"key1": "value1", "key2": "value2"}}], "provider": null}}}} [ WARNING] Missing information for the attribute ami_pushitem.src, leaving it unset. [ WARNING] Missing information for the attribute ami_pushitem.build, leaving it unset. [ WARNING] Missing information for the attribute ami_pushitem.description, leaving it unset. @@ -32,39 +30,10 @@ [ WARNING] Missing information for the attribute ami_pushitem.access_endpoint_url, leaving it unset. [ INFO] Uploading the item ami_pushitem to AWS-NA. [ INFO] Upload finished for ami_pushitem on AWS-NA -[ INFO] Uploading the item ami_pushitem to AWS-EMEA. -[ INFO] Upload finished for ami_pushitem on AWS-EMEA -[ WARNING] Missing information for the attribute vhd_pushitem.src, leaving it unset. -[ WARNING] Missing information for the attribute vhd_pushitem.build, leaving it unset. -[ WARNING] Missing information for the attribute vhd_pushitem.description, leaving it unset. -[ WARNING] Missing information for the attribute vhd_pushitem.boot_mode, leaving it unset. -[ WARNING] Missing information for the attribute vhd_pushitem.cloud_info, leaving it unset. -[ WARNING] Missing information for the attribute vhd_pushitem.marketplace_title_template, leaving it unset. -[ WARNING] Missing information for the attribute vhd_pushitem.marketplace_name, leaving it unset. -[ WARNING] Missing information for the attribute vhd_pushitem.sku_id, leaving it unset. -[ WARNING] Missing information for the attribute vhd_pushitem.support_legacy, leaving it unset. -[ WARNING] Missing information for the attribute vhd_pushitem.legacy_sku_id, leaving it unset. -[ WARNING] Missing information for the attribute vhd_pushitem.disk_version, leaving it unset. -[ WARNING] Missing information for the attribute vhd_pushitem.recommended_sizes, leaving it unset. -[ WARNING] Missing information for the attribute vhd_pushitem.sas_uri, leaving it unset. -[ INFO] Uploading the item vhd_pushitem to AZURE-NA. -[ INFO] Upload finished for vhd_pushitem on AZURE-NA [ INFO] Preparing to publish the item ami_pushitem to ffffffff-ffff-ffff-ffff-ffffffffffff on AWS-NA. [ INFO] Preparation complete for item ami_pushitem to AWS-NA. -[ INFO] Preparing to publish the item ami_pushitem to 00000000-0000-0000-0000-000000000000 on AWS-EMEA. -[ INFO] Preparation complete for item ami_pushitem to AWS-EMEA. -[ INFO] Preparing to publish the item vhd_pushitem to destination_offer_main/plan1 on AZURE-NA. -[ INFO] Preparation complete for item vhd_pushitem to AZURE-NA. -[ INFO] Preparing to publish the item vhd_pushitem to destination_offer_main/plan2 on AZURE-NA. -[ INFO] Preparation complete for item vhd_pushitem to AZURE-NA. -[ INFO] Preparing to publish the item vhd_pushitem to destination_offer_main/plan3 on AZURE-NA. -[ INFO] Preparation complete for item vhd_pushitem to AZURE-NA. [ INFO] Pushing "ami_pushitem" (pre-push=False) to ffffffff-ffff-ffff-ffff-ffffffffffff on AWS-NA. [ ERROR] Failed to publish ami_pushitem on AWS-NA: Random exception -[ INFO] Pushing "ami_pushitem" (pre-push=False) to 00000000-0000-0000-0000-000000000000 on AWS-EMEA. -[ ERROR] Failed to publish ami_pushitem on AWS-EMEA: Random exception -[ INFO] Pushing "vhd_pushitem" (pre-push=False) to destination_offer_main/plan1 on AZURE-NA. -[ ERROR] Failed to publish vhd_pushitem on AZURE-NA: Random exception [ INFO] Collecting results [ ERROR] Marketplace VM push failed # Raised: 30 diff --git a/tests/push/conftest.py b/tests/push/conftest.py index 1722857..e5bf10b 100644 --- a/tests/push/conftest.py +++ b/tests/push/conftest.py @@ -98,6 +98,30 @@ def starmap_response_aws() -> Dict[str, Any]: } +@pytest.fixture +def starmap_response_aws_1m() -> Dict[str, Any]: + return { + "mappings": { + "aws-na": { + "destinations": [ + { + "architecture": "x86_64", + "destination": "ffffffff-ffff-ffff-ffff-ffffffffffff", + "overwrite": True, + "restrict_version": False, + "meta": {"tag1": "aws-na-value1", "tag2": "aws-na-value2"}, + "tags": {"key1": "value1", "key2": "value2"}, + "ami_version_template": "{major}.{minor}.{patch}", + } + ] + }, + }, + "name": "sample-product", + "workflow": "stratosphere", + "cloud": "aws", + } + + @pytest.fixture def starmap_response_azure() -> Dict[str, Any]: return { @@ -139,6 +163,11 @@ def starmap_query_aws(starmap_response_aws: Dict[str, Any]) -> QueryResponseEnti return QueryResponseEntity.from_json(starmap_response_aws) +@pytest.fixture +def starmap_query_aws_1m(starmap_response_aws_1m: Dict[str, Any]) -> QueryResponseEntity: + return QueryResponseEntity.from_json(starmap_response_aws_1m) + + @pytest.fixture def starmap_query_azure(starmap_response_azure: Dict[str, Any]) -> QueryResponseEntity: return QueryResponseEntity.from_json(starmap_response_azure) diff --git a/tests/push/test_push.py b/tests/push/test_push.py index f9b0d44..1fe3e3f 100644 --- a/tests/push/test_push.py +++ b/tests/push/test_push.py @@ -45,6 +45,15 @@ def fake_source( yield m +@pytest.fixture() +def fake_source_1m( + ami_push_item: AmiPushItem, vhd_push_item: VHDPushItem +) -> Generator[mock.MagicMock, None, None]: + with mock.patch("pubtools._marketplacesvm.tasks.push.command.Source") as m: + m.get.return_value.__enter__.return_value = [ami_push_item] + yield m + + @pytest.fixture() def fake_starmap( starmap_query_aws: QueryResponseEntity, starmap_query_azure: QueryResponseEntity @@ -57,6 +66,18 @@ def fake_starmap( yield m +@pytest.fixture() +def fake_starmap_1m( + starmap_query_aws_1m: QueryResponseEntity, starmap_query_azure: QueryResponseEntity +) -> Generator[mock.MagicMock, None, None]: + with mock.patch("pubtools._marketplacesvm.tasks.push.MarketplacesVMPush.starmap") as m: + m.query_image_by_name.side_effect = [ + QueryResponseContainer([x]) for x in [starmap_query_aws_1m] + ] + + yield m + + @pytest.fixture() def fake_cloud_instance() -> Generator[mock.MagicMock, None, None]: with mock.patch("pubtools._marketplacesvm.tasks.push.MarketplacesVMPush.cloud_instance") as m: @@ -455,8 +476,8 @@ def test_push_item_fail_upload( @mock.patch("pubtools._marketplacesvm.tasks.push.MarketplacesVMPush.cloud_instance") def test_push_item_fail_publish( mock_cloud_instance: mock.MagicMock, - fake_source: mock.MagicMock, - fake_starmap: mock.MagicMock, + fake_source_1m: mock.MagicMock, + fake_starmap_1m: mock.MagicMock, ami_push_item: AmiPushItem, vhd_push_item: VHDPushItem, command_tester: CommandTester, @@ -478,14 +499,14 @@ def _publish(self, push_item, nochannel, overwrite, **kwargs): "--credentials", "eyJtYXJrZXRwbGFjZV9hY2NvdW50IjogInRlc3QtbmEiLCAiYXV0aCI6eyJmb28iOiJiYXIifQo=", "--debug", - "koji:https://fakekoji.com?vmi_build=ami_build,azure_build", + "koji:https://fakekoji.com?vmi_build=ami_build", ], ) - starmap_calls = [mock.call(name="test-build", version="7.0") for _ in range(2)] - fake_starmap.query_image_by_name.assert_has_calls(starmap_calls) + starmap_calls = [mock.call(name="test-build", version="7.0") for _ in range(1)] + fake_starmap_1m.query_image_by_name.assert_has_calls(starmap_calls) # get_provider, upload calls for "aws-na", "aws-emea", "azure-na" with # publish calls only for "aws-na" and "azure-na" - assert mock_cloud_instance.call_count == 11 + assert mock_cloud_instance.call_count == 3 @mock.patch("pubtools._marketplacesvm.tasks.push.command.Source")