Fixes asyncio to generate threads for thread locked functions

Author: jajreidy

src/pubtools/_marketplacesvm/task.py

@@ -13,7 +13,7 @@
 from .utils import BuildIdBorg
 
 LOG = logging.getLogger("pubtools.marketplacesvm")
-LOG_FORMAT = "%(asctime)s [%(levelname)-8s] %(message)s"
+LOG_FORMAT = "%(asctime)s [%(levelname)-8s][%(threadName)s] %(message)s"
 
 
 RUN_RESULT = namedtuple('RUN_RESULT', ['success', 'skipped', 'collected_result'])

src/pubtools/_marketplacesvm/tasks/push/command.py

@@ -276,11 +276,14 @@ async def push_function(mapped_item, marketplace, starmap_query) -> Dict[str, An
 
             # Associate image with Product/Offer/Plan and publish only if it's not a pre-push
             if pi.state != State.UPLOADFAILED and not self.args.pre_push:
-                pi = self._publish(
-                    marketplace,
-                    pi,
-                    pre_push=False,
-                )
+                kwargs = {
+                    "marketplace": marketplace,
+                    "push_item": pi,
+                    "pre_push": False
+                }
+                coro = asyncio.to_thread(self._publish, **kwargs)
+                task = asyncio.create_task(coro)
+                pi = await task
             elif pi.state != State.UPLOADFAILED and self.args.pre_push:
                 # Set the state as PUSHED when the operation is nochannel
                 pi = evolve(pi, state=State.PUSHED)

tests/logs/push/test_push/test_push_item_fail_publish.txt

@@ -1,8 +1,6 @@
-[    INFO] Loading items from koji:https://fakekoji.com?vmi_build=ami_build,azure_build
+[    INFO] Loading items from koji:https://fakekoji.com?vmi_build=ami_build
 [    INFO] Retrieving the mappings for ami_pushitem from https://starmap-example.com
-[    INFO] starmap query returned for ami_pushitem : {"name": "test-build", "version": "7.0", "query_response": {"meta": null, "name": "sample-product", "billing_code_config": {}, "cloud": "aws", "workflow": "stratosphere", "mappings": {"aws-na": {"meta": {}, "destinations": [{"meta": {"tag1": "aws-na-value1", "tag2": "aws-na-value2"}, "id": null, "architecture": "x86_64", "destination": "ffffffff-ffff-ffff-ffff-ffffffffffff", "overwrite": true, "restrict_version": false, "restrict_major": null, "restrict_minor": null, "ami_version_template": "{major}.{minor}.{patch}", "provider": null, "tags": {"key1": "value1", "key2": "value2"}}], "provider": null}, "aws-emea": {"meta": {}, "destinations": [{"meta": {"tag1": "aws-emea-value1", "tag2": "aws-emea-value2"}, "id": null, "architecture": "x86_64", "destination": "00000000-0000-0000-0000-000000000000", "overwrite": true, "restrict_version": false, "restrict_major": null, "restrict_minor": null, "ami_version_template": null, "provider": null, "tags": {"key3": "value3", "key4": "value4"}}], "provider": null}}}}
-[    INFO] Retrieving the mappings for vhd_pushitem from https://starmap-example.com
-[    INFO] starmap query returned for vhd_pushitem : {"name": "test-build", "version": "7.0", "query_response": {"meta": null, "name": "sample-product", "billing_code_config": {}, "cloud": "azure", "workflow": "stratosphere", "mappings": {"azure-na": {"meta": {}, "destinations": [{"meta": {"tag1": "value1", "tag2": "value2"}, "id": null, "architecture": "x86_64", "destination": "destination_offer_main/plan1", "overwrite": true, "restrict_version": false, "restrict_major": null, "restrict_minor": null, "ami_version_template": null, "provider": null, "tags": {"key1": "value1", "key2": "value2"}}, {"meta": {"tag3": "value3", "tag4": "value5"}, "id": null, "architecture": "x86_64", "destination": "destination_offer_main/plan2", "overwrite": false, "restrict_version": false, "restrict_major": null, "restrict_minor": null, "ami_version_template": null, "provider": null, "tags": null}, {"meta": {}, "id": null, "architecture": "x86_64", "destination": "destination_offer_main/plan3", "overwrite": false, "restrict_version": false, "restrict_major": null, "restrict_minor": null, "ami_version_template": null, "provider": null, "tags": null}], "provider": null}}}}
+[    INFO] starmap query returned for ami_pushitem : {"name": "test-build", "version": "7.0", "query_response": {"meta": null, "name": "sample-product", "billing_code_config": {}, "cloud": "aws", "workflow": "stratosphere", "mappings": {"aws-na": {"meta": {}, "destinations": [{"meta": {"tag1": "aws-na-value1", "tag2": "aws-na-value2"}, "id": null, "architecture": "x86_64", "destination": "ffffffff-ffff-ffff-ffff-ffffffffffff", "overwrite": true, "restrict_version": false, "restrict_major": null, "restrict_minor": null, "ami_version_template": "{major}.{minor}.{patch}", "provider": null, "tags": {"key1": "value1", "key2": "value2"}}], "provider": null}}}}
 [ WARNING] Missing information for the attribute ami_pushitem.src, leaving it unset.
 [ WARNING] Missing information for the attribute ami_pushitem.build, leaving it unset.
 [ WARNING] Missing information for the attribute ami_pushitem.description, leaving it unset.
@@ -32,39 +30,10 @@
 [ WARNING] Missing information for the attribute ami_pushitem.access_endpoint_url, leaving it unset.
 [    INFO] Uploading the item ami_pushitem to AWS-NA.
 [    INFO] Upload finished for ami_pushitem on AWS-NA
-[    INFO] Uploading the item ami_pushitem to AWS-EMEA.
-[    INFO] Upload finished for ami_pushitem on AWS-EMEA
-[ WARNING] Missing information for the attribute vhd_pushitem.src, leaving it unset.
-[ WARNING] Missing information for the attribute vhd_pushitem.build, leaving it unset.
-[ WARNING] Missing information for the attribute vhd_pushitem.description, leaving it unset.
-[ WARNING] Missing information for the attribute vhd_pushitem.boot_mode, leaving it unset.
-[ WARNING] Missing information for the attribute vhd_pushitem.cloud_info, leaving it unset.
-[ WARNING] Missing information for the attribute vhd_pushitem.marketplace_title_template, leaving it unset.
-[ WARNING] Missing information for the attribute vhd_pushitem.marketplace_name, leaving it unset.
-[ WARNING] Missing information for the attribute vhd_pushitem.sku_id, leaving it unset.
-[ WARNING] Missing information for the attribute vhd_pushitem.support_legacy, leaving it unset.
-[ WARNING] Missing information for the attribute vhd_pushitem.legacy_sku_id, leaving it unset.
-[ WARNING] Missing information for the attribute vhd_pushitem.disk_version, leaving it unset.
-[ WARNING] Missing information for the attribute vhd_pushitem.recommended_sizes, leaving it unset.
-[ WARNING] Missing information for the attribute vhd_pushitem.sas_uri, leaving it unset.
-[    INFO] Uploading the item vhd_pushitem to AZURE-NA.
-[    INFO] Upload finished for vhd_pushitem on AZURE-NA
 [    INFO] Preparing to publish the item ami_pushitem to ffffffff-ffff-ffff-ffff-ffffffffffff on AWS-NA.
 [    INFO] Preparation complete for item ami_pushitem to AWS-NA.
-[    INFO] Preparing to publish the item ami_pushitem to 00000000-0000-0000-0000-000000000000 on AWS-EMEA.
-[    INFO] Preparation complete for item ami_pushitem to AWS-EMEA.
-[    INFO] Preparing to publish the item vhd_pushitem to destination_offer_main/plan1 on AZURE-NA.
-[    INFO] Preparation complete for item vhd_pushitem to AZURE-NA.
-[    INFO] Preparing to publish the item vhd_pushitem to destination_offer_main/plan2 on AZURE-NA.
-[    INFO] Preparation complete for item vhd_pushitem to AZURE-NA.
-[    INFO] Preparing to publish the item vhd_pushitem to destination_offer_main/plan3 on AZURE-NA.
-[    INFO] Preparation complete for item vhd_pushitem to AZURE-NA.
 [    INFO] Pushing "ami_pushitem" (pre-push=False) to ffffffff-ffff-ffff-ffff-ffffffffffff on AWS-NA.
 [   ERROR] Failed to publish ami_pushitem on AWS-NA: Random exception
-[    INFO] Pushing "ami_pushitem" (pre-push=False) to 00000000-0000-0000-0000-000000000000 on AWS-EMEA.
-[   ERROR] Failed to publish ami_pushitem on AWS-EMEA: Random exception
-[    INFO] Pushing "vhd_pushitem" (pre-push=False) to destination_offer_main/plan1 on AZURE-NA.
-[   ERROR] Failed to publish vhd_pushitem on AZURE-NA: Random exception
 [    INFO] Collecting results
 [   ERROR] Marketplace VM push failed
 # Raised: 30

tests/push/conftest.py

@@ -98,6 +98,30 @@ def starmap_response_aws() -> Dict[str, Any]:
     }
 
 
+@pytest.fixture
+def starmap_response_aws_1m() -> Dict[str, Any]:
+    return {
+        "mappings": {
+            "aws-na": {
+                "destinations": [
+                    {
+                        "architecture": "x86_64",
+                        "destination": "ffffffff-ffff-ffff-ffff-ffffffffffff",
+                        "overwrite": True,
+                        "restrict_version": False,
+                        "meta": {"tag1": "aws-na-value1", "tag2": "aws-na-value2"},
+                        "tags": {"key1": "value1", "key2": "value2"},
+                        "ami_version_template": "{major}.{minor}.{patch}",
+                    }
+                ]
+            },
+        },
+        "name": "sample-product",
+        "workflow": "stratosphere",
+        "cloud": "aws",
+    }
+
+
 @pytest.fixture
 def starmap_response_azure() -> Dict[str, Any]:
     return {
@@ -139,6 +163,11 @@ def starmap_query_aws(starmap_response_aws: Dict[str, Any]) -> QueryResponseEnti
     return QueryResponseEntity.from_json(starmap_response_aws)
 
 
+@pytest.fixture
+def starmap_query_aws_1m(starmap_response_aws_1m: Dict[str, Any]) -> QueryResponseEntity:
+    return QueryResponseEntity.from_json(starmap_response_aws_1m)
+
+
 @pytest.fixture
 def starmap_query_azure(starmap_response_azure: Dict[str, Any]) -> QueryResponseEntity:
     return QueryResponseEntity.from_json(starmap_response_azure)

tests/push/test_push.py

@@ -45,6 +45,15 @@ def fake_source(
         yield m
 
 
+@pytest.fixture()
+def fake_source_1m(
+    ami_push_item: AmiPushItem, vhd_push_item: VHDPushItem
+) -> Generator[mock.MagicMock, None, None]:
+    with mock.patch("pubtools._marketplacesvm.tasks.push.command.Source") as m:
+        m.get.return_value.__enter__.return_value = [ami_push_item]
+        yield m
+
+
 @pytest.fixture()
 def fake_starmap(
     starmap_query_aws: QueryResponseEntity, starmap_query_azure: QueryResponseEntity
@@ -57,6 +66,18 @@ def fake_starmap(
         yield m
 
 
+@pytest.fixture()
+def fake_starmap_1m(
+    starmap_query_aws_1m: QueryResponseEntity, starmap_query_azure: QueryResponseEntity
+) -> Generator[mock.MagicMock, None, None]:
+    with mock.patch("pubtools._marketplacesvm.tasks.push.MarketplacesVMPush.starmap") as m:
+        m.query_image_by_name.side_effect = [
+            QueryResponseContainer([x]) for x in [starmap_query_aws_1m]
+        ]
+
+        yield m
+
+
 @pytest.fixture()
 def fake_cloud_instance() -> Generator[mock.MagicMock, None, None]:
     with mock.patch("pubtools._marketplacesvm.tasks.push.MarketplacesVMPush.cloud_instance") as m:
@@ -455,8 +476,8 @@ def test_push_item_fail_upload(
 @mock.patch("pubtools._marketplacesvm.tasks.push.MarketplacesVMPush.cloud_instance")
 def test_push_item_fail_publish(
     mock_cloud_instance: mock.MagicMock,
-    fake_source: mock.MagicMock,
-    fake_starmap: mock.MagicMock,
+    fake_source_1m: mock.MagicMock,
+    fake_starmap_1m: mock.MagicMock,
     ami_push_item: AmiPushItem,
     vhd_push_item: VHDPushItem,
     command_tester: CommandTester,
@@ -478,14 +499,14 @@ def _publish(self, push_item, nochannel, overwrite, **kwargs):
             "--credentials",
             "eyJtYXJrZXRwbGFjZV9hY2NvdW50IjogInRlc3QtbmEiLCAiYXV0aCI6eyJmb28iOiJiYXIifQo=",
             "--debug",
-            "koji:https://fakekoji.com?vmi_build=ami_build,azure_build",
+            "koji:https://fakekoji.com?vmi_build=ami_build",
         ],
     )
-    starmap_calls = [mock.call(name="test-build", version="7.0") for _ in range(2)]
-    fake_starmap.query_image_by_name.assert_has_calls(starmap_calls)
+    starmap_calls = [mock.call(name="test-build", version="7.0") for _ in range(1)]
+    fake_starmap_1m.query_image_by_name.assert_has_calls(starmap_calls)
     # get_provider, upload calls for "aws-na", "aws-emea", "azure-na" with
     # publish calls only for "aws-na" and "azure-na"
-    assert mock_cloud_instance.call_count == 11
+    assert mock_cloud_instance.call_count == 3
 
 
 @mock.patch("pubtools._marketplacesvm.tasks.push.command.Source")