Skip to content

Commit c3401d5

Browse files
jajreidyjajreidy
committed
Fixes asyncio to generate threads for thread locked functions
1 parent c10d559 commit c3401d5

File tree

5 files changed

+67
-45
lines changed

5 files changed

+67
-45
lines changed

‎src/pubtools/_marketplacesvm/task.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
from .utils import BuildIdBorg
1414

1515
LOG = logging.getLogger("pubtools.marketplacesvm")
16-
LOG_FORMAT = "%(asctime)s [%(levelname)-8s] %(message)s"
16+
LOG_FORMAT = "%(asctime)s [%(levelname)-8s][%(threadName)s] %(message)s"
1717

1818

1919
RUN_RESULT = namedtuple('RUN_RESULT', ['success', 'skipped', 'collected_result'])

‎src/pubtools/_marketplacesvm/tasks/push/command.py

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -276,11 +276,14 @@ async def push_function(mapped_item, marketplace, starmap_query) -> Dict[str, An
276276

277277
# Associate image with Product/Offer/Plan and publish only if it's not a pre-push
278278
if pi.state != State.UPLOADFAILED and not self.args.pre_push:
279-
pi = self._publish(
280-
marketplace,
281-
pi,
282-
pre_push=False,
283-
)
279+
kwargs = {
280+
"marketplace": marketplace,
281+
"push_item": pi,
282+
"pre_push": False
283+
}
284+
coro = asyncio.to_thread(self._publish, **kwargs)
285+
task = asyncio.create_task(coro)
286+
pi = await task
284287
elif pi.state != State.UPLOADFAILED and self.args.pre_push:
285288
# Set the state as PUSHED when the operation is nochannel
286289
pi = evolve(pi, state=State.PUSHED)

‎tests/logs/push/test_push/test_push_item_fail_publish.txt

Lines changed: 2 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
1-
[ INFO] Loading items from koji:https://fakekoji.com?vmi_build=ami_build,azure_build
1+
[ INFO] Loading items from koji:https://fakekoji.com?vmi_build=ami_build
22
[ INFO] Retrieving the mappings for ami_pushitem from https://starmap-example.com
3-
[ INFO] starmap query returned for ami_pushitem : {"name": "test-build", "version": "7.0", "query_response": {"meta": null, "name": "sample-product", "billing_code_config": {}, "cloud": "aws", "workflow": "stratosphere", "mappings": {"aws-na": {"meta": {}, "destinations": [{"meta": {"tag1": "aws-na-value1", "tag2": "aws-na-value2"}, "id": null, "architecture": "x86_64", "destination": "ffffffff-ffff-ffff-ffff-ffffffffffff", "overwrite": true, "restrict_version": false, "restrict_major": null, "restrict_minor": null, "ami_version_template": "{major}.{minor}.{patch}", "provider": null, "tags": {"key1": "value1", "key2": "value2"}}], "provider": null}, "aws-emea": {"meta": {}, "destinations": [{"meta": {"tag1": "aws-emea-value1", "tag2": "aws-emea-value2"}, "id": null, "architecture": "x86_64", "destination": "00000000-0000-0000-0000-000000000000", "overwrite": true, "restrict_version": false, "restrict_major": null, "restrict_minor": null, "ami_version_template": null, "provider": null, "tags": {"key3": "value3", "key4": "value4"}}], "provider": null}}}}
4-
[ INFO] Retrieving the mappings for vhd_pushitem from https://starmap-example.com
5-
[ INFO] starmap query returned for vhd_pushitem : {"name": "test-build", "version": "7.0", "query_response": {"meta": null, "name": "sample-product", "billing_code_config": {}, "cloud": "azure", "workflow": "stratosphere", "mappings": {"azure-na": {"meta": {}, "destinations": [{"meta": {"tag1": "value1", "tag2": "value2"}, "id": null, "architecture": "x86_64", "destination": "destination_offer_main/plan1", "overwrite": true, "restrict_version": false, "restrict_major": null, "restrict_minor": null, "ami_version_template": null, "provider": null, "tags": {"key1": "value1", "key2": "value2"}}, {"meta": {"tag3": "value3", "tag4": "value5"}, "id": null, "architecture": "x86_64", "destination": "destination_offer_main/plan2", "overwrite": false, "restrict_version": false, "restrict_major": null, "restrict_minor": null, "ami_version_template": null, "provider": null, "tags": null}, {"meta": {}, "id": null, "architecture": "x86_64", "destination": "destination_offer_main/plan3", "overwrite": false, "restrict_version": false, "restrict_major": null, "restrict_minor": null, "ami_version_template": null, "provider": null, "tags": null}], "provider": null}}}}
3+
[ INFO] starmap query returned for ami_pushitem : {"name": "test-build", "version": "7.0", "query_response": {"meta": null, "name": "sample-product", "billing_code_config": {}, "cloud": "aws", "workflow": "stratosphere", "mappings": {"aws-na": {"meta": {}, "destinations": [{"meta": {"tag1": "aws-na-value1", "tag2": "aws-na-value2"}, "id": null, "architecture": "x86_64", "destination": "ffffffff-ffff-ffff-ffff-ffffffffffff", "overwrite": true, "restrict_version": false, "restrict_major": null, "restrict_minor": null, "ami_version_template": "{major}.{minor}.{patch}", "provider": null, "tags": {"key1": "value1", "key2": "value2"}}], "provider": null}}}}
64
[ WARNING] Missing information for the attribute ami_pushitem.src, leaving it unset.
75
[ WARNING] Missing information for the attribute ami_pushitem.build, leaving it unset.
86
[ WARNING] Missing information for the attribute ami_pushitem.description, leaving it unset.
@@ -32,39 +30,10 @@
3230
[ WARNING] Missing information for the attribute ami_pushitem.access_endpoint_url, leaving it unset.
3331
[ INFO] Uploading the item ami_pushitem to AWS-NA.
3432
[ INFO] Upload finished for ami_pushitem on AWS-NA
35-
[ INFO] Uploading the item ami_pushitem to AWS-EMEA.
36-
[ INFO] Upload finished for ami_pushitem on AWS-EMEA
37-
[ WARNING] Missing information for the attribute vhd_pushitem.src, leaving it unset.
38-
[ WARNING] Missing information for the attribute vhd_pushitem.build, leaving it unset.
39-
[ WARNING] Missing information for the attribute vhd_pushitem.description, leaving it unset.
40-
[ WARNING] Missing information for the attribute vhd_pushitem.boot_mode, leaving it unset.
41-
[ WARNING] Missing information for the attribute vhd_pushitem.cloud_info, leaving it unset.
42-
[ WARNING] Missing information for the attribute vhd_pushitem.marketplace_title_template, leaving it unset.
43-
[ WARNING] Missing information for the attribute vhd_pushitem.marketplace_name, leaving it unset.
44-
[ WARNING] Missing information for the attribute vhd_pushitem.sku_id, leaving it unset.
45-
[ WARNING] Missing information for the attribute vhd_pushitem.support_legacy, leaving it unset.
46-
[ WARNING] Missing information for the attribute vhd_pushitem.legacy_sku_id, leaving it unset.
47-
[ WARNING] Missing information for the attribute vhd_pushitem.disk_version, leaving it unset.
48-
[ WARNING] Missing information for the attribute vhd_pushitem.recommended_sizes, leaving it unset.
49-
[ WARNING] Missing information for the attribute vhd_pushitem.sas_uri, leaving it unset.
50-
[ INFO] Uploading the item vhd_pushitem to AZURE-NA.
51-
[ INFO] Upload finished for vhd_pushitem on AZURE-NA
5233
[ INFO] Preparing to publish the item ami_pushitem to ffffffff-ffff-ffff-ffff-ffffffffffff on AWS-NA.
5334
[ INFO] Preparation complete for item ami_pushitem to AWS-NA.
54-
[ INFO] Preparing to publish the item ami_pushitem to 00000000-0000-0000-0000-000000000000 on AWS-EMEA.
55-
[ INFO] Preparation complete for item ami_pushitem to AWS-EMEA.
56-
[ INFO] Preparing to publish the item vhd_pushitem to destination_offer_main/plan1 on AZURE-NA.
57-
[ INFO] Preparation complete for item vhd_pushitem to AZURE-NA.
58-
[ INFO] Preparing to publish the item vhd_pushitem to destination_offer_main/plan2 on AZURE-NA.
59-
[ INFO] Preparation complete for item vhd_pushitem to AZURE-NA.
60-
[ INFO] Preparing to publish the item vhd_pushitem to destination_offer_main/plan3 on AZURE-NA.
61-
[ INFO] Preparation complete for item vhd_pushitem to AZURE-NA.
6235
[ INFO] Pushing "ami_pushitem" (pre-push=False) to ffffffff-ffff-ffff-ffff-ffffffffffff on AWS-NA.
6336
[ ERROR] Failed to publish ami_pushitem on AWS-NA: Random exception
64-
[ INFO] Pushing "ami_pushitem" (pre-push=False) to 00000000-0000-0000-0000-000000000000 on AWS-EMEA.
65-
[ ERROR] Failed to publish ami_pushitem on AWS-EMEA: Random exception
66-
[ INFO] Pushing "vhd_pushitem" (pre-push=False) to destination_offer_main/plan1 on AZURE-NA.
67-
[ ERROR] Failed to publish vhd_pushitem on AZURE-NA: Random exception
6837
[ INFO] Collecting results
6938
[ ERROR] Marketplace VM push failed
7039
# Raised: 30

‎tests/push/conftest.py

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,30 @@ def starmap_response_aws() -> Dict[str, Any]:
9898
}
9999

100100

101+
@pytest.fixture
102+
def starmap_response_aws_1m() -> Dict[str, Any]:
103+
return {
104+
"mappings": {
105+
"aws-na": {
106+
"destinations": [
107+
{
108+
"architecture": "x86_64",
109+
"destination": "ffffffff-ffff-ffff-ffff-ffffffffffff",
110+
"overwrite": True,
111+
"restrict_version": False,
112+
"meta": {"tag1": "aws-na-value1", "tag2": "aws-na-value2"},
113+
"tags": {"key1": "value1", "key2": "value2"},
114+
"ami_version_template": "{major}.{minor}.{patch}",
115+
}
116+
]
117+
},
118+
},
119+
"name": "sample-product",
120+
"workflow": "stratosphere",
121+
"cloud": "aws",
122+
}
123+
124+
101125
@pytest.fixture
102126
def starmap_response_azure() -> Dict[str, Any]:
103127
return {
@@ -139,6 +163,11 @@ def starmap_query_aws(starmap_response_aws: Dict[str, Any]) -> QueryResponseEnti
139163
return QueryResponseEntity.from_json(starmap_response_aws)
140164

141165

166+
@pytest.fixture
167+
def starmap_query_aws_1m(starmap_response_aws_1m: Dict[str, Any]) -> QueryResponseEntity:
168+
return QueryResponseEntity.from_json(starmap_response_aws_1m)
169+
170+
142171
@pytest.fixture
143172
def starmap_query_azure(starmap_response_azure: Dict[str, Any]) -> QueryResponseEntity:
144173
return QueryResponseEntity.from_json(starmap_response_azure)

‎tests/push/test_push.py

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,15 @@ def fake_source(
4545
yield m
4646

4747

48+
@pytest.fixture()
49+
def fake_source_1m(
50+
ami_push_item: AmiPushItem, vhd_push_item: VHDPushItem
51+
) -> Generator[mock.MagicMock, None, None]:
52+
with mock.patch("pubtools._marketplacesvm.tasks.push.command.Source") as m:
53+
m.get.return_value.__enter__.return_value = [ami_push_item]
54+
yield m
55+
56+
4857
@pytest.fixture()
4958
def fake_starmap(
5059
starmap_query_aws: QueryResponseEntity, starmap_query_azure: QueryResponseEntity
@@ -57,6 +66,18 @@ def fake_starmap(
5766
yield m
5867

5968

69+
@pytest.fixture()
70+
def fake_starmap_1m(
71+
starmap_query_aws_1m: QueryResponseEntity, starmap_query_azure: QueryResponseEntity
72+
) -> Generator[mock.MagicMock, None, None]:
73+
with mock.patch("pubtools._marketplacesvm.tasks.push.MarketplacesVMPush.starmap") as m:
74+
m.query_image_by_name.side_effect = [
75+
QueryResponseContainer([x]) for x in [starmap_query_aws_1m]
76+
]
77+
78+
yield m
79+
80+
6081
@pytest.fixture()
6182
def fake_cloud_instance() -> Generator[mock.MagicMock, None, None]:
6283
with mock.patch("pubtools._marketplacesvm.tasks.push.MarketplacesVMPush.cloud_instance") as m:
@@ -455,8 +476,8 @@ def test_push_item_fail_upload(
455476
@mock.patch("pubtools._marketplacesvm.tasks.push.MarketplacesVMPush.cloud_instance")
456477
def test_push_item_fail_publish(
457478
mock_cloud_instance: mock.MagicMock,
458-
fake_source: mock.MagicMock,
459-
fake_starmap: mock.MagicMock,
479+
fake_source_1m: mock.MagicMock,
480+
fake_starmap_1m: mock.MagicMock,
460481
ami_push_item: AmiPushItem,
461482
vhd_push_item: VHDPushItem,
462483
command_tester: CommandTester,
@@ -478,14 +499,14 @@ def _publish(self, push_item, nochannel, overwrite, **kwargs):
478499
"--credentials",
479500
"eyJtYXJrZXRwbGFjZV9hY2NvdW50IjogInRlc3QtbmEiLCAiYXV0aCI6eyJmb28iOiJiYXIifQo=",
480501
"--debug",
481-
"koji:https://fakekoji.com?vmi_build=ami_build,azure_build",
502+
"koji:https://fakekoji.com?vmi_build=ami_build",
482503
],
483504
)
484-
starmap_calls = [mock.call(name="test-build", version="7.0") for _ in range(2)]
485-
fake_starmap.query_image_by_name.assert_has_calls(starmap_calls)
505+
starmap_calls = [mock.call(name="test-build", version="7.0") for _ in range(1)]
506+
fake_starmap_1m.query_image_by_name.assert_has_calls(starmap_calls)
486507
# get_provider, upload calls for "aws-na", "aws-emea", "azure-na" with
487508
# publish calls only for "aws-na" and "azure-na"
488-
assert mock_cloud_instance.call_count == 11
509+
assert mock_cloud_instance.call_count == 3
489510

490511

491512
@mock.patch("pubtools._marketplacesvm.tasks.push.command.Source")

0 commit comments

Comments
 (0)