Skip to content

Commit 4445404

Browse files
Do not retry spooling-related GET requests with code 200 and empty body
The client currently treats all GET requests to which Trino responds with status code 200 and empty body as failed and retries them. But the empty body is a normal response for segment acknowledgements requests, they should not be retried if the status is 200. This change fixes that by turning off body emptiness checking (effectively reverting PR #603) for spooling-releted requests. Additional minor change is to access `content` rather than `text` when checking for emptiness, to avoid decoding binary responses to text just to see if they are not empty.
1 parent c4b2c57 commit 4445404

3 files changed

Lines changed: 109 additions & 21 deletions

File tree

‎tests/unit/test_client.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1752,3 +1752,43 @@ def statement_uri(token):
17521752
assert query.stats["state"] == "FINISHED"
17531753
# The count row survives draining and the rows stay lazily iterable.
17541754
assert list(result) == [[3]]
1755+
1756+
1757+
def _retryable_get(monkeypatch, body: bytes, status_code: int = 200):
1758+
"""Serve a canned response from Session.get and return (request, recorder)."""
1759+
http_resp = TrinoRequest.http.Response()
1760+
http_resp.status_code = status_code
1761+
http_resp._content = body
1762+
1763+
get_retry = RetryRecorder(result=http_resp)
1764+
monkeypatch.setattr(TrinoRequest.http.Session, "get", get_retry)
1765+
1766+
req = TrinoRequest(
1767+
host="coordinator",
1768+
port=8080,
1769+
client_session=ClientSession(user="test"),
1770+
max_attempts=3,
1771+
)
1772+
return req, get_retry
1773+
1774+
1775+
def test_empty_body_retry_check_does_not_decode_body(monkeypatch):
1776+
# Reading .text would run charset detection over the whole payload merely to test emptiness,
1777+
# which is wasteful for a large binary body. The check must look at the raw bytes instead.
1778+
req, get_retry = _retryable_get(monkeypatch, body=b"\x89PNG\r\n\x1a\n" * 1024)
1779+
1780+
with mock.patch.object(
1781+
TrinoRequest.http.Response, "text", new_callable=mock.PropertyMock
1782+
) as response_text:
1783+
req.get("URL")
1784+
1785+
response_text.assert_not_called()
1786+
assert get_retry.retry_count == 1
1787+
1788+
1789+
def test_whitespace_only_200_response_retry(monkeypatch):
1790+
req, get_retry = _retryable_get(monkeypatch, body=b" \r\n\t ")
1791+
1792+
req.get("URL")
1793+
1794+
assert get_retry.retry_count == 3

‎tests/unit/test_client_spooling.py‎

Lines changed: 42 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,9 @@
1313
import time
1414
from unittest import mock
1515

16+
import httpretty
1617
import pytest
18+
from httpretty import httprettified
1719

1820
from trino.client import _RequestHeartbeat
1921
from trino.client import ClientSession
@@ -287,7 +289,7 @@ def fake_get(uri, headers=None, **kwargs):
287289
recorded["headers"] = headers
288290
return mock.Mock(ok=True)
289291

290-
segment._request._get = fake_get
292+
segment._request._get_accept_empty_body = fake_get
291293
segment._send_spooling_request(segment.uri)
292294

293295
assert recorded["headers"]["X-Auth-Gateway-Token"] == "user-token"
@@ -304,7 +306,7 @@ def fake_get(uri, headers=None, **kwargs):
304306
recorded["headers"] = headers
305307
return mock.Mock(ok=True)
306308

307-
segment._request._get = fake_get
309+
segment._request._get_accept_empty_body = fake_get
308310
external_uri = "https://s3.amazonaws.com/bucket/seg1?X-Amz-Signature=abc"
309311
segment._send_spooling_request(external_uri)
310312

@@ -323,7 +325,44 @@ def fake_get(uri, headers=None, **kwargs):
323325
recorded["headers"] = headers
324326
return mock.Mock(ok=True)
325327

326-
segment._request._get = fake_get
328+
segment._request._get_accept_empty_body = fake_get
327329
segment._send_spooling_request(segment.uri)
328330

329331
assert recorded["headers"]["X-Trino-Spooling-Token"] == "token-abc"
332+
333+
334+
def _spooled_segment_for_ack(max_attempts):
335+
request = TrinoRequest(
336+
host="coordinator",
337+
port=8080,
338+
client_session=ClientSession(user="test"),
339+
http_scheme="http",
340+
max_attempts=max_attempts,
341+
)
342+
segment_to = {
343+
"type": "spooled",
344+
"uri": "http://coordinator/v1/spooled/download/seg1",
345+
"ackUri": "http://coordinator/v1/spooled/ack/seg1",
346+
"metadata": {"segmentSize": "1", "uncompressedSize": "1"},
347+
}
348+
return SpooledSegment(segment_to, request)
349+
350+
351+
@httprettified
352+
def test_acknowledge_request_retries_on_error_status_code():
353+
segment = _spooled_segment_for_ack(max_attempts=3)
354+
httpretty.register_uri(method=httpretty.GET, uri=segment.ack_uri, body="", status=503)
355+
356+
segment._send_acknowledgement()
357+
358+
assert len(httpretty.latest_requests()) == 3
359+
360+
361+
@httprettified
362+
def test_acknowledge_request_does_not_retry_on_empty_200_response():
363+
segment = _spooled_segment_for_ack(max_attempts=3)
364+
httpretty.register_uri(method=httpretty.GET, uri=segment.ack_uri, body="", status=200)
365+
366+
segment._send_acknowledgement()
367+
368+
assert len(httpretty.latest_requests()) == 1

‎trino/client.py‎

Lines changed: 27 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -636,29 +636,36 @@ def max_attempts(self) -> int:
636636
def max_attempts(self, value: int) -> None:
637637
self._max_attempts = value
638638
if value == 1: # No retry
639-
self._get = self._http_session.get
639+
self._get = self._get_accept_empty_body = self._http_session.get
640640
self._post = self._http_session.post
641641
self._delete = self._http_session.delete
642642
self._head = self._http_session.head
643643
return
644644

645+
def has_error_status(response: Response) -> bool:
646+
return getattr(response, "status_code", None) in (429, 502, 503, 504)
647+
648+
def has_ok_status_but_no_body(response: Response) -> bool:
649+
return getattr(response, "status_code", None) == 200 and not getattr(response, "content", b"").strip()
650+
645651
with_retry = _retry_with(
646652
self._handle_retry,
647653
handled_exceptions=self._exceptions,
648-
conditions=(
649-
# need retry when there is no exception but the status code is 429, 502, 503, or 504
650-
lambda response: getattr(response, "status_code", None)
651-
in (429, 502, 503, 504),
652-
# need retry when the server returns 200 with an empty body (transient under load)
653-
lambda response: getattr(response, "status_code", None) == 200
654-
and not getattr(response, "text", "").strip(),
655-
),
654+
# retry when there is no exception but error status_code, and when status code is 200 but there's no body
655+
conditions=(has_error_status, has_ok_status_but_no_body),
656656
max_attempts=self._max_attempts,
657657
)
658+
658659
self._get = with_retry(self._http_session.get)
659660
self._post = with_retry(self._http_session.post)
660661
self._delete = with_retry(self._http_session.delete)
661662
self._head = with_retry(self._http_session.head)
663+
self._get_accept_empty_body = _retry_with(
664+
self._handle_retry,
665+
handled_exceptions=self._exceptions,
666+
conditions=(has_error_status,),
667+
max_attempts=self._max_attempts,
668+
)(self._http_session.get)
662669

663670
def get_url(self, path: str) -> str:
664671
return "{protocol}://{host}:{port}{path}".format(
@@ -1153,6 +1160,7 @@ def decorated(*args, **kwargs):
11531160
try:
11541161
result = func(*args, **kwargs)
11551162
if any(guard(result) for guard in conditions):
1163+
print(f"Retrying request {attempt} of {max_attempts}, url {result.url}, status_code {result.status_code}")
11561164
if result.status_code == 429 and "Retry-After" in result.headers:
11571165
retry_after = _parse_retry_after_header(result.headers.get("Retry-After"))
11581166
handle_retry_sleep = _RetryAfterSleep(retry_after)
@@ -1310,15 +1318,16 @@ def headers(self) -> Dict[str, List[str]]:
13101318
return self._segment.get("headers", {})
13111319

13121320
def acknowledge(self) -> None:
1313-
def acknowledge_request():
1314-
try:
1315-
http_response = self._send_spooling_request(self.ack_uri, timeout=2)
1316-
if not http_response.ok:
1317-
self._request.raise_response_error(http_response)
1318-
except Exception as e:
1319-
logger.error(f"Failed to acknowledge spooling request for segment {self}: {e}")
13201321
# Start the acknowledgment in the executor thread
1321-
executor.submit(acknowledge_request)
1322+
executor.submit(self._send_acknowledgement)
1323+
1324+
def _send_acknowledgement(self):
1325+
try:
1326+
http_response = self._send_spooling_request(self.ack_uri, timeout=2)
1327+
if not http_response.ok:
1328+
self._request.raise_response_error(http_response)
1329+
except Exception as e:
1330+
logger.error(f"Failed to acknowledge spooling request for segment {self}: {e}")
13221331

13231332
def _send_spooling_request(self, uri: str, **kwargs) -> requests.Response:
13241333
headers: Dict[str, str] = {}
@@ -1332,7 +1341,7 @@ def _send_spooling_request(self, uri: str, **kwargs) -> requests.Response:
13321341
if len(values) > 1:
13331342
raise ValueError(f"Header '{key}' contains multiple values: {values}")
13341343
headers[key] = values[0]
1335-
return self._request._get(uri, headers=headers, **kwargs)
1344+
return self._request._get_accept_empty_body(uri, headers=headers, **kwargs)
13361345

13371346
def __repr__(self):
13381347
return (

0 commit comments

Comments
 (0)