Bug: Too many requests error missed retry (#279)

Author: yanghua

tosfs/retry.py

@@ -164,10 +164,10 @@ def _get_sleep_time(err: TosError, retry_count: int) -> float:
             err.status_code == TOO_MANY_REQUESTS_CODE
             or err.status_code == SERVICE_UNAVAILABLE
         )
-        and "retry-after" in err.headers
+        and "retry-after" in err.header
     ):
         try:
-            sleep_time = max(int(err.headers["retry-after"]), int(sleep_time))
+            sleep_time = max(int(err.header["retry-after"]), int(sleep_time))
         except Exception as e:
             from tosfs.core import logger
 

tosfs/tests/test_retry.py

@@ -20,7 +20,7 @@
 from tos.http import Response
 from urllib3.exceptions import ProtocolError
 
-from tosfs.retry import is_retryable_exception
+from tosfs.retry import _get_sleep_time, is_retryable_exception
 
 mock_resp = Mock(spec=requests.Response)
 
@@ -86,3 +86,38 @@ def test_is_retry_exception(
     expected,
 ):
     assert is_retryable_exception(exception) == expected
+
+
+def test_get_sleep_time():
+    mock_resp = Mock(spec=requests.Response)
+
+    sleep_time = 1000
+
+    mock_resp.status_code = 429
+    mock_resp.headers = {
+        "Content-Type": "application/json",
+        "Date": "Thu, 07 Nov 2024 17:19:56 GMT",
+        "Retry-After": str(sleep_time),
+        "Server": "TosServer",
+        "x-tos-ec": "0004-00000007",
+        "x-tos-id-2": "0000-00000-0000",
+        "x-tos-request-id": "0000-0000-0000",
+        "x-tos-server-time": "1",
+        "Content-Length": "277",
+    }
+    mock_resp.iter_content = Mock(return_value=[b"chunk1", b"chunk2", b"chunk3"])
+    mock_resp.json = Mock(return_value={"key": "value"})
+
+    response = Response(mock_resp)
+
+    err = TosServerError(
+        response,
+        "Exceed account external rate limit. Too much throughput in a "
+        "short period of time, please slow down.",
+        "ExceedAccountExternalRateLimit",
+        "KmsJSKDKhjasdlKmsduwRETYHB",
+        "",
+        "0004-00000001",
+    )
+
+    assert _get_sleep_time(err, 1) == sleep_time