Skip to content

Commit 776c2d2

Browse files
ChrisB-TLChrisB-TL
committed
anthropic rate limit handling
1 parent 1f3270a commit 776c2d2

File tree

4 files changed

+176
-11
lines changed

4 files changed

+176
-11
lines changed

Diff for: ‎src/Exceptions/PrismRateLimitedException.php

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace EchoLabs\Prism\Exceptions;
6+
7+
use EchoLabs\Prism\ValueObjects\ProviderRateLimit;
8+
9+
class PrismRateLimitedException extends PrismException
10+
{
11+
/**
12+
* @param ProviderRateLimit[] $rateLimits
13+
*/
14+
public function __construct(
15+
public readonly array $rateLimits,
16+
public readonly ?int $retryAfter = null
17+
) {
18+
$message = 'You hit a provider rate limit';
19+
20+
if ($retryAfter) {
21+
$message .= ' - retry after '.$retryAfter.' seconds';
22+
}
23+
24+
$message .= '. Details: '.json_encode($rateLimits);
25+
26+
parent::__construct($message);
27+
}
28+
29+
/**
30+
* @param ProviderRateLimit[] $rateLimits
31+
*/
32+
public static function make(array $rateLimits = [], ?int $retryAfter = null): self
33+
{
34+
return new self(rateLimits: $rateLimits, retryAfter: $retryAfter);
35+
}
36+
}

Diff for: ‎src/Providers/Anthropic/Handlers/AnthropicHandlerAbstract.php

+64-11
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,14 @@
66

77
use EchoLabs\Prism\Contracts\PrismRequest;
88
use EchoLabs\Prism\Exceptions\PrismException;
9+
use EchoLabs\Prism\Exceptions\PrismRateLimitedException;
10+
use EchoLabs\Prism\ValueObjects\ProviderRateLimit;
911
use EchoLabs\Prism\ValueObjects\ProviderResponse;
1012
use Illuminate\Http\Client\PendingRequest;
1113
use Illuminate\Http\Client\Response;
14+
use Illuminate\Support\Arr;
15+
use Illuminate\Support\Carbon;
16+
use Illuminate\Support\Str;
1217
use Throwable;
1318

1419
abstract class AnthropicHandlerAbstract
@@ -35,17 +40,7 @@ public function handle(PrismRequest $request): ProviderResponse
3540
throw PrismException::providerRequestError($this->request->model, $e); // @phpstan-ignore property.notFound
3641
}
3742

38-
$data = $this->httpResponse->json();
39-
40-
if (data_get($data, 'type') === 'error') {
41-
throw PrismException::providerResponseError(vsprintf(
42-
'Anthropic Error: [%s] %s',
43-
[
44-
data_get($data, 'error.type', 'unknown'),
45-
data_get($data, 'error.message'),
46-
]
47-
));
48-
}
43+
$this->handleResponseErrors();
4944

5045
return $this->buildProviderResponse();
5146
}
@@ -75,4 +70,62 @@ protected function extractText(array $data): string
7570
return $text;
7671
}, '');
7772
}
73+
74+
protected function handleResponseErrors(): void
75+
{
76+
if ($this->httpResponse->getStatusCode() === 429) {
77+
$this->rateLimitException();
78+
}
79+
80+
$data = $this->httpResponse->json();
81+
82+
if (data_get($data, 'type') === 'error') {
83+
throw PrismException::providerResponseError(vsprintf(
84+
'Anthropic Error: [%s] %s',
85+
[
86+
data_get($data, 'error.type', 'unknown'),
87+
data_get($data, 'error.message'),
88+
]
89+
));
90+
}
91+
}
92+
93+
protected function rateLimitException(): void
94+
{
95+
$rate_limits = [];
96+
97+
foreach ($this->httpResponse->getHeaders() as $headerName => $headerValues) {
98+
if (Str::startsWith($headerName, 'anthropic-ratelimit-') === false) {
99+
continue;
100+
}
101+
102+
$limit_name = Str::of($headerName)->after('anthropic-ratelimit-')->beforeLast('-')->toString();
103+
104+
$field_name = Str::of($headerName)->afterLast('-')->toString();
105+
106+
$rate_limits[$limit_name][$field_name] = $headerValues[0];
107+
}
108+
109+
$rate_limits = Arr::map($rate_limits, function ($fields, $limit_name): ProviderRateLimit {
110+
$resets_at = data_get($fields, 'reset');
111+
112+
return new ProviderRateLimit(
113+
name: $limit_name,
114+
limit: data_get($fields, 'limit')
115+
? (int) data_get($fields, 'limit')
116+
: null,
117+
remaining: data_get($fields, 'remaining')
118+
? (int) data_get($fields, 'remaining')
119+
: null,
120+
resetsAt: data_get($fields, 'reset') ? new Carbon($resets_at) : null
121+
);
122+
});
123+
124+
throw PrismRateLimitedException::make(
125+
rateLimits: array_values($rate_limits),
126+
retryAfter: $this->httpResponse->hasHeader('retry-after')
127+
? (int) $this->httpResponse->getHeader('retry-after')[0]
128+
: null
129+
);
130+
}
78131
}

Diff for: ‎src/ValueObjects/ProviderRateLimit.php

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
<?php
2+
3+
namespace EchoLabs\Prism\ValueObjects;
4+
5+
use Carbon\Carbon;
6+
7+
class ProviderRateLimit
8+
{
9+
public function __construct(
10+
public readonly string $name,
11+
public readonly ?int $limit = null,
12+
public readonly ?int $remaining = null,
13+
public readonly ?Carbon $resetsAt = null
14+
) {}
15+
}

Diff for: ‎tests/Providers/Anthropic/AnthropicRateLimitExceptionTest.php

+61
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
<?php
2+
3+
use EchoLabs\Prism\Exceptions\PrismRateLimitedException;
4+
use EchoLabs\Prism\Prism;
5+
use EchoLabs\Prism\ValueObjects\ProviderRateLimit;
6+
use Illuminate\Support\Carbon;
7+
use Illuminate\Support\Facades\Http;
8+
9+
it('throws a RateLimitException if the Anthropic responds with a 429', function (): void {
10+
Http::fake([
11+
'https://api.anthropic.com/*' => Http::response(
12+
status: 429,
13+
),
14+
])->preventStrayRequests();
15+
16+
Prism::text()
17+
->using('anthropic', 'claude-3-5-sonnet-20240620')
18+
->withPrompt('Hello world!')
19+
->generate();
20+
21+
})->throws(PrismRateLimitedException::class);
22+
23+
it('sets the correct data on the RateLimitException', function (): void {
24+
$requests_reset = Carbon::now()->addSeconds(30);
25+
26+
Http::fake([
27+
'https://api.anthropic.com/*' => Http::response(
28+
status: 429,
29+
headers: [
30+
'anthropic-ratelimit-requests-limit' => 1000,
31+
'anthropic-ratelimit-requests-remaining' => 500,
32+
'anthropic-ratelimit-requests-reset' => $requests_reset->toISOString(),
33+
'anthropic-ratelimit-input-tokens-limit' => 80000,
34+
'anthropic-ratelimit-input-tokens-remaining' => 0,
35+
'anthropic-ratelimit-input-tokens-reset' => Carbon::now()->addSeconds(60)->toISOString(),
36+
'anthropic-ratelimit-output-tokens-limit' => 16000,
37+
'anthropic-ratelimit-output-tokens-remaining' => 15000,
38+
'anthropic-ratelimit-output-tokens-reset' => Carbon::now()->addSeconds(5)->toISOString(),
39+
'anthropic-ratelimit-tokens-limit' => 96000,
40+
'anthropic-ratelimit-tokens-remaining' => 15000,
41+
'anthropic-ratelimit-tokens-reset' => Carbon::now()->addSeconds(5)->toISOString(),
42+
'retry-after' => 40,
43+
]
44+
),
45+
])->preventStrayRequests();
46+
47+
try {
48+
Prism::text()
49+
->using('anthropic', 'claude-3-5-sonnet-20240620')
50+
->withPrompt('Hello world!')
51+
->generate();
52+
} catch (PrismRateLimitedException $e) {
53+
expect($e->retryAfter)->toEqual(40);
54+
expect($e->rateLimits)->toHaveCount(4);
55+
expect($e->rateLimits[0])->toBeInstanceOf(ProviderRateLimit::class);
56+
expect($e->rateLimits[0]->name)->toEqual('requests');
57+
expect($e->rateLimits[0]->limit)->toEqual(1000);
58+
expect($e->rateLimits[0]->remaining)->toEqual(500);
59+
expect($e->rateLimits[0]->resetsAt)->toEqual($requests_reset);
60+
}
61+
});

0 commit comments

Comments
 (0)