Skip to content

Commit 66fe942

Browse files
ChrisB-TLChrisB-TL
committed
Add rate limits to anthropic response provider meta
1 parent 776c2d2 commit 66fe942

File tree

7 files changed

+108
-16
lines changed

7 files changed

+108
-16
lines changed

Diff for: ‎src/Providers/Anthropic/Handlers/AnthropicHandlerAbstract.php

+12-11
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,12 @@ protected function extractText(array $data): string
7474
protected function handleResponseErrors(): void
7575
{
7676
if ($this->httpResponse->getStatusCode() === 429) {
77-
$this->rateLimitException();
77+
throw PrismRateLimitedException::make(
78+
rateLimits: array_values($this->processRateLimits()),
79+
retryAfter: $this->httpResponse->hasHeader('retry-after')
80+
? (int) $this->httpResponse->getHeader('retry-after')[0]
81+
: null
82+
);
7883
}
7984

8085
$data = $this->httpResponse->json();
@@ -90,7 +95,10 @@ protected function handleResponseErrors(): void
9095
}
9196
}
9297

93-
protected function rateLimitException(): void
98+
/**
99+
* @return ProviderRateLimit[]
100+
*/
101+
protected function processRateLimits(): array
94102
{
95103
$rate_limits = [];
96104

@@ -106,7 +114,7 @@ protected function rateLimitException(): void
106114
$rate_limits[$limit_name][$field_name] = $headerValues[0];
107115
}
108116

109-
$rate_limits = Arr::map($rate_limits, function ($fields, $limit_name): ProviderRateLimit {
117+
return array_values(Arr::map($rate_limits, function ($fields, $limit_name): ProviderRateLimit {
110118
$resets_at = data_get($fields, 'reset');
111119

112120
return new ProviderRateLimit(
@@ -119,13 +127,6 @@ protected function rateLimitException(): void
119127
: null,
120128
resetsAt: data_get($fields, 'reset') ? new Carbon($resets_at) : null
121129
);
122-
});
123-
124-
throw PrismRateLimitedException::make(
125-
rateLimits: array_values($rate_limits),
126-
retryAfter: $this->httpResponse->hasHeader('retry-after')
127-
? (int) $this->httpResponse->getHeader('retry-after')[0]
128-
: null
129-
);
130+
}));
130131
}
131132
}

Diff for: ‎src/Providers/Anthropic/Handlers/Structured.php

+1
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ protected function buildProviderResponse(): ProviderResponse
6969
responseMeta: new ResponseMeta(
7070
id: data_get($data, 'id'),
7171
model: data_get($data, 'model'),
72+
rateLimits: $this->processRateLimits()
7273
)
7374
);
7475
}

Diff for: ‎src/Providers/Anthropic/Handlers/Text.php

+1
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ protected function buildProviderResponse(): ProviderResponse
6565
responseMeta: new ResponseMeta(
6666
id: data_get($data, 'id'),
6767
model: data_get($data, 'model'),
68+
rateLimits: $this->processRateLimits()
6869
)
6970
);
7071
}

Diff for: ‎src/ValueObjects/ResponseMeta.php

+5-1
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,12 @@
66

77
class ResponseMeta
88
{
9+
/**
10+
* @param ProviderRateLimit[] $rateLimits
11+
*/
912
public function __construct(
1013
public readonly string $id,
11-
public readonly string $model
14+
public readonly string $model,
15+
public readonly array $rateLimits = []
1216
) {}
1317
}

Diff for: ‎tests/Fixtures/FixtureResponse.php

+2-2
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ public static function recordResponses(string $requestPath, string $name): void
4848
});
4949
}
5050

51-
public static function fakeResponseSequence(string $requestPath, string $name): void
51+
public static function fakeResponseSequence(string $requestPath, string $name, array $headers = []): void
5252
{
5353
$responses = collect(scandir(dirname(static::filePath($name))))
5454
->filter(function (string $file) use ($name): int|false {
@@ -61,7 +61,7 @@ public static function fakeResponseSequence(string $requestPath, string $name):
6161
->map(fn ($filePath) => Http::response(
6262
file_get_contents($filePath),
6363
200,
64-
[]
64+
$headers
6565
));
6666

6767
Http::fake([

Diff for: ‎tests/Providers/Anthropic/AnthropicTextTest.php

+37-2
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,9 @@
1010
use EchoLabs\Prism\ValueObjects\Messages\Support\Image;
1111
use EchoLabs\Prism\ValueObjects\Messages\SystemMessage;
1212
use EchoLabs\Prism\ValueObjects\Messages\UserMessage;
13+
use EchoLabs\Prism\ValueObjects\ProviderRateLimit;
1314
use Illuminate\Http\Client\Request;
15+
use Illuminate\Support\Carbon;
1416
use Illuminate\Support\Facades\Http;
1517
use Tests\Fixtures\FixtureResponse;
1618

@@ -164,8 +166,6 @@
164166
});
165167

166168
it('can calculate cache usage correctly', function (): void {
167-
config()->set('prism.providers.anthropic.beta_features', 'prompt-caching-2024-07-31');
168-
169169
FixtureResponse::fakeResponseSequence('v1/messages', 'anthropic/calculate-cache-usage');
170170

171171
$response = Prism::text()
@@ -179,3 +179,38 @@
179179
expect($response->usage->cacheWriteInputTokens)->toBe(200);
180180
expect($response->usage->cacheReadInputTokens)->ToBe(100);
181181
});
182+
183+
it('adds rate limit data to the responseMeta', function (): void {
184+
$requests_reset = Carbon::now()->addSeconds(30);
185+
186+
FixtureResponse::fakeResponseSequence(
187+
'v1/messages',
188+
'anthropic/generate-text-with-a-prompt',
189+
[
190+
'anthropic-ratelimit-requests-limit' => 1000,
191+
'anthropic-ratelimit-requests-remaining' => 500,
192+
'anthropic-ratelimit-requests-reset' => $requests_reset->toISOString(),
193+
'anthropic-ratelimit-input-tokens-limit' => 80000,
194+
'anthropic-ratelimit-input-tokens-remaining' => 0,
195+
'anthropic-ratelimit-input-tokens-reset' => Carbon::now()->addSeconds(60)->toISOString(),
196+
'anthropic-ratelimit-output-tokens-limit' => 16000,
197+
'anthropic-ratelimit-output-tokens-remaining' => 15000,
198+
'anthropic-ratelimit-output-tokens-reset' => Carbon::now()->addSeconds(5)->toISOString(),
199+
'anthropic-ratelimit-tokens-limit' => 96000,
200+
'anthropic-ratelimit-tokens-remaining' => 15000,
201+
'anthropic-ratelimit-tokens-reset' => Carbon::now()->addSeconds(5)->toISOString(),
202+
]
203+
);
204+
205+
$response = Prism::text()
206+
->using('anthropic', 'claude-3-5-sonnet-20240620')
207+
->withPrompt('Who are you?')
208+
->generate();
209+
210+
expect($response->responseMeta->rateLimits)->toHaveCount(4);
211+
expect($response->responseMeta->rateLimits[0])->toBeInstanceOf(ProviderRateLimit::class);
212+
expect($response->responseMeta->rateLimits[0]->name)->toEqual('requests');
213+
expect($response->responseMeta->rateLimits[0]->limit)->toEqual(1000);
214+
expect($response->responseMeta->rateLimits[0]->remaining)->toEqual(500);
215+
expect($response->responseMeta->rateLimits[0]->resetsAt)->toEqual($requests_reset);
216+
});

Diff for: ‎tests/Providers/Anthropic/StructuredTest.php

+50
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
use EchoLabs\Prism\Schema\BooleanSchema;
1010
use EchoLabs\Prism\Schema\ObjectSchema;
1111
use EchoLabs\Prism\Schema\StringSchema;
12+
use EchoLabs\Prism\ValueObjects\ProviderRateLimit;
13+
use Illuminate\Support\Carbon;
1214
use Tests\Fixtures\FixtureResponse;
1315

1416
it('returns structured output', function (): void {
@@ -42,3 +44,51 @@
4244
expect($response->structured['game_time'])->toBeString();
4345
expect($response->structured['coat_required'])->toBeBool();
4446
});
47+
48+
it('adds rate limit data to the responseMeta', function (): void {
49+
$requests_reset = Carbon::now()->addSeconds(30);
50+
51+
FixtureResponse::fakeResponseSequence(
52+
'messages',
53+
'anthropic/structured',
54+
[
55+
'anthropic-ratelimit-requests-limit' => 1000,
56+
'anthropic-ratelimit-requests-remaining' => 500,
57+
'anthropic-ratelimit-requests-reset' => $requests_reset->toISOString(),
58+
'anthropic-ratelimit-input-tokens-limit' => 80000,
59+
'anthropic-ratelimit-input-tokens-remaining' => 0,
60+
'anthropic-ratelimit-input-tokens-reset' => Carbon::now()->addSeconds(60)->toISOString(),
61+
'anthropic-ratelimit-output-tokens-limit' => 16000,
62+
'anthropic-ratelimit-output-tokens-remaining' => 15000,
63+
'anthropic-ratelimit-output-tokens-reset' => Carbon::now()->addSeconds(5)->toISOString(),
64+
'anthropic-ratelimit-tokens-limit' => 96000,
65+
'anthropic-ratelimit-tokens-remaining' => 15000,
66+
'anthropic-ratelimit-tokens-reset' => Carbon::now()->addSeconds(5)->toISOString(),
67+
]
68+
);
69+
70+
$schema = new ObjectSchema(
71+
'output',
72+
'the output object',
73+
[
74+
new StringSchema('weather', 'The weather forecast'),
75+
new StringSchema('game_time', 'The tigers game time'),
76+
new BooleanSchema('coat_required', 'whether a coat is required'),
77+
],
78+
['weather', 'game_time', 'coat_required']
79+
);
80+
81+
$response = Prism::structured()
82+
->withSchema($schema)
83+
->using(Provider::Anthropic, 'claude-3-5-sonnet-latest')
84+
->withSystemPrompt('The tigers game is at 3pm and the temperature will be 70º')
85+
->withPrompt('What time is the tigers game today and should I wear a coat?')
86+
->generate();
87+
88+
expect($response->responseMeta->rateLimits)->toHaveCount(4);
89+
expect($response->responseMeta->rateLimits[0])->toBeInstanceOf(ProviderRateLimit::class);
90+
expect($response->responseMeta->rateLimits[0]->name)->toEqual('requests');
91+
expect($response->responseMeta->rateLimits[0]->limit)->toEqual(1000);
92+
expect($response->responseMeta->rateLimits[0]->remaining)->toEqual(500);
93+
expect($response->responseMeta->rateLimits[0]->resetsAt)->toEqual($requests_reset);
94+
});

0 commit comments

Comments
 (0)