Skip to content

Commit 8762775

Browse files
sixlivesixlive
authored
feat(streaming): Tool Chunks (#384)
1 parent 4ae4de5 commit 8762775

File tree

19 files changed

+264
-174
lines changed

19 files changed

+264
-174
lines changed

‎docs/core-concepts/streaming-output.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,14 +67,14 @@ foreach ($response as $chunk) {
6767
$fullResponse .= $chunk->text;
6868

6969
// Check for tool calls
70-
if ($chunk->toolCalls) {
70+
if ($chunk->chunkType === ChunkType::ToolCall) {
7171
foreach ($chunk->toolCalls as $call) {
7272
echo "Tool called: " . $call->name;
7373
}
7474
}
7575

7676
// Check for tool results
77-
if ($chunk->toolResults) {
77+
if ($chunk->chunkType === ChunkType::ToolResult) {
7878
foreach ($chunk->toolResults as $result) {
7979
echo "Tool result: " . $result->result;
8080
}

‎src/Enums/ChunkType.php

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,6 @@ enum ChunkType: string
99
case Text = 'text';
1010
case Thinking = 'thinking';
1111
case Meta = 'meta';
12+
case ToolCall = 'tool_call';
13+
case ToolResult = 'tool_result';
1214
}

‎src/Providers/Anthropic/Handlers/Stream.php

Lines changed: 66 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
use Prism\Prism\ValueObjects\Messages\ToolResultMessage;
2727
use Prism\Prism\ValueObjects\Meta;
2828
use Prism\Prism\ValueObjects\ToolCall;
29+
use Prism\Prism\ValueObjects\ToolResult;
2930
use Psr\Http\Message\StreamInterface;
3031
use Throwable;
3132

@@ -155,7 +156,7 @@ protected function handleMessageStart(Response $response, array $chunk): Chunk
155156
/**
156157
* @param array<string, mixed> $chunk
157158
*/
158-
protected function handleContentBlockStart(array $chunk): void
159+
protected function handleContentBlockStart(array $chunk): null
159160
{
160161
$blockType = data_get($chunk, 'content_block.type');
161162
$blockIndex = (int) data_get($chunk, 'index');
@@ -171,6 +172,8 @@ protected function handleContentBlockStart(array $chunk): void
171172
'input' => '',
172173
]);
173174
}
175+
176+
return null;
174177
}
175178

176179
/**
@@ -315,9 +318,37 @@ protected function handleThinkingBlockDelta(array $chunk, ?string $deltaType): ?
315318
return null;
316319
}
317320

318-
protected function handleContentBlockStop(): void
321+
protected function handleContentBlockStop(): ?Chunk
319322
{
323+
$blockType = $this->state->tempContentBlockType();
324+
$blockIndex = $this->state->tempContentBlockIndex();
325+
326+
$chunk = null;
327+
328+
if ($blockType === 'tool_use' && $blockIndex !== null && isset($this->state->toolCalls()[$blockIndex])) {
329+
$toolCallData = $this->state->toolCalls()[$blockIndex];
330+
$input = data_get($toolCallData, 'input');
331+
332+
if (is_string($input) && $this->isValidJson($input)) {
333+
$input = json_decode($input, true);
334+
}
335+
336+
$toolCall = new ToolCall(
337+
id: data_get($toolCallData, 'id'),
338+
name: data_get($toolCallData, 'name'),
339+
arguments: $input
340+
);
341+
342+
$chunk = new Chunk(
343+
text: '',
344+
toolCalls: [$toolCall],
345+
chunkType: ChunkType::ToolCall
346+
);
347+
}
348+
320349
$this->state->resetContentBlock();
350+
351+
return $chunk;
321352
}
322353

323354
/**
@@ -525,14 +556,41 @@ protected function parseJsonData(string $jsonDataLine, ?string $eventType = null
525556
*/
526557
protected function handleToolCalls(Request $request, array $toolCalls, int $depth, ?array $additionalContent = null): Generator
527558
{
528-
$toolResults = $this->callTools($request->tools(), $toolCalls);
559+
$toolResults = [];
529560

530-
$this->addMessagesToRequest($request, $toolResults, $additionalContent);
561+
foreach ($toolCalls as $toolCall) {
562+
$tool = $this->resolveTool($toolCall->name, $request->tools());
531563

532-
yield new Chunk(
533-
text: '',
534-
toolResults: $toolResults,
535-
);
564+
try {
565+
$result = call_user_func_array(
566+
$tool->handle(...),
567+
$toolCall->arguments()
568+
);
569+
570+
$toolResult = new ToolResult(
571+
toolCallId: $toolCall->id,
572+
toolName: $toolCall->name,
573+
args: $toolCall->arguments(),
574+
result: $result,
575+
);
576+
577+
$toolResults[] = $toolResult;
578+
579+
yield new Chunk(
580+
text: '',
581+
toolResults: [$toolResult],
582+
chunkType: ChunkType::ToolResult
583+
);
584+
} catch (Throwable $e) {
585+
if ($e instanceof PrismException) {
586+
throw $e;
587+
}
588+
589+
throw PrismException::toolCallFailed($toolCall, $e);
590+
}
591+
}
592+
593+
$this->addMessagesToRequest($request, $toolResults, $additionalContent);
536594

537595
$nextResponse = $this->sendRequest($request);
538596
yield from $this->processStream($nextResponse, $request, $depth + 1);

‎src/Providers/Ollama/Concerns/MapsFinishReason.php

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,6 @@ trait MapsFinishReason
1414
*/
1515
protected function mapFinishReason(array $data): FinishReason
1616
{
17-
if (! empty(data_get($data, 'message.tool_calls'))) {
18-
return FinishReason::ToolCalls;
19-
}
20-
2117
return FinishReasonMap::map(data_get($data, 'done_reason', ''));
2218
}
2319
}

‎src/Providers/Ollama/Concerns/MapsToolCalls.php

Lines changed: 0 additions & 23 deletions
This file was deleted.

‎src/Providers/Ollama/Handlers/Stream.php

Lines changed: 29 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,24 +10,25 @@
1010
use Illuminate\Http\Client\Response;
1111
use Illuminate\Support\Arr;
1212
use Prism\Prism\Concerns\CallsTools;
13+
use Prism\Prism\Enums\ChunkType;
1314
use Prism\Prism\Enums\FinishReason;
1415
use Prism\Prism\Exceptions\PrismChunkDecodeException;
1516
use Prism\Prism\Exceptions\PrismException;
1617
use Prism\Prism\Exceptions\PrismRateLimitedException;
1718
use Prism\Prism\Providers\Ollama\Concerns\MapsFinishReason;
18-
use Prism\Prism\Providers\Ollama\Concerns\MapsToolCalls;
1919
use Prism\Prism\Providers\Ollama\Maps\MessageMap;
2020
use Prism\Prism\Providers\Ollama\Maps\ToolMap;
2121
use Prism\Prism\Text\Chunk;
2222
use Prism\Prism\Text\Request;
2323
use Prism\Prism\ValueObjects\Messages\AssistantMessage;
2424
use Prism\Prism\ValueObjects\Messages\ToolResultMessage;
25+
use Prism\Prism\ValueObjects\ToolCall;
2526
use Psr\Http\Message\StreamInterface;
2627
use Throwable;
2728

2829
class Stream
2930
{
30-
use CallsTools, MapsFinishReason, MapsToolCalls;
31+
use CallsTools, MapsFinishReason;
3132

3233
public function __construct(protected PendingRequest $client) {}
3334

@@ -61,13 +62,15 @@ protected function processStream(Response $response, Request $request, int $dept
6162
continue;
6263
}
6364

65+
// Accumulate tool calls if present
6466
if ($this->hasToolCalls($data)) {
6567
$toolCalls = $this->extractToolCalls($data, $toolCalls);
6668

6769
continue;
6870
}
6971

70-
if ($this->mapFinishReason($data) === FinishReason::ToolCalls) {
72+
// Handle tool call completion when stream is done
73+
if ((bool) data_get($data, 'done', false) && $toolCalls !== []) {
7174
yield from $this->handleToolCalls($request, $text, $toolCalls, $depth);
7275

7376
return;
@@ -142,17 +145,23 @@ protected function handleToolCalls(
142145

143146
$toolCalls = $this->mapToolCalls($toolCalls);
144147

145-
$toolResults = $this->callTools($request->tools(), $toolCalls);
148+
yield new Chunk(
149+
text: '',
150+
toolCalls: $toolCalls,
151+
chunkType: ChunkType::ToolCall,
152+
);
146153

147-
$request->addMessage(new AssistantMessage($text, $toolCalls));
148-
$request->addMessage(new ToolResultMessage($toolResults));
154+
$toolResults = $this->callTools($request->tools(), $toolCalls);
149155

150156
yield new Chunk(
151157
text: '',
152-
toolCalls: $toolCalls,
153158
toolResults: $toolResults,
159+
chunkType: ChunkType::ToolResult,
154160
);
155161

162+
$request->addMessage(new AssistantMessage($text, $toolCalls));
163+
$request->addMessage(new ToolResultMessage($toolResults));
164+
156165
$nextResponse = $this->sendRequest($request);
157166
yield from $this->processStream($nextResponse, $request, $depth + 1);
158167
}
@@ -217,4 +226,17 @@ protected function readLine(StreamInterface $stream): string
217226

218227
return $buffer;
219228
}
229+
230+
/**
231+
* @param array<int, array<string, mixed>> $toolCalls
232+
* @return array<int, ToolCall>
233+
*/
234+
protected function mapToolCalls(array $toolCalls): array
235+
{
236+
return array_map(fn (array $toolCall): ToolCall => new ToolCall(
237+
id: data_get($toolCall, 'id') ?? '',
238+
name: data_get($toolCall, 'name') ?? '',
239+
arguments: data_get($toolCall, 'arguments'),
240+
), $toolCalls);
241+
}
220242
}

‎src/Providers/Ollama/Handlers/Text.php

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
use Prism\Prism\Enums\FinishReason;
1111
use Prism\Prism\Exceptions\PrismException;
1212
use Prism\Prism\Providers\Ollama\Concerns\MapsFinishReason;
13-
use Prism\Prism\Providers\Ollama\Concerns\MapsToolCalls;
1413
use Prism\Prism\Providers\Ollama\Concerns\ValidatesResponse;
1514
use Prism\Prism\Providers\Ollama\Maps\MessageMap;
1615
use Prism\Prism\Providers\Ollama\Maps\ToolMap;
@@ -21,6 +20,7 @@
2120
use Prism\Prism\ValueObjects\Messages\AssistantMessage;
2221
use Prism\Prism\ValueObjects\Messages\ToolResultMessage;
2322
use Prism\Prism\ValueObjects\Meta;
23+
use Prism\Prism\ValueObjects\ToolCall;
2424
use Prism\Prism\ValueObjects\ToolResult;
2525
use Prism\Prism\ValueObjects\Usage;
2626
use Throwable;
@@ -29,7 +29,6 @@ class Text
2929
{
3030
use CallsTools;
3131
use MapsFinishReason;
32-
use MapsToolCalls;
3332
use ValidatesResponse;
3433

3534
protected ResponseBuilder $responseBuilder;
@@ -54,8 +53,12 @@ public function handle(Request $request): Response
5453

5554
$request->addMessage($responseMessage);
5655

56+
// Check for tool calls first, regardless of finish reason
57+
if (! empty(data_get($data, 'message.tool_calls'))) {
58+
return $this->handleToolCalls($data, $request);
59+
}
60+
5761
return match ($this->mapFinishReason($data)) {
58-
FinishReason::ToolCalls => $this->handleToolCalls($data, $request),
5962
FinishReason::Stop => $this->handleStop($data, $request),
6063
default => throw new PrismException('Ollama: unknown finish reason'),
6164
};
@@ -152,4 +155,17 @@ protected function addStep(array $data, Request $request, array $toolResults = [
152155
systemPrompts: $request->systemPrompts(),
153156
));
154157
}
158+
159+
/**
160+
* @param array<int, array<string, mixed>> $toolCalls
161+
* @return array<int, ToolCall>
162+
*/
163+
protected function mapToolCalls(array $toolCalls): array
164+
{
165+
return array_map(fn (array $toolCall): ToolCall => new ToolCall(
166+
id: data_get($toolCall, 'id') ?? '',
167+
name: data_get($toolCall, 'function.name'),
168+
arguments: data_get($toolCall, 'function.arguments'),
169+
), $toolCalls);
170+
}
155171
}

‎src/Providers/Ollama/Maps/MessageMap.php

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,11 @@
1111
use Prism\Prism\ValueObjects\Messages\SystemMessage;
1212
use Prism\Prism\ValueObjects\Messages\ToolResultMessage;
1313
use Prism\Prism\ValueObjects\Messages\UserMessage;
14+
use Prism\Prism\ValueObjects\ToolCall;
1415

1516
class MessageMap
1617
{
17-
/** @var array<int, array{role: string, content: string}> */
18+
/** @var array<int, mixed> */
1819
protected array $mappedMessages = [];
1920

2021
/**
@@ -87,9 +88,15 @@ protected function mapUserMessage(UserMessage $message): void
8788

8889
protected function mapAssistantMessage(AssistantMessage $message): void
8990
{
90-
$this->mappedMessages[] = [
91+
$this->mappedMessages[] = array_filter([
9192
'role' => 'assistant',
9293
'content' => $message->content,
93-
];
94+
'tool_calls' => $message->toolCalls ? array_map(fn (ToolCall $toolCall): array => [
95+
'function' => [
96+
'name' => $toolCall->name,
97+
'arguments' => $toolCall->arguments(),
98+
],
99+
], $message->toolCalls) : null,
100+
]);
94101
}
95102
}

‎src/Providers/OpenAI/Handlers/Stream.php

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
use Illuminate\Support\Arr;
1212
use Illuminate\Support\Str;
1313
use Prism\Prism\Concerns\CallsTools;
14+
use Prism\Prism\Enums\ChunkType;
1415
use Prism\Prism\Enums\FinishReason;
1516
use Prism\Prism\Exceptions\PrismChunkDecodeException;
1617
use Prism\Prism\Exceptions\PrismException;
@@ -149,23 +150,25 @@ protected function handleToolCalls(
149150
array $toolCalls,
150151
int $depth
151152
): Generator {
152-
// Convert collected tool call data to ToolCall objects
153153
$toolCalls = $this->mapToolCalls($toolCalls);
154154

155-
// Call the tools and get results
156-
$toolResults = $this->callTools($request->tools(), $toolCalls);
155+
yield new Chunk(
156+
text: '',
157+
toolCalls: $toolCalls,
158+
chunkType: ChunkType::ToolCall,
159+
);
157160

158-
$request->addMessage(new AssistantMessage($text, $toolCalls));
159-
$request->addMessage(new ToolResultMessage($toolResults));
161+
$toolResults = $this->callTools($request->tools(), $toolCalls);
160162

161-
// Yield the tool call chunk
162163
yield new Chunk(
163164
text: '',
164-
toolCalls: $toolCalls,
165165
toolResults: $toolResults,
166+
chunkType: ChunkType::ToolResult,
166167
);
167168

168-
// Continue the conversation with tool results
169+
$request->addMessage(new AssistantMessage($text, $toolCalls));
170+
$request->addMessage(new ToolResultMessage($toolResults));
171+
169172
$nextResponse = $this->sendRequest($request);
170173
yield from $this->processStream($nextResponse, $request, $depth + 1);
171174
}

‎tests/Fixtures/ollama/generate-text-with-multiple-tools-1.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"model":"qwen2.5:14b","created_at":"2025-01-29T23:30:02.912929Z","message":{"role":"assistant","content":"","tool_calls":[{"function":{"name":"search","arguments":{"query":"time of tigers game today in detroit"}}},{"function":{"name":"weather","arguments":{"city":"Detroit"}}}]},"done_reason":"stop","done":true,"total_duration":8327953791,"load_duration":29762166,"prompt_eval_count":235,"prompt_eval_duration":282000000,"eval_count":112,"eval_duration":8014000000}
1+
{"model":"qwen2.5:14b","created_at":"2025-06-09T18:55:26.684517Z","message":{"role":"assistant","content":"","tool_calls":[{"function":{"name":"search","arguments":{"query":"time of Detroit Tigers game today"}}},{"function":{"name":"weather","arguments":{"city":"Detroit"}}}]},"done_reason":"stop","done":true,"total_duration":8210142000,"load_duration":22224542,"prompt_eval_count":235,"prompt_eval_duration":269880958,"eval_count":111,"eval_duration":7916594250}

0 commit comments

Comments
 (0)