Skip to content

Commit c84c642

Browse files
chr-hertelchr-hertel
committed
feat: extend Claude support for images and pdf input
1 parent 8e6a7d0 commit c84c642

15 files changed

+310
-70
lines changed

‎README.md

+1
Original file line numberDiff line numberDiff line change
@@ -821,3 +821,4 @@ For testing multi-modal features, the repository contains binary media content,
821821

822822
* `tests/Fixture/image.jpg`: Chris F., Creative Commons, see [pexels.com](https://www.pexels.com/photo/blauer-und-gruner-elefant-mit-licht-1680755/)
823823
* `tests/Fixture/audio.mp3`: davidbain, Creative Commons, see [freesound.org](https://freesound.org/people/davidbain/sounds/136777/)
824+
* `tests/Ficture/document.pdf`: Chem8240ja, Public Domain, see [Wikipedia](https://en.m.wikipedia.org/wiki/File:Re_example.pdf)

‎examples/anthropic/image-input-binary.php

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
<?php
2+
3+
use PhpLlm\LlmChain\Bridge\Anthropic\Claude;
4+
use PhpLlm\LlmChain\Bridge\Anthropic\PlatformFactory;
5+
use PhpLlm\LlmChain\Chain;
6+
use PhpLlm\LlmChain\Model\Message\Content\Image;
7+
use PhpLlm\LlmChain\Model\Message\Message;
8+
use PhpLlm\LlmChain\Model\Message\MessageBag;
9+
use Symfony\Component\Dotenv\Dotenv;
10+
11+
require_once dirname(__DIR__, 2).'/vendor/autoload.php';
12+
(new Dotenv())->loadEnv(dirname(__DIR__, 2).'/.env');
13+
14+
if (empty($_ENV['ANTHROPIC_API_KEY'])) {
15+
echo 'Please set the ANTHROPIC_API_KEY environment variable.'.PHP_EOL;
16+
exit(1);
17+
}
18+
19+
$platform = PlatformFactory::create($_ENV['ANTHROPIC_API_KEY']);
20+
$llm = new Claude(Claude::SONNET_37);
21+
22+
$chain = new Chain($platform, $llm);
23+
$messages = new MessageBag(
24+
Message::forSystem('You are an image analyzer bot that helps identify the content of images.'),
25+
Message::ofUser(
26+
Image::fromFile(dirname(__DIR__, 2).'/tests/Fixture/image.jpg'),
27+
'Describe this image.',
28+
),
29+
);
30+
$response = $chain->call($messages);
31+
32+
echo $response->getContent().PHP_EOL;

‎examples/anthropic/image-input-url.php

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
<?php
2+
3+
use PhpLlm\LlmChain\Bridge\Anthropic\Claude;
4+
use PhpLlm\LlmChain\Bridge\Anthropic\PlatformFactory;
5+
use PhpLlm\LlmChain\Chain;
6+
use PhpLlm\LlmChain\Model\Message\Content\ImageUrl;
7+
use PhpLlm\LlmChain\Model\Message\Message;
8+
use PhpLlm\LlmChain\Model\Message\MessageBag;
9+
use Symfony\Component\Dotenv\Dotenv;
10+
11+
require_once dirname(__DIR__, 2).'/vendor/autoload.php';
12+
(new Dotenv())->loadEnv(dirname(__DIR__, 2).'/.env');
13+
14+
if (empty($_ENV['ANTHROPIC_API_KEY'])) {
15+
echo 'Please set the ANTHROPIC_API_KEY environment variable.'.PHP_EOL;
16+
exit(1);
17+
}
18+
19+
$platform = PlatformFactory::create($_ENV['ANTHROPIC_API_KEY']);
20+
$llm = new Claude(Claude::SONNET_37);
21+
22+
$chain = new Chain($platform, $llm);
23+
$messages = new MessageBag(
24+
Message::forSystem('You are an image analyzer bot that helps identify the content of images.'),
25+
Message::ofUser(
26+
new ImageUrl('https://upload.wikimedia.org/wikipedia/commons/a/a7/Camponotus_flavomarginatus_ant.jpg'),
27+
'Describe this image.',
28+
),
29+
);
30+
$response = $chain->call($messages);
31+
32+
echo $response->getContent().PHP_EOL;

‎examples/anthropic/pdf-input-binary.php

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
<?php
2+
3+
use PhpLlm\LlmChain\Bridge\Anthropic\Claude;
4+
use PhpLlm\LlmChain\Bridge\Anthropic\PlatformFactory;
5+
use PhpLlm\LlmChain\Chain;
6+
use PhpLlm\LlmChain\Model\Message\Content\Document;
7+
use PhpLlm\LlmChain\Model\Message\Message;
8+
use PhpLlm\LlmChain\Model\Message\MessageBag;
9+
use Symfony\Component\Dotenv\Dotenv;
10+
11+
require_once dirname(__DIR__, 2).'/vendor/autoload.php';
12+
(new Dotenv())->loadEnv(dirname(__DIR__, 2).'/.env');
13+
14+
if (empty($_ENV['ANTHROPIC_API_KEY'])) {
15+
echo 'Please set the ANTHROPIC_API_KEY environment variable.'.PHP_EOL;
16+
exit(1);
17+
}
18+
19+
$platform = PlatformFactory::create($_ENV['ANTHROPIC_API_KEY']);
20+
$llm = new Claude(Claude::SONNET_37);
21+
22+
$chain = new Chain($platform, $llm);
23+
$messages = new MessageBag(
24+
Message::ofUser(
25+
Document::fromFile(dirname(__DIR__, 2).'/tests/Fixture/document.pdf'),
26+
'What is this document about?',
27+
),
28+
);
29+
$response = $chain->call($messages);
30+
31+
echo $response->getContent().PHP_EOL;

‎examples/anthropic/pdf-input-url.php

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
<?php
2+
3+
use PhpLlm\LlmChain\Bridge\Anthropic\Claude;
4+
use PhpLlm\LlmChain\Bridge\Anthropic\PlatformFactory;
5+
use PhpLlm\LlmChain\Chain;
6+
use PhpLlm\LlmChain\Model\Message\Content\DocumentUrl;
7+
use PhpLlm\LlmChain\Model\Message\Message;
8+
use PhpLlm\LlmChain\Model\Message\MessageBag;
9+
use Symfony\Component\Dotenv\Dotenv;
10+
11+
require_once dirname(__DIR__, 2).'/vendor/autoload.php';
12+
(new Dotenv())->loadEnv(dirname(__DIR__, 2).'/.env');
13+
14+
if (empty($_ENV['ANTHROPIC_API_KEY'])) {
15+
echo 'Please set the ANTHROPIC_API_KEY environment variable.'.PHP_EOL;
16+
exit(1);
17+
}
18+
19+
$platform = PlatformFactory::create($_ENV['ANTHROPIC_API_KEY']);
20+
$llm = new Claude(Claude::SONNET_37);
21+
22+
$chain = new Chain($platform, $llm);
23+
$messages = new MessageBag(
24+
Message::ofUser(
25+
new DocumentUrl('https://upload.wikimedia.org/wikipedia/commons/2/20/Re_example.pdf'),
26+
'What is this document about?',
27+
),
28+
);
29+
$response = $chain->call($messages);
30+
31+
echo $response->getContent().PHP_EOL;

‎examples/anthropic/toolbox-stream.php

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
<?php
2+
3+
use PhpLlm\LlmChain\Bridge\Anthropic\Claude;
4+
use PhpLlm\LlmChain\Bridge\Anthropic\PlatformFactory;
5+
use PhpLlm\LlmChain\Chain;
6+
use PhpLlm\LlmChain\Chain\Toolbox\ChainProcessor;
7+
use PhpLlm\LlmChain\Chain\Toolbox\Tool\YouTubeTranscriber;
8+
use PhpLlm\LlmChain\Chain\Toolbox\Toolbox;
9+
use PhpLlm\LlmChain\Model\Message\Message;
10+
use PhpLlm\LlmChain\Model\Message\MessageBag;
11+
use Symfony\Component\Dotenv\Dotenv;
12+
use Symfony\Component\HttpClient\HttpClient;
13+
14+
require_once dirname(__DIR__, 2).'/vendor/autoload.php';
15+
(new Dotenv())->loadEnv(dirname(__DIR__, 2).'/.env');
16+
17+
if (empty($_ENV['ANTHROPIC_API_KEY'])) {
18+
echo 'Please set the ANTHROPIC_API_KEY environment variable.'.PHP_EOL;
19+
exit(1);
20+
}
21+
22+
$platform = PlatformFactory::create($_ENV['ANTHROPIC_API_KEY']);
23+
$llm = new Claude();
24+
25+
$transcriber = new YouTubeTranscriber(HttpClient::create());
26+
$toolbox = Toolbox::create($transcriber);
27+
$processor = new ChainProcessor($toolbox);
28+
$chain = new Chain($platform, $llm, [$processor], [$processor]);
29+
30+
$messages = new MessageBag(Message::ofUser('Please summarize this video for me: https://www.youtube.com/watch?v=6uXW-ulpj0s'));
31+
$response = $chain->call($messages, [
32+
'stream' => true, // enable streaming of response text
33+
]);
34+
35+
foreach ($response->getContent() as $word) {
36+
echo $word;
37+
}
38+
echo PHP_EOL;

‎src/Bridge/Anthropic/Claude.php

+1-1
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ public function supportsAudioInput(): bool
4242

4343
public function supportsImageInput(): bool
4444
{
45-
return false; // it does, but implementation here is still open.
45+
return true;
4646
}
4747

4848
public function supportsStreaming(): bool

‎src/Bridge/Anthropic/ModelHandler.php renamed to ‎src/Bridge/Anthropic/ModelClient.php

+2-61
Original file line numberDiff line numberDiff line change
@@ -5,27 +5,19 @@
55
namespace PhpLlm\LlmChain\Bridge\Anthropic;
66

77
use PhpLlm\LlmChain\Chain\Toolbox\Metadata;
8-
use PhpLlm\LlmChain\Exception\RuntimeException;
98
use PhpLlm\LlmChain\Model\Message\AssistantMessage;
109
use PhpLlm\LlmChain\Model\Message\MessageBagInterface;
1110
use PhpLlm\LlmChain\Model\Message\MessageInterface;
1211
use PhpLlm\LlmChain\Model\Message\ToolCallMessage;
1312
use PhpLlm\LlmChain\Model\Model;
14-
use PhpLlm\LlmChain\Model\Response\ResponseInterface as LlmResponse;
15-
use PhpLlm\LlmChain\Model\Response\StreamResponse;
16-
use PhpLlm\LlmChain\Model\Response\TextResponse;
1713
use PhpLlm\LlmChain\Model\Response\ToolCall;
18-
use PhpLlm\LlmChain\Model\Response\ToolCallResponse;
19-
use PhpLlm\LlmChain\Platform\ModelClient;
20-
use PhpLlm\LlmChain\Platform\ResponseConverter;
21-
use Symfony\Component\HttpClient\Chunk\ServerSentEvent;
14+
use PhpLlm\LlmChain\Platform\ModelClient as PlatformModelClient;
2215
use Symfony\Component\HttpClient\EventSourceHttpClient;
23-
use Symfony\Component\HttpClient\Exception\JsonException;
2416
use Symfony\Contracts\HttpClient\HttpClientInterface;
2517
use Symfony\Contracts\HttpClient\ResponseInterface;
2618
use Webmozart\Assert\Assert;
2719

28-
final readonly class ModelHandler implements ModelClient, ResponseConverter
20+
final readonly class ModelClient implements PlatformModelClient
2921
{
3022
private EventSourceHttpClient $httpClient;
3123

@@ -108,55 +100,4 @@ public function request(Model $model, object|array|string $input, array $options
108100
'json' => array_merge($options, $body),
109101
]);
110102
}
111-
112-
public function convert(ResponseInterface $response, array $options = []): LlmResponse
113-
{
114-
if ($options['stream'] ?? false) {
115-
return new StreamResponse($this->convertStream($response));
116-
}
117-
118-
$data = $response->toArray();
119-
120-
if (!isset($data['content']) || 0 === count($data['content'])) {
121-
throw new RuntimeException('Response does not contain any content');
122-
}
123-
124-
if (!isset($data['content'][0]['text'])) {
125-
throw new RuntimeException('Response content does not contain any text');
126-
}
127-
128-
$toolCalls = [];
129-
foreach ($data['content'] as $content) {
130-
if ('tool_use' === $content['type']) {
131-
$toolCalls[] = new ToolCall($content['id'], $content['name'], $content['input']);
132-
}
133-
}
134-
if (!empty($toolCalls)) {
135-
return new ToolCallResponse(...$toolCalls);
136-
}
137-
138-
return new TextResponse($data['content'][0]['text']);
139-
}
140-
141-
private function convertStream(ResponseInterface $response): \Generator
142-
{
143-
foreach ((new EventSourceHttpClient())->stream($response) as $chunk) {
144-
if (!$chunk instanceof ServerSentEvent || '[DONE]' === $chunk->getData()) {
145-
continue;
146-
}
147-
148-
try {
149-
$data = $chunk->getArrayData();
150-
} catch (JsonException) {
151-
// try catch only needed for Symfony 6.4
152-
continue;
153-
}
154-
155-
if ('content_block_delta' != $data['type'] || !isset($data['delta']['text'])) {
156-
continue;
157-
}
158-
159-
yield $data['delta']['text'];
160-
}
161-
}
162103
}

‎src/Bridge/Anthropic/PlatformFactory.php

+1-2
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,7 @@ public static function create(
1717
?HttpClientInterface $httpClient = null,
1818
): Platform {
1919
$httpClient = $httpClient instanceof EventSourceHttpClient ? $httpClient : new EventSourceHttpClient($httpClient);
20-
$responseHandler = new ModelHandler($httpClient, $apiKey, $version);
2120

22-
return new Platform([$responseHandler], [$responseHandler]);
21+
return new Platform([new ModelClient($httpClient, $apiKey, $version)], [new ResponseConverter()]);
2322
}
2423
}

‎src/Bridge/Anthropic/ResponseConverter.php

+78
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace PhpLlm\LlmChain\Bridge\Anthropic;
6+
7+
use PhpLlm\LlmChain\Exception\RuntimeException;
8+
use PhpLlm\LlmChain\Model\Message\MessageBagInterface;
9+
use PhpLlm\LlmChain\Model\Model;
10+
use PhpLlm\LlmChain\Model\Response\ResponseInterface as LlmResponse;
11+
use PhpLlm\LlmChain\Model\Response\StreamResponse;
12+
use PhpLlm\LlmChain\Model\Response\TextResponse;
13+
use PhpLlm\LlmChain\Model\Response\ToolCall;
14+
use PhpLlm\LlmChain\Model\Response\ToolCallResponse;
15+
use PhpLlm\LlmChain\Platform\ResponseConverter as PlatformResponseConverter;
16+
use Symfony\Component\HttpClient\Chunk\ServerSentEvent;
17+
use Symfony\Component\HttpClient\EventSourceHttpClient;
18+
use Symfony\Component\HttpClient\Exception\JsonException;
19+
use Symfony\Contracts\HttpClient\ResponseInterface;
20+
21+
final readonly class ResponseConverter implements PlatformResponseConverter
22+
{
23+
public function supports(Model $model, array|string|object $input): bool
24+
{
25+
return $model instanceof Claude && $input instanceof MessageBagInterface;
26+
}
27+
28+
public function convert(ResponseInterface $response, array $options = []): LlmResponse
29+
{
30+
if ($options['stream'] ?? false) {
31+
return new StreamResponse($this->convertStream($response));
32+
}
33+
34+
$data = $response->toArray();
35+
36+
if (!isset($data['content']) || 0 === count($data['content'])) {
37+
throw new RuntimeException('Response does not contain any content');
38+
}
39+
40+
if (!isset($data['content'][0]['text'])) {
41+
throw new RuntimeException('Response content does not contain any text');
42+
}
43+
44+
$toolCalls = [];
45+
foreach ($data['content'] as $content) {
46+
if ('tool_use' === $content['type']) {
47+
$toolCalls[] = new ToolCall($content['id'], $content['name'], $content['input']);
48+
}
49+
}
50+
if (!empty($toolCalls)) {
51+
return new ToolCallResponse(...$toolCalls);
52+
}
53+
54+
return new TextResponse($data['content'][0]['text']);
55+
}
56+
57+
private function convertStream(ResponseInterface $response): \Generator
58+
{
59+
foreach ((new EventSourceHttpClient())->stream($response) as $chunk) {
60+
if (!$chunk instanceof ServerSentEvent || '[DONE]' === $chunk->getData()) {
61+
continue;
62+
}
63+
64+
try {
65+
$data = $chunk->getArrayData();
66+
} catch (JsonException) {
67+
// try catch only needed for Symfony 6.4
68+
continue;
69+
}
70+
71+
if ('content_block_delta' != $data['type'] || !isset($data['delta']['text'])) {
72+
continue;
73+
}
74+
75+
yield $data['delta']['text'];
76+
}
77+
}
78+
}

‎src/Model/Message/Content/Document.php

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace PhpLlm\LlmChain\Model\Message\Content;
6+
7+
final readonly class Document extends File implements Content
8+
{
9+
/**
10+
* @return array{type: 'document', source: array{type: 'base64', media_type: string, data: string}}
11+
*/
12+
public function jsonSerialize(): array
13+
{
14+
return [
15+
'type' => 'document',
16+
'source' => [
17+
'type' => 'base64',
18+
'media_type' => $this->getFormat(),
19+
'data' => $this->asBase64(),
20+
],
21+
];
22+
}
23+
}

0 commit comments

Comments
 (0)