diff --git a/README.md b/README.md index 5d5057b..426c108 100644 --- a/README.md +++ b/README.md @@ -821,3 +821,4 @@ For testing multi-modal features, the repository contains binary media content, * `tests/Fixture/image.jpg`: Chris F., Creative Commons, see [pexels.com](https://www.pexels.com/photo/blauer-und-gruner-elefant-mit-licht-1680755/) * `tests/Fixture/audio.mp3`: davidbain, Creative Commons, see [freesound.org](https://freesound.org/people/davidbain/sounds/136777/) +* `tests/Fixture/document.pdf`: Chem8240ja, Public Domain, see [Wikipedia](https://en.m.wikipedia.org/wiki/File:Re_example.pdf) diff --git a/examples/anthropic/image-input-binary.php b/examples/anthropic/image-input-binary.php new file mode 100644 index 0000000..45fdd60 --- /dev/null +++ b/examples/anthropic/image-input-binary.php @@ -0,0 +1,32 @@ +loadEnv(dirname(__DIR__, 2).'/.env'); + +if (empty($_ENV['ANTHROPIC_API_KEY'])) { + echo 'Please set the ANTHROPIC_API_KEY environment variable.'.PHP_EOL; + exit(1); +} + +$platform = PlatformFactory::create($_ENV['ANTHROPIC_API_KEY']); +$llm = new Claude(Claude::SONNET_37); + +$chain = new Chain($platform, $llm); +$messages = new MessageBag( + Message::forSystem('You are an image analyzer bot that helps identify the content of images.'), + Message::ofUser( + Image::fromFile(dirname(__DIR__, 2).'/tests/Fixture/image.jpg'), + 'Describe this image.', + ), +); +$response = $chain->call($messages); + +echo $response->getContent().PHP_EOL; diff --git a/examples/anthropic/image-input-url.php b/examples/anthropic/image-input-url.php new file mode 100644 index 0000000..bc87d33 --- /dev/null +++ b/examples/anthropic/image-input-url.php @@ -0,0 +1,32 @@ +loadEnv(dirname(__DIR__, 2).'/.env'); + +if (empty($_ENV['ANTHROPIC_API_KEY'])) { + echo 'Please set the ANTHROPIC_API_KEY environment variable.'.PHP_EOL; + exit(1); +} + +$platform = PlatformFactory::create($_ENV['ANTHROPIC_API_KEY']); +$llm = new Claude(Claude::SONNET_37); + +$chain = new Chain($platform, $llm); +$messages = new MessageBag( + Message::forSystem('You are an image analyzer bot that helps identify the content of images.'), + Message::ofUser( + new ImageUrl('https://upload.wikimedia.org/wikipedia/commons/a/a7/Camponotus_flavomarginatus_ant.jpg'), + 'Describe this image.', + ), +); +$response = $chain->call($messages); + +echo $response->getContent().PHP_EOL; diff --git a/examples/anthropic/pdf-input-binary.php b/examples/anthropic/pdf-input-binary.php new file mode 100644 index 0000000..ea29fc6 --- /dev/null +++ b/examples/anthropic/pdf-input-binary.php @@ -0,0 +1,31 @@ +loadEnv(dirname(__DIR__, 2).'/.env'); + +if (empty($_ENV['ANTHROPIC_API_KEY'])) { + echo 'Please set the ANTHROPIC_API_KEY environment variable.'.PHP_EOL; + exit(1); +} + +$platform = PlatformFactory::create($_ENV['ANTHROPIC_API_KEY']); +$llm = new Claude(Claude::SONNET_37); + +$chain = new Chain($platform, $llm); +$messages = new MessageBag( + Message::ofUser( + Document::fromFile(dirname(__DIR__, 2).'/tests/Fixture/document.pdf'), + 'What is this document about?', + ), +); +$response = $chain->call($messages); + +echo $response->getContent().PHP_EOL; diff --git a/examples/anthropic/pdf-input-url.php b/examples/anthropic/pdf-input-url.php new file mode 100644 index 0000000..e7aa2d0 --- /dev/null +++ b/examples/anthropic/pdf-input-url.php @@ -0,0 +1,31 @@ +loadEnv(dirname(__DIR__, 2).'/.env'); + +if (empty($_ENV['ANTHROPIC_API_KEY'])) { + echo 'Please set the ANTHROPIC_API_KEY environment variable.'.PHP_EOL; + exit(1); +} + +$platform = PlatformFactory::create($_ENV['ANTHROPIC_API_KEY']); +$llm = new Claude(Claude::SONNET_37); + +$chain = new Chain($platform, $llm); +$messages = new MessageBag( + Message::ofUser( + new DocumentUrl('https://upload.wikimedia.org/wikipedia/commons/2/20/Re_example.pdf'), + 'What is this document about?', + ), +); +$response = $chain->call($messages); + +echo $response->getContent().PHP_EOL; diff --git a/examples/anthropic/toolbox-stream.php b/examples/anthropic/toolbox-stream.php new file mode 100644 index 0000000..9ce2b41 --- /dev/null +++ b/examples/anthropic/toolbox-stream.php @@ -0,0 +1,38 @@ +loadEnv(dirname(__DIR__, 2).'/.env'); + +if (empty($_ENV['ANTHROPIC_API_KEY'])) { + echo 'Please set the ANTHROPIC_API_KEY environment variable.'.PHP_EOL; + exit(1); +} + +$platform = PlatformFactory::create($_ENV['ANTHROPIC_API_KEY']); +$llm = new Claude(); + +$transcriber = new YouTubeTranscriber(HttpClient::create()); +$toolbox = Toolbox::create($transcriber); +$processor = new ChainProcessor($toolbox); +$chain = new Chain($platform, $llm, [$processor], [$processor]); + +$messages = new MessageBag(Message::ofUser('Please summarize this video for me: https://www.youtube.com/watch?v=6uXW-ulpj0s')); +$response = $chain->call($messages, [ + 'stream' => true, // enable streaming of response text +]); + +foreach ($response->getContent() as $word) { + echo $word; +} +echo PHP_EOL; diff --git a/src/Bridge/Anthropic/Claude.php b/src/Bridge/Anthropic/Claude.php index 3096a4b..6d75ec6 100644 --- a/src/Bridge/Anthropic/Claude.php +++ b/src/Bridge/Anthropic/Claude.php @@ -42,7 +42,7 @@ public function supportsAudioInput(): bool public function supportsImageInput(): bool { - return false; // it does, but implementation here is still open. + return true; } public function supportsStreaming(): bool diff --git a/src/Bridge/Anthropic/ModelHandler.php b/src/Bridge/Anthropic/ModelClient.php similarity index 62% rename from src/Bridge/Anthropic/ModelHandler.php rename to src/Bridge/Anthropic/ModelClient.php index 07f631f..b0dae82 100644 --- a/src/Bridge/Anthropic/ModelHandler.php +++ b/src/Bridge/Anthropic/ModelClient.php @@ -5,27 +5,19 @@ namespace PhpLlm\LlmChain\Bridge\Anthropic; use PhpLlm\LlmChain\Chain\Toolbox\Metadata; -use PhpLlm\LlmChain\Exception\RuntimeException; use PhpLlm\LlmChain\Model\Message\AssistantMessage; use PhpLlm\LlmChain\Model\Message\MessageBagInterface; use PhpLlm\LlmChain\Model\Message\MessageInterface; use PhpLlm\LlmChain\Model\Message\ToolCallMessage; use PhpLlm\LlmChain\Model\Model; -use PhpLlm\LlmChain\Model\Response\ResponseInterface as LlmResponse; -use PhpLlm\LlmChain\Model\Response\StreamResponse; -use PhpLlm\LlmChain\Model\Response\TextResponse; use PhpLlm\LlmChain\Model\Response\ToolCall; -use PhpLlm\LlmChain\Model\Response\ToolCallResponse; -use PhpLlm\LlmChain\Platform\ModelClient; -use PhpLlm\LlmChain\Platform\ResponseConverter; -use Symfony\Component\HttpClient\Chunk\ServerSentEvent; +use PhpLlm\LlmChain\Platform\ModelClient as PlatformModelClient; use Symfony\Component\HttpClient\EventSourceHttpClient; -use Symfony\Component\HttpClient\Exception\JsonException; use Symfony\Contracts\HttpClient\HttpClientInterface; use Symfony\Contracts\HttpClient\ResponseInterface; use Webmozart\Assert\Assert; -final readonly class ModelHandler implements ModelClient, ResponseConverter +final readonly class ModelClient implements PlatformModelClient { private EventSourceHttpClient $httpClient; @@ -108,55 +100,4 @@ public function request(Model $model, object|array|string $input, array $options 'json' => array_merge($options, $body), ]); } - - public function convert(ResponseInterface $response, array $options = []): LlmResponse - { - if ($options['stream'] ?? false) { - return new StreamResponse($this->convertStream($response)); - } - - $data = $response->toArray(); - - if (!isset($data['content']) || 0 === count($data['content'])) { - throw new RuntimeException('Response does not contain any content'); - } - - if (!isset($data['content'][0]['text'])) { - throw new RuntimeException('Response content does not contain any text'); - } - - $toolCalls = []; - foreach ($data['content'] as $content) { - if ('tool_use' === $content['type']) { - $toolCalls[] = new ToolCall($content['id'], $content['name'], $content['input']); - } - } - if (!empty($toolCalls)) { - return new ToolCallResponse(...$toolCalls); - } - - return new TextResponse($data['content'][0]['text']); - } - - private function convertStream(ResponseInterface $response): \Generator - { - foreach ((new EventSourceHttpClient())->stream($response) as $chunk) { - if (!$chunk instanceof ServerSentEvent || '[DONE]' === $chunk->getData()) { - continue; - } - - try { - $data = $chunk->getArrayData(); - } catch (JsonException) { - // try catch only needed for Symfony 6.4 - continue; - } - - if ('content_block_delta' != $data['type'] || !isset($data['delta']['text'])) { - continue; - } - - yield $data['delta']['text']; - } - } } diff --git a/src/Bridge/Anthropic/PlatformFactory.php b/src/Bridge/Anthropic/PlatformFactory.php index 644c462..f68ef49 100644 --- a/src/Bridge/Anthropic/PlatformFactory.php +++ b/src/Bridge/Anthropic/PlatformFactory.php @@ -17,8 +17,7 @@ public static function create( ?HttpClientInterface $httpClient = null, ): Platform { $httpClient = $httpClient instanceof EventSourceHttpClient ? $httpClient : new EventSourceHttpClient($httpClient); - $responseHandler = new ModelHandler($httpClient, $apiKey, $version); - return new Platform([$responseHandler], [$responseHandler]); + return new Platform([new ModelClient($httpClient, $apiKey, $version)], [new ResponseConverter()]); } } diff --git a/src/Bridge/Anthropic/ResponseConverter.php b/src/Bridge/Anthropic/ResponseConverter.php new file mode 100644 index 0000000..451f0d9 --- /dev/null +++ b/src/Bridge/Anthropic/ResponseConverter.php @@ -0,0 +1,78 @@ +convertStream($response)); + } + + $data = $response->toArray(); + + if (!isset($data['content']) || 0 === count($data['content'])) { + throw new RuntimeException('Response does not contain any content'); + } + + if (!isset($data['content'][0]['text'])) { + throw new RuntimeException('Response content does not contain any text'); + } + + $toolCalls = []; + foreach ($data['content'] as $content) { + if ('tool_use' === $content['type']) { + $toolCalls[] = new ToolCall($content['id'], $content['name'], $content['input']); + } + } + if (!empty($toolCalls)) { + return new ToolCallResponse(...$toolCalls); + } + + return new TextResponse($data['content'][0]['text']); + } + + private function convertStream(ResponseInterface $response): \Generator + { + foreach ((new EventSourceHttpClient())->stream($response) as $chunk) { + if (!$chunk instanceof ServerSentEvent || '[DONE]' === $chunk->getData()) { + continue; + } + + try { + $data = $chunk->getArrayData(); + } catch (JsonException) { + // try catch only needed for Symfony 6.4 + continue; + } + + if ('content_block_delta' != $data['type'] || !isset($data['delta']['text'])) { + continue; + } + + yield $data['delta']['text']; + } + } +} diff --git a/src/Model/Message/Content/Document.php b/src/Model/Message/Content/Document.php new file mode 100644 index 0000000..4ee1bd2 --- /dev/null +++ b/src/Model/Message/Content/Document.php @@ -0,0 +1,23 @@ + 'document', + 'source' => [ + 'type' => 'base64', + 'media_type' => $this->getFormat(), + 'data' => $this->asBase64(), + ], + ]; + } +} diff --git a/src/Model/Message/Content/DocumentUrl.php b/src/Model/Message/Content/DocumentUrl.php new file mode 100644 index 0000000..5534f6b --- /dev/null +++ b/src/Model/Message/Content/DocumentUrl.php @@ -0,0 +1,27 @@ + 'document', + 'source' => [ + 'type' => 'url', + 'url' => $this->url, + ], + ]; + } +} diff --git a/src/Model/Message/Content/Image.php b/src/Model/Message/Content/Image.php index 330b65c..5dd97a0 100644 --- a/src/Model/Message/Content/Image.php +++ b/src/Model/Message/Content/Image.php @@ -7,13 +7,17 @@ final readonly class Image extends File implements Content { /** - * @return array{type: 'image_url', image_url: array{url: string}} + * @return array{type: 'image', source: array{type: 'base64', media_type: string, data: string}} */ public function jsonSerialize(): array { return [ - 'type' => 'image_url', - 'image_url' => ['url' => $this->asDataUrl()], + 'type' => 'image', + 'source' => [ + 'type' => 'base64', + 'media_type' => $this->getFormat(), + 'data' => $this->asBase64(), + ], ]; } } diff --git a/src/Model/Message/Content/ImageUrl.php b/src/Model/Message/Content/ImageUrl.php index a9e6379..fbcc61a 100644 --- a/src/Model/Message/Content/ImageUrl.php +++ b/src/Model/Message/Content/ImageUrl.php @@ -12,13 +12,16 @@ public function __construct( } /** - * @return array{type: 'image_url', image_url: array{url: string}} + * @return array{type: 'image', source: array{type: 'url', url: string}} */ public function jsonSerialize(): array { return [ - 'type' => 'image_url', - 'image_url' => ['url' => $this->url], + 'type' => 'image', + 'source' => [ + 'type' => 'url', + 'url' => $this->url, + ], ]; } } diff --git a/tests/Fixture/document.pdf b/tests/Fixture/document.pdf new file mode 100644 index 0000000..00172f2 Binary files /dev/null and b/tests/Fixture/document.pdf differ