Skip to content

Commit 615ad93

Browse files
DZunkeDZunke
authored
feat: Vision support and message Splitting to allow multiple message implementations (#25)
* Allow usage of gpt visions api * Add tests * Rename content classes * Move content casting of user message to message object * Simplify message interface with content classes * Rename variable * Fix image describer after latest interface change * Some review adjustments
1 parent e443b48 commit 615ad93

19 files changed

+643
-158
lines changed

‎examples/image-describer.php

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
<?php
2+
3+
use PhpLlm\LlmChain\Chain;
4+
use PhpLlm\LlmChain\Message\Content\Image;
5+
use PhpLlm\LlmChain\Message\Message;
6+
use PhpLlm\LlmChain\Message\MessageBag;
7+
use PhpLlm\LlmChain\OpenAI\Model\Gpt;
8+
use PhpLlm\LlmChain\OpenAI\Model\Gpt\Version;
9+
use PhpLlm\LlmChain\OpenAI\Runtime\OpenAI;
10+
use Symfony\Component\HttpClient\HttpClient;
11+
12+
require_once dirname(__DIR__).'/vendor/autoload.php';
13+
14+
$runtime = new OpenAI(HttpClient::create(), getenv('OPENAI_API_KEY'));
15+
$llm = new Gpt($runtime, Version::gpt4oMini());
16+
17+
$chain = new Chain($llm);
18+
$messages = new MessageBag(
19+
Message::forSystem('You are an image analyzer bot that helps identify the content of images.'),
20+
Message::ofUser(
21+
'Describe the images as a comedian would do it.',
22+
new Image('https://upload.wikimedia.org/wikipedia/commons/thumb/3/31/Webysther_20160423_-_Elephpant.svg/350px-Webysther_20160423_-_Elephpant.svg.png'),
23+
new Image('https://upload.wikimedia.org/wikipedia/commons/thumb/3/37/African_Bush_Elephant.jpg/320px-African_Bush_Elephant.jpg'),
24+
),
25+
);
26+
$response = $chain->call($messages);
27+
28+
echo $response.PHP_EOL;

‎src/Message/AssistantMessage.php

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace PhpLlm\LlmChain\Message;
6+
7+
use PhpLlm\LlmChain\Response\ToolCall;
8+
9+
final readonly class AssistantMessage implements MessageInterface
10+
{
11+
/**
12+
* @param ?ToolCall[] $toolCalls
13+
*/
14+
public function __construct(
15+
public ?string $content = null,
16+
public ?array $toolCalls = null,
17+
) {
18+
}
19+
20+
public function getRole(): Role
21+
{
22+
return Role::Assistant;
23+
}
24+
25+
public function hasToolCalls(): bool
26+
{
27+
return null !== $this->toolCalls && 0 !== \count($this->toolCalls);
28+
}
29+
30+
/**
31+
* @return array{
32+
* role: Role::Assistant,
33+
* content: ?string,
34+
* tool_calls?: ToolCall[],
35+
* }
36+
*/
37+
public function jsonSerialize(): array
38+
{
39+
$array = [
40+
'role' => Role::Assistant,
41+
];
42+
43+
if (null !== $this->content) {
44+
$array['content'] = $this->content;
45+
}
46+
47+
if ($this->hasToolCalls()) {
48+
$array['tool_calls'] = $this->toolCalls;
49+
}
50+
51+
return $array;
52+
}
53+
}

‎src/Message/Content/ContentInterface.php

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace PhpLlm\LlmChain\Message\Content;
6+
7+
interface ContentInterface extends \JsonSerializable
8+
{
9+
}

‎src/Message/Content/Image.php

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace PhpLlm\LlmChain\Message\Content;
6+
7+
final readonly class Image implements ContentInterface
8+
{
9+
/**
10+
* @param string $url An URL like "http://localhost:3000/my-image.png" or a data url like "data:image/png;base64,iVBOR[...]"
11+
*/
12+
public function __construct(public string $url)
13+
{
14+
}
15+
16+
/**
17+
* @return array{type: 'image_url', image_url: array{url: string}}
18+
*/
19+
public function jsonSerialize(): array
20+
{
21+
return ['type' => 'image_url', 'image_url' => ['url' => $this->url]];
22+
}
23+
}

‎src/Message/Content/Text.php

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace PhpLlm\LlmChain\Message\Content;
6+
7+
final readonly class Text implements ContentInterface
8+
{
9+
public function __construct(public string $text)
10+
{
11+
}
12+
13+
/**
14+
* @return array{type: 'text', text: string}
15+
*/
16+
public function jsonSerialize(): array
17+
{
18+
return ['type' => 'text', 'text' => $this->text];
19+
}
20+
}

‎src/Message/Message.php

Lines changed: 18 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -4,94 +4,42 @@
44

55
namespace PhpLlm\LlmChain\Message;
66

7+
use PhpLlm\LlmChain\Message\Content\ContentInterface;
8+
use PhpLlm\LlmChain\Message\Content\Text;
79
use PhpLlm\LlmChain\Response\ToolCall;
810

9-
final readonly class Message implements \JsonSerializable
11+
final readonly class Message
1012
{
11-
/**
12-
* @param ?ToolCall[] $toolCalls
13-
*/
14-
public function __construct(
15-
public ?string $content,
16-
public Role $role,
17-
public ?array $toolCalls = null,
18-
) {
13+
// Disabled by default, just a bridge to the specific messages
14+
private function __construct()
15+
{
1916
}
2017

21-
public static function forSystem(string $content): self
18+
public static function forSystem(string $content): SystemMessage
2219
{
23-
return new self($content, Role::System);
20+
return new SystemMessage($content);
2421
}
2522

2623
/**
2724
* @param ?ToolCall[] $toolCalls
2825
*/
29-
public static function ofAssistant(?string $content = null, ?array $toolCalls = null): self
30-
{
31-
return new self($content, Role::Assistant, $toolCalls);
32-
}
33-
34-
public static function ofUser(string $content): self
26+
public static function ofAssistant(?string $content = null, ?array $toolCalls = null): AssistantMessage
3527
{
36-
return new self($content, Role::User);
28+
return new AssistantMessage($content, $toolCalls);
3729
}
3830

39-
public static function ofToolCall(ToolCall $toolCall, string $content): self
31+
public static function ofUser(string|ContentInterface ...$content): UserMessage
4032
{
41-
return new self($content, Role::ToolCall, [$toolCall]);
42-
}
33+
$content = \array_map(
34+
static fn (string|ContentInterface $entry) => \is_string($entry) ? new Text($entry) : $entry,
35+
$content,
36+
);
4337

44-
public function isSystem(): bool
45-
{
46-
return Role::System === $this->role;
38+
return new UserMessage(...$content);
4739
}
4840

49-
public function isAssistant(): bool
41+
public static function ofToolCall(ToolCall $toolCall, string $content): ToolCallMessage
5042
{
51-
return Role::Assistant === $this->role;
52-
}
53-
54-
public function isUser(): bool
55-
{
56-
return Role::User === $this->role;
57-
}
58-
59-
public function isToolCall(): bool
60-
{
61-
return Role::ToolCall === $this->role;
62-
}
63-
64-
public function hasToolCalls(): bool
65-
{
66-
return null !== $this->toolCalls && 0 !== count($this->toolCalls);
67-
}
68-
69-
/**
70-
* @return array{
71-
* role: 'system'|'assistant'|'user'|'tool',
72-
* content: ?string,
73-
* tool_calls?: ToolCall[],
74-
* tool_call_id?: string
75-
* }
76-
*/
77-
public function jsonSerialize(): array
78-
{
79-
$array = [
80-
'role' => $this->role->value,
81-
];
82-
83-
if (null !== $this->content) {
84-
$array['content'] = $this->content;
85-
}
86-
87-
if ($this->hasToolCalls() && $this->isToolCall()) {
88-
$array['tool_call_id'] = $this->toolCalls[0]->id;
89-
}
90-
91-
if ($this->hasToolCalls() && $this->isAssistant()) {
92-
$array['tool_calls'] = $this->toolCalls;
93-
}
94-
95-
return $array;
43+
return new ToolCallMessage($toolCall, $content);
9644
}
9745
}

‎src/Message/MessageBag.php

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,27 +5,27 @@
55
namespace PhpLlm\LlmChain\Message;
66

77
/**
8-
* @template-extends \ArrayObject<int, Message>
8+
* @template-extends \ArrayObject<int, MessageInterface>
99
*/
1010
final class MessageBag extends \ArrayObject implements \JsonSerializable
1111
{
12-
public function __construct(Message ...$messages)
12+
public function __construct(MessageInterface ...$messages)
1313
{
1414
parent::__construct(array_values($messages));
1515
}
1616

17-
public function getSystemMessage(): ?Message
17+
public function getSystemMessage(): ?SystemMessage
1818
{
1919
foreach ($this as $message) {
20-
if (Role::System === $message->role) {
20+
if ($message instanceof SystemMessage) {
2121
return $message;
2222
}
2323
}
2424

2525
return null;
2626
}
2727

28-
public function with(Message $message): self
28+
public function with(MessageInterface $message): self
2929
{
3030
$messages = clone $this;
3131
$messages->append($message);
@@ -45,13 +45,16 @@ public function withoutSystemMessage(): self
4545
{
4646
$messages = clone $this;
4747
$messages->exchangeArray(
48-
array_values(array_filter($messages->getArrayCopy(), fn (Message $message) => !$message->isSystem()))
48+
array_values(array_filter(
49+
$messages->getArrayCopy(),
50+
static fn (MessageInterface $message) => !$message instanceof SystemMessage,
51+
))
4952
);
5053

5154
return $messages;
5255
}
5356

54-
public function prepend(Message $message): self
57+
public function prepend(MessageInterface $message): self
5558
{
5659
$messages = clone $this;
5760
$messages->exchangeArray(array_merge([$message], $messages->getArrayCopy()));
@@ -60,7 +63,7 @@ public function prepend(Message $message): self
6063
}
6164

6265
/**
63-
* @return Message[]
66+
* @return MessageInterface[]
6467
*/
6568
public function jsonSerialize(): array
6669
{

‎src/Message/MessageInterface.php

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace PhpLlm\LlmChain\Message;
6+
7+
interface MessageInterface extends \JsonSerializable
8+
{
9+
public function getRole(): Role;
10+
}

‎src/Message/SystemMessage.php

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace PhpLlm\LlmChain\Message;
6+
7+
final readonly class SystemMessage implements MessageInterface
8+
{
9+
public function __construct(public string $content)
10+
{
11+
}
12+
13+
public function getRole(): Role
14+
{
15+
return Role::System;
16+
}
17+
18+
/**
19+
* @return array{
20+
* role: Role::System,
21+
* content: string
22+
* }
23+
*/
24+
public function jsonSerialize(): array
25+
{
26+
return [
27+
'role' => Role::System,
28+
'content' => $this->content,
29+
];
30+
}
31+
}

‎src/Message/ToolCallMessage.php

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace PhpLlm\LlmChain\Message;
6+
7+
use PhpLlm\LlmChain\Response\ToolCall;
8+
9+
final readonly class ToolCallMessage implements MessageInterface
10+
{
11+
public function __construct(
12+
public ToolCall $toolCall,
13+
public string $content,
14+
) {
15+
}
16+
17+
public function getRole(): Role
18+
{
19+
return Role::ToolCall;
20+
}
21+
22+
/**
23+
* @return array{
24+
* role: Role::ToolCall,
25+
* content: string,
26+
* tool_call_id: string,
27+
* }
28+
*/
29+
public function jsonSerialize(): array
30+
{
31+
return [
32+
'role' => Role::ToolCall,
33+
'content' => $this->content,
34+
'tool_call_id' => $this->toolCall->id,
35+
];
36+
}
37+
}

0 commit comments

Comments
 (0)