Skip to content

Commit ec37a3a

Browse files
chr-hertelchr-hertel
authored
feat: introduce optional fault tolerant toolbox (#213)
1 parent d7ecb4d commit ec37a3a

15 files changed

+229
-67
lines changed

‎README.md

+18-4
Original file line numberDiff line numberDiff line change
@@ -139,9 +139,6 @@ Tool calling can be enabled by registering the processors in the chain:
139139
use PhpLlm\LlmChain\Chain\ToolBox\ChainProcessor;
140140
use PhpLlm\LlmChain\Chain\ToolBox\ToolAnalyzer;
141141
use PhpLlm\LlmChain\Chain\ToolBox\ToolBox;
142-
use Symfony\Component\Serializer\Encoder\JsonEncoder;
143-
use Symfony\Component\Serializer\Normalizer\ObjectNormalizer;
144-
use Symfony\Component\Serializer\Serializer;
145142

146143
// Platform & LLM instantiation
147144

@@ -180,7 +177,6 @@ You can configure the method to be called by the LLM with the `#[AsTool]` attrib
180177
```php
181178
use PhpLlm\LlmChain\ToolBox\Attribute\AsTool;
182179

183-
184180
#[AsTool(name: 'weather_current', description: 'get current weather for a location', method: 'current')]
185181
#[AsTool(name: 'weather_forecast', description: 'get weather forecast for a location', method: 'forecast')]
186182
final readonly class OpenMeteo
@@ -231,6 +227,24 @@ See attribute class [ToolParameter](src/Chain/ToolBox/Attribute/ToolParameter.ph
231227
> [!NOTE]
232228
> Please be aware, that this is only converted in a JSON Schema for the LLM to respect, but not validated by LLM Chain.
233229
230+
#### Fault Tolerance
231+
232+
To gracefully handle errors that occur during tool calling, e.g. wrong tool names or runtime errors, you can use the
233+
`FaultTolerantToolBox` as a decorator for the `ToolBox`. It will catch the exceptions and return readable error messages
234+
to the LLM.
235+
236+
```php
237+
use PhpLlm\LlmChain\Chain\ToolBox\ChainProcessor;
238+
use PhpLlm\LlmChain\Chain\ToolBox\FaultTolerantToolBox;
239+
240+
// Platform, LLM & ToolBox instantiation
241+
242+
$toolBox = new FaultTolerantToolBox($innerToolBox);
243+
$toolProcessor = new ChainProcessor($toolBox);
244+
245+
$chain = new Chain($platform, $llm, inputProcessor: [$toolProcessor], outputProcessor: [$toolProcessor]);
246+
```
247+
234248
#### Tool Result Interception
235249

236250
To react to the result of a tool, you can implement an EventListener or EventSubscriber, that listens to the

‎src/Chain/ToolBox/Exception/ExceptionInterface.php

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace PhpLlm\LlmChain\Chain\ToolBox\Exception;
6+
7+
use PhpLlm\LlmChain\Exception\ExceptionInterface as BaseExceptionInterface;
8+
9+
interface ExceptionInterface extends BaseExceptionInterface
10+
{
11+
}

‎src/Chain/ToolBox/Exception/ToolConfigurationException.php

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace PhpLlm\LlmChain\Chain\ToolBox\Exception;
6+
7+
use PhpLlm\LlmChain\Chain\ToolBox\Attribute\AsTool;
8+
use PhpLlm\LlmChain\Exception\InvalidArgumentException;
9+
10+
final class ToolConfigurationException extends InvalidArgumentException implements ExceptionInterface
11+
{
12+
public static function missingAttribute(string $className): self
13+
{
14+
return new self(sprintf('The class "%s" is not a tool, please add %s attribute.', $className, AsTool::class));
15+
}
16+
17+
public static function invalidMethod(string $toolClass, string $methodName): self
18+
{
19+
return new self(sprintf('Method "%s" not found in tool "%s".', $methodName, $toolClass));
20+
}
21+
}

‎src/Chain/ToolBox/Exception/ToolExecutionException.php

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace PhpLlm\LlmChain\Chain\ToolBox\Exception;
6+
7+
use PhpLlm\LlmChain\Model\Response\ToolCall;
8+
9+
final class ToolExecutionException extends \RuntimeException implements ExceptionInterface
10+
{
11+
public ?ToolCall $toolCall = null;
12+
13+
public static function executionFailed(ToolCall $toolCall, \Throwable $previous): self
14+
{
15+
$exception = new self(sprintf('Execution of tool "%s" failed with error: %s', $toolCall->name, $previous->getMessage()), previous: $previous);
16+
$exception->toolCall = $toolCall;
17+
18+
return $exception;
19+
}
20+
}

‎src/Chain/ToolBox/Exception/ToolNotFoundException.php

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace PhpLlm\LlmChain\Chain\ToolBox\Exception;
6+
7+
use PhpLlm\LlmChain\Model\Response\ToolCall;
8+
9+
final class ToolNotFoundException extends \RuntimeException implements ExceptionInterface
10+
{
11+
public ?ToolCall $toolCall = null;
12+
13+
public static function notFoundForToolCall(ToolCall $toolCall): self
14+
{
15+
$exception = new self(sprintf('Tool not found for call: %s.', $toolCall->name));
16+
$exception->toolCall = $toolCall;
17+
18+
return $exception;
19+
}
20+
}

‎src/Chain/ToolBox/FaultTolerantToolBox.php

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace PhpLlm\LlmChain\Chain\ToolBox;
6+
7+
use PhpLlm\LlmChain\Chain\ToolBox\Exception\ToolExecutionException;
8+
use PhpLlm\LlmChain\Chain\ToolBox\Exception\ToolNotFoundException;
9+
use PhpLlm\LlmChain\Model\Response\ToolCall;
10+
11+
/**
12+
* Catches exceptions thrown by the inner tool box and returns error messages for the LLM instead.
13+
*/
14+
final readonly class FaultTolerantToolBox implements ToolBoxInterface
15+
{
16+
public function __construct(
17+
private ToolBoxInterface $innerToolBox,
18+
) {
19+
}
20+
21+
public function getMap(): array
22+
{
23+
return $this->innerToolBox->getMap();
24+
}
25+
26+
public function execute(ToolCall $toolCall): mixed
27+
{
28+
try {
29+
return $this->innerToolBox->execute($toolCall);
30+
} catch (ToolExecutionException $e) {
31+
return sprintf('An error occurred while executing tool "%s".', $e->toolCall->name);
32+
} catch (ToolNotFoundException) {
33+
$names = array_map(fn (Metadata $metadata) => $metadata->name, $this->getMap());
34+
35+
return sprintf('Tool "%s" was not found, please use one of these: %s', $toolCall->name, implode(', ', $names));
36+
}
37+
}
38+
}

‎src/Chain/ToolBox/ParameterAnalyzer.php

+2-2
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
namespace PhpLlm\LlmChain\Chain\ToolBox;
66

77
use PhpLlm\LlmChain\Chain\ToolBox\Attribute\ToolParameter;
8-
use PhpLlm\LlmChain\Exception\ToolBoxException;
8+
use PhpLlm\LlmChain\Chain\ToolBox\Exception\ToolConfigurationException;
99

1010
/**
1111
* @phpstan-type ParameterDefinition array{
@@ -46,7 +46,7 @@ public function getDefinition(string $className, string $methodName): ?array
4646
try {
4747
$reflection = new \ReflectionMethod($className, $methodName);
4848
} catch (\ReflectionException) {
49-
throw ToolBoxException::invalidMethod($className, $methodName);
49+
throw ToolConfigurationException::invalidMethod($className, $methodName);
5050
}
5151
$parameters = $reflection->getParameters();
5252

‎src/Chain/ToolBox/ToolAnalyzer.php

+2-2
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
namespace PhpLlm\LlmChain\Chain\ToolBox;
66

77
use PhpLlm\LlmChain\Chain\ToolBox\Attribute\AsTool;
8-
use PhpLlm\LlmChain\Exception\InvalidToolImplementation;
8+
use PhpLlm\LlmChain\Chain\ToolBox\Exception\ToolConfigurationException;
99

1010
final readonly class ToolAnalyzer
1111
{
@@ -25,7 +25,7 @@ public function getMetadata(string $className): iterable
2525
$attributes = $reflectionClass->getAttributes(AsTool::class);
2626

2727
if (0 === count($attributes)) {
28-
throw InvalidToolImplementation::missingAttribute($className);
28+
throw ToolConfigurationException::missingAttribute($className);
2929
}
3030

3131
foreach ($attributes as $attribute) {

‎src/Chain/ToolBox/ToolBox.php

+4-3
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@
44

55
namespace PhpLlm\LlmChain\Chain\ToolBox;
66

7-
use PhpLlm\LlmChain\Exception\ToolBoxException;
7+
use PhpLlm\LlmChain\Chain\ToolBox\Exception\ToolExecutionException;
8+
use PhpLlm\LlmChain\Chain\ToolBox\Exception\ToolNotFoundException;
89
use PhpLlm\LlmChain\Model\Response\ToolCall;
910
use Psr\Log\LoggerInterface;
1011
use Psr\Log\NullLogger;
@@ -61,13 +62,13 @@ public function execute(ToolCall $toolCall): mixed
6162
$result = $tool->{$metadata->method}(...$toolCall->arguments);
6263
} catch (\Throwable $e) {
6364
$this->logger->warning(sprintf('Failed to execute tool "%s".', $metadata->name), ['exception' => $e]);
64-
throw ToolBoxException::executionFailed($toolCall, $e);
65+
throw ToolExecutionException::executionFailed($toolCall, $e);
6566
}
6667

6768
return $result;
6869
}
6970
}
7071

71-
throw ToolBoxException::notFoundForToolCall($toolCall);
72+
throw ToolNotFoundException::notFoundForToolCall($toolCall);
7273
}
7374
}

‎src/Chain/ToolBox/ToolBoxInterface.php

+6
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44

55
namespace PhpLlm\LlmChain\Chain\ToolBox;
66

7+
use PhpLlm\LlmChain\Chain\ToolBox\Exception\ToolExecutionException;
8+
use PhpLlm\LlmChain\Chain\ToolBox\Exception\ToolNotFoundException;
79
use PhpLlm\LlmChain\Model\Response\ToolCall;
810

911
interface ToolBoxInterface
@@ -13,5 +15,9 @@ interface ToolBoxInterface
1315
*/
1416
public function getMap(): array;
1517

18+
/**
19+
* @throws ToolExecutionException if the tool execution fails
20+
* @throws ToolNotFoundException if the tool is not found
21+
*/
1622
public function execute(ToolCall $toolCall): mixed;
1723
}

‎src/Exception/InvalidToolImplementation.php

-15
This file was deleted.

‎src/Exception/ToolBoxException.php

-33
This file was deleted.

‎tests/Chain/ToolBox/FaultTolerantToolBoxTest.php

+78
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace PhpLlm\LlmChain\Tests\Chain\ToolBox;
6+
7+
use PhpLlm\LlmChain\Chain\ToolBox\Exception\ToolExecutionException;
8+
use PhpLlm\LlmChain\Chain\ToolBox\Exception\ToolNotFoundException;
9+
use PhpLlm\LlmChain\Chain\ToolBox\FaultTolerantToolBox;
10+
use PhpLlm\LlmChain\Chain\ToolBox\Metadata;
11+
use PhpLlm\LlmChain\Chain\ToolBox\ToolBoxInterface;
12+
use PhpLlm\LlmChain\Model\Response\ToolCall;
13+
use PhpLlm\LlmChain\Tests\Fixture\Tool\ToolNoParams;
14+
use PhpLlm\LlmChain\Tests\Fixture\Tool\ToolRequiredParams;
15+
use PHPUnit\Framework\Attributes\CoversClass;
16+
use PHPUnit\Framework\Attributes\Test;
17+
use PHPUnit\Framework\TestCase;
18+
19+
#[CoversClass(FaultTolerantToolBox::class)]
20+
final class FaultTolerantToolBoxTest extends TestCase
21+
{
22+
#[Test]
23+
public function faultyToolExecution(): void
24+
{
25+
$faultyToolBox = $this->createFaultyToolBox(
26+
fn (ToolCall $toolCall) => ToolExecutionException::executionFailed($toolCall, new \Exception('error'))
27+
);
28+
29+
$faultTolerantToolBox = new FaultTolerantToolBox($faultyToolBox);
30+
$expected = 'An error occurred while executing tool "tool_foo".';
31+
32+
$toolCall = new ToolCall('987654321', 'tool_foo');
33+
$actual = $faultTolerantToolBox->execute($toolCall);
34+
35+
self::assertSame($expected, $actual);
36+
}
37+
38+
#[Test]
39+
public function faultyToolCall(): void
40+
{
41+
$faultyToolBox = $this->createFaultyToolBox(
42+
fn (ToolCall $toolCall) => ToolNotFoundException::notFoundForToolCall($toolCall)
43+
);
44+
45+
$faultTolerantToolBox = new FaultTolerantToolBox($faultyToolBox);
46+
$expected = 'Tool "tool_xyz" was not found, please use one of these: tool_no_params, tool_required_params';
47+
48+
$toolCall = new ToolCall('123456789', 'tool_xyz');
49+
$actual = $faultTolerantToolBox->execute($toolCall);
50+
51+
self::assertSame($expected, $actual);
52+
}
53+
54+
private function createFaultyToolBox(\Closure $exceptionFactory): ToolBoxInterface
55+
{
56+
return new class($exceptionFactory) implements ToolBoxInterface {
57+
public function __construct(private readonly \Closure $exceptionFactory)
58+
{
59+
}
60+
61+
/**
62+
* @return Metadata[]
63+
*/
64+
public function getMap(): array
65+
{
66+
return [
67+
new Metadata(ToolNoParams::class, 'tool_no_params', 'A tool without parameters', '__invoke', null),
68+
new Metadata(ToolRequiredParams::class, 'tool_required_params', 'A tool with required parameters', 'bar', null),
69+
];
70+
}
71+
72+
public function execute(ToolCall $toolCall): mixed
73+
{
74+
throw ($this->exceptionFactory)($toolCall);
75+
}
76+
};
77+
}
78+
}

‎tests/Chain/ToolBox/ToolAnalyzerTest.php

+3-3
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@
55
namespace PhpLlm\LlmChain\Tests\Chain\ToolBox;
66

77
use PhpLlm\LlmChain\Chain\ToolBox\Attribute\AsTool;
8+
use PhpLlm\LlmChain\Chain\ToolBox\Exception\ToolConfigurationException;
89
use PhpLlm\LlmChain\Chain\ToolBox\Metadata;
910
use PhpLlm\LlmChain\Chain\ToolBox\ParameterAnalyzer;
1011
use PhpLlm\LlmChain\Chain\ToolBox\ToolAnalyzer;
11-
use PhpLlm\LlmChain\Exception\InvalidToolImplementation;
1212
use PhpLlm\LlmChain\Tests\Fixture\Tool\ToolMultiple;
1313
use PhpLlm\LlmChain\Tests\Fixture\Tool\ToolRequiredParams;
1414
use PhpLlm\LlmChain\Tests\Fixture\Tool\ToolWrong;
@@ -21,7 +21,7 @@
2121
#[UsesClass(AsTool::class)]
2222
#[UsesClass(Metadata::class)]
2323
#[UsesClass(ParameterAnalyzer::class)]
24-
#[UsesClass(InvalidToolImplementation::class)]
24+
#[UsesClass(ToolConfigurationException::class)]
2525
final class ToolAnalyzerTest extends TestCase
2626
{
2727
private ToolAnalyzer $toolAnalyzer;
@@ -34,7 +34,7 @@ protected function setUp(): void
3434
#[Test]
3535
public function withoutAttribute(): void
3636
{
37-
$this->expectException(InvalidToolImplementation::class);
37+
$this->expectException(ToolConfigurationException::class);
3838
iterator_to_array($this->toolAnalyzer->getMetadata(ToolWrong::class));
3939
}
4040

0 commit comments

Comments
 (0)