Test utilities (#54)

sixlive · web-flow · commit 6ad1789a2761 · 2024-11-09T18:06:26.000-05:00
diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts
@@ -102,6 +102,10 @@ export default defineConfig({
                 text: "Prism Server",
                 link: "/core-concepts/prism-server",
               },
+              {
+                text: "Testing",
+                link: "/core-concepts/testing",
+              },
             ],
           },
           {
diff --git a/docs/core-concepts/testing.md b/docs/core-concepts/testing.md
@@ -0,0 +1,162 @@
+# Testing
+
+Want to make sure your Prism integrations work flawlessly? Let's dive into testing! Prism provides a powerful fake implementation that makes it a breeze to test your AI-powered features.
+
+## Basic Test Setup
+
+First, let's look at how to set up basic response faking:
+
+```php
+use EchoLabs\Prism\Facades\Prism;
+use EchoLabs\Prism\ValueObjects\Usage;
+use EchoLabs\Prism\Enums\FinishReason;
+use EchoLabs\Prism\Providers\ProviderResponse;
+
+public function test_can_generate_text(): void
+{
+    // Create a fake provider response
+    $fakeResponse = new ProviderResponse(
+        text: 'Hello, I am Claude!',
+        toolCalls: [],
+        usage: new Usage(10, 20),
+        finishReason: FinishReason::Stop,
+        response: ['id' => 'fake-1', 'model' => 'fake-model']
+    );
+
+    // Set up the fake
+    $fake = Prism::fake([$fakeResponse]);
+
+    // Run your code
+    $response = Prism::text()
+        ->using('anthropic', 'claude-3-sonnet')
+        ->withPrompt('Who are you?')
+        ->generate();
+
+    // Make assertions
+    $this->assertEquals('Hello, I am Claude!', $response->text);
+}
+```
+
+## Testing Multiple Responses
+
+When testing conversations or tool usage, you might need to simulate multiple responses:
+
+```php
+public function test_can_handle_tool_calls(): void
+{
+    $responses = [
+        new ProviderResponse(
+            text: '',
+            toolCalls: [
+                new ToolCall(
+                    id: 'call_1',
+                    name: 'search',
+                    arguments: ['query' => 'Latest news']
+                )
+            ],
+            usage: new Usage(15, 25),
+            finishReason: FinishReason::ToolCalls,
+            response: ['id' => 'fake-1', 'model' => 'fake-model']
+        ),
+        new ProviderResponse(
+            text: 'Here are the latest news...',
+            toolCalls: [],
+            usage: new Usage(20, 30),
+            finishReason: FinishReason::Stop,
+            response: ['id' => 'fake-2', 'model' => 'fake-model']
+        ),
+    ];
+
+    $fake = Prism::fake($responses);
+}
+```
+
+## Assertions
+
+Prism's fake implementation provides several helpful assertion methods:
+
+```php
+// Assert specific prompt was sent
+$fake->assertPrompt('Who are you?');
+
+// Assert number of calls made
+$fake->assertCallCount(2);
+
+// Assert detailed request properties
+$fake->assertRequest(function ($requests) {
+    $this->assertEquals('anthropic', $requests[0]->provider);
+    $this->assertEquals('claude-3-sonnet', $requests[0]->model);
+});
+```
+
+## Testing Tools
+
+When testing tools, you'll want to verify both the tool calls and their results. Here's a complete example:
+
+```php
+public function test_can_use_weather_tool(): void
+{
+    // Define the expected tool call and response sequence
+    $responses = [
+        // First response: AI decides to use the weather tool
+        new ProviderResponse(
+            text: '', // Empty text since the AI is using a tool
+            toolCalls: [
+                new ToolCall(
+                    id: 'call_123',
+                    name: 'weather',
+                    arguments: ['city' => 'Paris']
+                )
+            ],
+            usage: new Usage(15, 25),
+            finishReason: FinishReason::ToolCalls,
+            response: ['id' => 'fake-1', 'model' => 'fake-model']
+        ),
+        // Second response: AI uses the tool result to form a response
+        new ProviderResponse(
+            text: 'Based on current conditions, the weather in Paris is sunny with a temperature of 72°F.',
+            toolCalls: [],
+            usage: new Usage(20, 30),
+            finishReason: FinishReason::Stop,
+            response: ['id' => 'fake-2', 'model' => 'fake-model']
+        ),
+    ];
+
+    // Set up the fake
+    $fake = Prism::fake($responses);
+
+    // Create the weather tool
+    $weatherTool = Tool::as('weather')
+        ->for('Get weather information')
+        ->withStringParameter('city', 'City name')
+        ->using(fn (string $city) => "The weather in {$city} is sunny with a temperature of 72°F");
+
+    // Run the actual test
+    $response = Prism::text()
+        ->using('anthropic', 'claude-3-sonnet')
+        ->withPrompt('What\'s the weather in Paris?')
+        ->withTools([$weatherTool])
+        ->generate();
+
+    // Assert the correct number of API calls were made
+    $fake->assertCallCount(2);
+
+    // Assert tool calls were made correctly
+    $this->assertCount(1, $response->steps[0]->toolCalls);
+    $this->assertEquals('weather', $response->steps[0]->toolCalls[0]->name);
+    $this->assertEquals(['city' => 'Paris'], $response->steps[0]->toolCalls[0]->arguments());
+
+    // Assert tool results were processed
+    $this->assertCount(1, $response->toolResults);
+    $this->assertEquals(
+        'The weather in Paris is sunny with a temperature of 72°F',
+        $response->toolResults[0]->result
+    );
+
+    // Assert final response
+    $this->assertEquals(
+        'Based on current conditions, the weather in Paris is sunny with a temperature of 72°F.',
+        $response->text
+    );
+}
+```
diff --git a/src/Concerns/BuildsTextRequests.php b/src/Concerns/BuildsTextRequests.php
@@ -151,6 +151,7 @@ protected function textRequest(): TextRequest
         return new TextRequest(
             model: $this->model,
             systemPrompt: $this->systemPrompt,
+            prompt: $this->prompt,
             messages: $this->state->messages()->toArray(),
             temperature: $this->temperature,
             maxTokens: $this->maxTokens,
diff --git a/src/Prism.php b/src/Prism.php
@@ -5,11 +5,34 @@
 namespace EchoLabs\Prism;
 
 use EchoLabs\Prism\Contracts\Provider;
+use EchoLabs\Prism\Enums\Provider as ProviderEnum;
 use EchoLabs\Prism\Generators\TextGenerator;
+use EchoLabs\Prism\Providers\ProviderResponse;
+use EchoLabs\Prism\Testing\PrismFake;
 
 class Prism
 {
-    protected Provider $provider;
+    /**
+     * @param  array<int, ProviderResponse>  $responses
+     */
+    public static function fake(array $responses = []): PrismFake
+    {
+        $fake = new PrismFake($responses);
+
+        app()->instance(PrismManager::class, new class($fake) extends PrismManager
+        {
+            public function __construct(
+                private readonly PrismFake $fake
+            ) {}
+
+            public function resolve(ProviderEnum|string $name): Provider
+            {
+                return $this->fake;
+            }
+        });
+
+        return $fake;
+    }
 
     public static function text(): TextGenerator
     {
diff --git a/src/Requests/TextRequest.php b/src/Requests/TextRequest.php
@@ -18,6 +18,7 @@ class TextRequest
     public function __construct(
         public readonly string $model,
         public readonly ?string $systemPrompt,
+        public readonly ?string $prompt,
         public readonly array $messages,
         public readonly ?int $maxTokens,
         public readonly int|float|null $temperature,
diff --git a/src/Testing/PrismFake.php b/src/Testing/PrismFake.php
@@ -0,0 +1,90 @@
+<?php
+
+declare(strict_types=1);
+
+namespace EchoLabs\Prism\Testing;
+
+use Closure;
+use EchoLabs\Prism\Contracts\Provider;
+use EchoLabs\Prism\Enums\FinishReason;
+use EchoLabs\Prism\Providers\ProviderResponse;
+use EchoLabs\Prism\Requests\TextRequest;
+use EchoLabs\Prism\ValueObjects\Usage;
+use Exception;
+use PHPUnit\Framework\Assert as PHPUnit;
+
+class PrismFake implements Provider
+{
+    protected int $responseSequence = 0;
+
+    /** @var array<int, TextRequest> */
+    protected array $recorded = [];
+
+    /**
+     * @param  array<int, ProviderResponse>  $responses
+     */
+    public function __construct(protected array $responses = []) {}
+
+    #[\Override]
+    public function text(TextRequest $request): ProviderResponse
+    {
+        $this->recorded[] = $request;
+
+        return $this->nextResponse() ?? new ProviderResponse(
+            text: '',
+            toolCalls: [],
+            usage: new Usage(0, 0),
+            finishReason: FinishReason::Stop,
+            response: ['id' => 'fake', 'model' => 'fake']
+        );
+    }
+
+    /**
+     * @param  Closure(array<int, TextRequest>):void  $fn
+     */
+    public function assertRequest(Closure $fn): void
+    {
+        $fn($this->recorded);
+    }
+
+    public function assertPrompt(string $prompt): void
+    {
+        $prompts = collect($this->recorded)
+            ->flatten()
+            ->map
+            ->prompt;
+
+        PHPUnit::assertTrue(
+            $prompts->contains($prompt),
+            "Could not find the prompt {$prompt} in the recorded requests"
+        );
+    }
+
+    /**
+     * Assert number of calls made
+     */
+    public function assertCallCount(int $expectedCount): void
+    {
+        $actualCount = count($this->recorded ?? []);
+
+        PHPUnit::assertEquals($expectedCount, $actualCount, "Expected {$expectedCount} calls, got {$actualCount}");
+    }
+
+    protected function nextResponse(): ?ProviderResponse
+    {
+        if (! isset($this->responses)) {
+            return null;
+        }
+
+        $responses = $this->responses;
+        $sequence = $this->responseSequence;
+
+        if (! isset($responses[$sequence])) {
+            throw new Exception('Could not find a response for the request');
+        }
+
+        $this->responseSequence++;
+
+        return $responses[$sequence];
+    }
+}
diff --git a/tests/Testing/PrismFakeTest.php b/tests/Testing/PrismFakeTest.php
@@ -0,0 +1,61 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Tests\Testing;
+
+use EchoLabs\Prism\Enums\FinishReason;
+use EchoLabs\Prism\Prism;
+use EchoLabs\Prism\Providers\ProviderResponse;
+use EchoLabs\Prism\Requests\TextRequest;
+use EchoLabs\Prism\ValueObjects\Usage;
+use Exception;
+
+it('fake responses using the prism fake', function (): void {
+    $fake = Prism::fake([
+        new ProviderResponse(
+            text: 'The meaning of life is 42',
+            toolCalls: [],
+            usage: new Usage(42, 42),
+            finishReason: FinishReason::Stop,
+            response: ['id' => 'cpl_1234', 'model' => 'claude-3-sonnet'],
+        ),
+    ]);
+
+    Prism::text()
+        ->using('anthropic', 'claude-3-sonnet')
+        ->withPrompt('What is the meaning of life?')
+        ->generate();
+
+    $fake->assertCallCount(1);
+    $fake->assertPrompt('What is the meaning of life?');
+    $fake->assertRequest(function (array $requests): void {
+        expect($requests)->toHaveCount(1);
+        expect($requests[0])->toBeInstanceOf(TextRequest::class);
+    });
+});
+
+it("throws an exception when it can't runs out of responses", function (): void {
+    $this->expectException(Exception::class);
+    $this->expectExceptionMessage('Could not find a response for the request');
+
+    Prism::fake([
+        new ProviderResponse(
+            text: 'The meaning of life is 42',
+            toolCalls: [],
+            usage: new Usage(42, 42),
+            finishReason: FinishReason::Stop,
+            response: ['id' => 'cpl_1234', 'model' => 'claude-3-sonnet'],
+        ),
+    ]);
+
+    Prism::text()
+        ->using('anthropic', 'claude-3-sonnet')
+        ->withPrompt('What is the meaning of life?')
+        ->generate();
+
+    Prism::text()
+        ->using('anthropic', 'claude-3-sonnet')
+        ->withPrompt('What is the meaning of life?')
+        ->generate();
+});
