refactor: better media handling (#321)

Author: ChrisB-TL

composer.json

@@ -19,6 +19,7 @@
   ],
   "require": {
     "php": "^8.2",
+    "ext-fileinfo": "*",
     "laravel/framework": "^11.0|^12.0"
   },
   "config": {

docs/components/ProviderSupport.vue

@@ -169,20 +169,10 @@ export default {
         "Documents",
       ],
       providers: [
-        {
-          name: "Amazon Bedrock",
-          text: Planned,
-          streaming: Unsupported,
-          structured: Unsupported,
-          embeddings: Planned,
-          image: Planned,
-          tools: Planned,
-          documents: Unsupported,
-        },
         {
           name: "Anthropic",
           text: Supported,
-          streaming: Planned,
+          streaming: Supported,
           structured: Adapted,
           embeddings: Unsupported,
           image: Supported,
@@ -200,12 +190,42 @@ export default {
           documents: Unsupported,
         },
         {
-          name: "DeepSeek",
+          name: "Bedrock - Anthropic",
           text: Supported,
           streaming: Unsupported,
           structured: Supported,
           embeddings: Unsupported,
+          image: Supported,
+          tools: Supported,
+          documents: Unsupported,
+        },
+        {
+          name: "Bedrock - Cohere",
+          text: Unsupported,
+          streaming: Unsupported,
+          structured: Unsupported,
+          embeddings: Supported,
           image: Unsupported,
+          tools: Unsupported,
+          documents: Unsupported,
+        },
+        {
+          name: "Bedrock - Converse",
+          text: Supported,
+          streaming: Unsupported,
+          structured: Supported,
+          embeddings: Unsupported,
+          image: Supported,
+          tools: Supported,
+          documents: Supported,
+        },
+        {
+          name: "DeepSeek",
+          text: Supported,
+          streaming: Unsupported,
+          structured: Supported,
+          embeddings: Unsupported,
+          image: Supported,
           tools: Supported,
           documents: Unsupported,
         },
@@ -257,7 +277,7 @@ export default {
           embeddings: Supported,
           image: Supported,
           tools: Supported,
-          documents: Unsupported,
+          documents: Supported,
         },
         {
           name: "VoyageAI",

docs/input-modalities/documents.md

@@ -1,40 +1,41 @@
 # Documents
 
-Prism currently supports documents with Gemini and Anthropic.
+Prism supports including documents in your messages with some providers.
+
+See the [provider support table](/getting-started/introduction.html#provider-support) to check whether Prism supports your chosen provider.
+
+Note however that provider support may differ by model. If you receive error messages with a provider that Prism indicates is supported, check the provider's documentation as to whether the model you are using supports documents.
 
 ## Supported file types
 
-Different providers support different document types.
-
-At the time of writing:
-- Anthropic supports 
-    - pdf (application/pdf) 
-    - txt (text/plain)
-    - md (text/md)
-- Gemini supports:
-    - pdf (application/pdf)
-    - javascript (text/javascript)
-    - python (text/x-python)
-    - txt (text/plain)
-    - html (text/html)
-    - css (text/css)
-    - md (text/md)
-    - csv (text/csv)
-    - xml (text/xml)
-    - rtf (text/rtf)
-- Mistral supports:
-  - PDF (application/pdf)
-  - CSV (text/csv)
-  - text files (text/plain)
-- OpenAI supports:
-    - PDF (application/pdf)
-    - `file_id` (previously uploaded pdf file id.)
-
-All of these formats should work with Prism.
+> [!TIP]
+> If provider interoperability is important to your app, we recommend converting documents to markdown.
+
+Please check provider documentation for supported file/mime types, as support differs widely.
+
+The most supported file types are pdf and text/plain (which may include markdown).
+
+## Transfer mediums 
+
+> [!TIP]
+> If provider interoperability is important to your app, we recommend using rawContent or base64.
+
+Providers are not consistent in their support of sending file raw contents, base64 and/or URLs. 
+
+Prism tries to smooth over these rough edges, but its not always possible.
+
+### Supported conversions
+- Where a provider does not support URLs: Prism will fetch the URL and use base64 or rawContent.
+- Where you provide a file, base64 or rawContent: Prism will switch between base64 and rawContent depending on what the provider accepts.
+
+### Limitations
+
+- Where a provider only supports URLs: if you provide a file path, raw contents, base64 or chunks, for security reasons Prism does not create a URL for you and your request will fail.
+- Chunks cannot be passed between providers, as they could be in different formats (however, currently only Anthropic supports them).
 
 ## Getting started
 
-To add an image to your message, add a `Document` value object to the `additionalContent` property:
+To add a document to your message, add a `Document` value object to the `additionalContent` property:
 
 ```php
 use Prism\Prism\Enums\Provider;
@@ -44,29 +45,82 @@ use Prism\Prism\ValueObjects\Messages\Support\Document;
 use Prism\Prism\ValueObjects\Messages\Support\OpenAIFile;
 
 Prism::text()
-    ->using(Provider::Anthropic, 'claude-3-5-sonnet-20241022')
+    ->using('my-provider', 'my-model')
     ->withMessages([
+        // From a local path
+        new UserMessage('Here is the document from a local path', [
+            Document::fromLocalPath(
+                path: 'tests/Fixtures/test-pdf.pdf', 
+                title: 'My document title' // optional
+            ),
+        ]),
+        // From a storage path
+        new UserMessage('Here is the document from a storage path', [
+            Document::fromStoragePath(
+                path: 'mystoragepath/file.pdf', 
+                disk: 'my-disk', // optional - omit/null for default disk
+                title: 'My document title' // optional
+            ),
+        ]),
         // From base64
         new UserMessage('Here is the document from base64', [
-            Document::fromBase64(base64_encode(file_get_contents('tests/Fixtures/test-pdf.pdf')), 'application/pdf'),
+            Document::fromBase64(
+                base64: $baseFromDB, 
+                mimeType: 'optional/mimetype', // optional 
+                title: 'My document title' // optional
+            ),
         ]),
-        // Or from a path
-        new UserMessage('Here is the document from a local path', [
-            Document::fromPath('tests/Fixtures/test-pdf.pdf'),
+        // From raw content
+        new UserMessage('Here is the document from raw content', [
+            Document::fromRawContent(
+                rawContent: $rawContent, 
+                mimeType: 'optional/mimetype', // optional 
+                title: 'My document title' // optional
+            ),
         ]),
-        // Or from a text string
+        // From a text string
         new UserMessage('Here is the document from a text string (e.g. from your database)', [
-            Document::fromText('Hello world!'),
+            Document::fromText(
+                text: 'Hello world!', 
+                title: 'My document title' // optional
+            ),
         ]),
-        // Or from an URL
-        new UserMessage('Here is the document from a url (make sure this is publically accessable)', [
-            Document::fromUrl('https://example.com/test-pdf.pdf'),
+        // From an URL
+        new UserMessage('Here is the document from a url (make sure this is publically accessible)', [
+            Document::fromUrl(
+                url: 'https://example.com/test-pdf.pdf', 
+                title: 'My document title' // optional
+            ),
         ]),
-        // Or from a file_id
+        // From chunks
+        new UserMessage('Here is a chunked document', [
+            Document::fromChunks(
+                chunks: [
+                    'chunk one',
+                    'chunk two'
+                ], 
+                title: 'My document title' // optional
+            ),
+        ]),
+    ])
+    ->asText();
+
+```
+
+Or, if using an OpenAI file_id - add an `OpenAIFile`:
+
+```php
+use Prism\Enums\Provider;
+use Prism\Prism\Prism;
+use Prism\Prism\ValueObjects\Messages\UserMessage;
+use Prism\Prism\ValueObjects\Messages\Support\OpenAIFile;
+
+Prism::text()
+    ->using(Provider::Anthropic, 'claude-3-5-sonnet-20241022')
+    ->withMessages([
         new UserMessage('Here is the document from file_id', [
             new OpenAIFile('file-lsfgSXyV2xEb8gw8fYjXU6'),
         ]),
     ])
     ->asText();
-
 ```

docs/input-modalities/images.md

@@ -4,7 +4,7 @@ Prism supports including images in your messages for vision analysis for most pr
 
 See the [provider support table](/getting-started/introduction.html#provider-support) to check whether Prism supports your chosen provider.
 
-Note however that not all models with a supported provider support vision. If you are running into issues with not supported messages, double check the provider model documentation for support.
+Note however that provider support may differ by model. If you receive error messages with a provider that Prism indicates is supported, check the provider's documentation as to whether the model you are using supports images.
 
 ## Getting started
 
@@ -14,39 +14,54 @@ To add an image to your message, add an `Image` value object to the `additionalC
 use Prism\Prism\ValueObjects\Messages\UserMessage;
 use Prism\Prism\ValueObjects\Messages\Support\Image;
 
-// From a local file
+// From a local path
 $message = new UserMessage(
     "What's in this image?",
-    [Image::fromPath('/path/to/image.jpg')]
+    [Image::fromLocalPath(path: '/path/to/image.jpg')]
+);
+
+// From a path on a storage disk
+$message = new UserMessage(
+    "What's in this image?",
+    [Image::fromStoragePath(
+        path: '/path/to/image.jpg', 
+        disk: 'my-disk' // optional - omit/null for default disk
+    )]
 );
 
 // From a URL
 $message = new UserMessage(
     'Analyze this diagram:',
-    [Image::fromUrl('https://example.com/diagram.png')]
+    [Image::fromUrl(url: 'https://example.com/diagram.png')]
 );
 
-// From a URL which does not end in the image format,
-// you can pass the mime type as the second argument
-// e.g. if you are generating a temporary URL
+// From base64
 $message = new UserMessage(
     'Analyze this diagram:',
-    [Image::fromUrl(
-        'https://storage.example.com/diagram.png?AccessID=test&Expires=1742330260&Signature=dVQaFcIk9FJWIVnvV1%2FWu',
-        'image/png'
-    )]
+    [Image::fromBase64(base64: base64_encode(file_get_contents('/path/to/image.jpg')))]
 );
 
-// From a Base64
-$image = base64_encode(file_get_contents('/path/to/image.jpg'));
-
+// From raw content
 $message = new UserMessage(
     'Analyze this diagram:',
-    [Image::fromBase64($image)]
+    [Image::fromRawContent(rawContent: file_get_contents('/path/to/image.jpg'))]
 );
 
 $response = Prism::text()
     ->using(Provider::Anthropic, 'claude-3-5-sonnet-20241022')
     ->withMessages([$message])
     ->generate();
 ```
+
+## Transfer mediums 
+
+Providers are not consistent in their support of sending raw contents, base64 and/or URLs (as noted above). 
+
+Prism tries to smooth over these rough edges, but its not always possible.
+
+### Supported conversions
+- Where a provider does not support URLs: Prism will fetch the URL and use base64 or rawContent.
+- Where you provide a file, base64 or rawContent: Prism will switch between base64 and rawContent depending on what the provider accepts.
+
+### Limitations
+- Where a provider only supports URLs: if you provide a file path, raw contents or base64, for security reasons Prism does not create a URL for you and your request will fail.

src/Contracts/ProviderMediaMapper.php

@@ -0,0 +1,27 @@
+<?php
+
+namespace Prism\Prism\Contracts;
+
+use Prism\Prism\Enums\Provider;
+use Prism\Prism\Exceptions\PrismException;
+use Prism\Prism\ValueObjects\Messages\Support\Media;
+
+abstract class ProviderMediaMapper
+{
+    public function __construct(public readonly Media $media)
+    {
+        if ($this->validateMedia() === false) {
+            $providerName = $this->provider() instanceof Provider ? $this->provider()->value : $this->provider();
+
+            $calledClass = static::class;
+
+            throw new PrismException("The $providerName provider does not support the mediums available in the provided `$calledClass`. Pleae consult the Prism documentation for more information on which mediums the $providerName provider supports.");
+        }
+    }
+
+    abstract public function toPayload(): mixed;
+
+    abstract protected function provider(): string|Provider;
+
+    abstract protected function validateMedia(): bool;
+}