feat: audio example

Author: chr-hertel

assets/app.js

@@ -1,6 +1,7 @@
 import './bootstrap.js';
 import 'bootstrap/dist/css/bootstrap.min.css';
 import './styles/app.css';
+import './styles/audio.css';
 import './styles/blog.css';
 import './styles/youtube.css';
 import './styles/wikipedia.css';

assets/controllers/audio_controller.js

@@ -0,0 +1,89 @@
+import { Controller } from '@hotwired/stimulus';
+import { getComponent } from '@symfony/ux-live-component';
+
+export default class extends Controller {
+    async initialize() {
+        this.component = await getComponent(this.element);
+        this.scrollToBottom();
+
+        const resetButton = document.getElementById('chat-reset');
+        resetButton.addEventListener('click', (event) => {
+            this.component.action('reset');
+        });
+
+        const startButton = document.getElementById('micro-start');
+        const stopButton = document.getElementById('micro-stop');
+        const botThinkingButton = document.getElementById('bot-thinking');
+
+        startButton.addEventListener('click', (event) => {
+            event.preventDefault();
+            startButton.classList.add('d-none');
+            stopButton.classList.remove('d-none');
+            this.startRecording();
+        });
+        stopButton.addEventListener('click', (event) => {
+            event.preventDefault();
+            stopButton.classList.add('d-none');
+            botThinkingButton.classList.remove('d-none');
+            this.mediaRecorder.stop();
+        });
+
+        this.component.on('loading.state:started', (e,r) => {
+            if (r.actions.includes('reset')) {
+                return;
+            }
+            document.getElementById('welcome')?.remove();
+            document.getElementById('loading-message').removeAttribute('class');
+            this.scrollToBottom();
+        });
+
+        this.component.on('loading.state:finished', () => {
+            document.getElementById('loading-message').setAttribute('class', 'd-none');
+            botThinkingButton.classList.add('d-none');
+            startButton.classList.remove('d-none');
+        });
+
+        this.component.on('render:finished', () => {
+            this.scrollToBottom();
+        });
+    };
+
+    async startRecording() {
+        const stream = await navigator.mediaDevices.getUserMedia({audio: true});
+        this.mediaRecorder = new MediaRecorder(stream);
+        let audioChunks = [];
+
+        this.mediaRecorder.ondataavailable = (event) => {
+            audioChunks.push(event.data);
+        };
+
+        this.mediaRecorder.onstop = async () => {
+            const audioBlob = new Blob(audioChunks, {type: 'audio/wav'});
+
+            const base64String = await this.blobToBase64(audioBlob);
+            this.component.action('submit', { audio: base64String });
+        };
+
+        this.mediaRecorder.start();
+    }
+
+    scrollToBottom() {
+        const chatBody = document.getElementById('chat-body');
+        chatBody.scrollTop = chatBody.scrollHeight;
+    }
+
+    blobToBase64(blob) {
+        return new Promise((resolve) => {
+            const reader = new FileReader();
+            reader.readAsDataURL(blob);
+            reader.onloadend = () => resolve(reader.result.split(',')[1]);
+        });
+    }
+
+    playBase64Audio(base64String) {
+        const audioSrc = "data:audio/wav;base64," + base64String;
+        const audio = new Audio(audioSrc);
+
+        audio.play().catch(error => console.error("Playback error:", error));
+    }
+}

assets/icons/iconoir/microphone-mute-solid.svg

@@ -0,0 +1,61 @@
+.audio {
+    body&, .card-img-top {
+        background: #df662f;
+        background: linear-gradient(0deg, #df662f 0%, #a80a1d 100%);
+    }
+
+    .card-img-top {
+        color: #ffffff;
+    }
+
+    &.chat {
+        .user-message {
+            background: #df662f;
+            color: #ffffff;
+        }
+
+        .bot-message {
+            color: #ffffff;
+            background: #215d9a;
+
+            &.loading {
+                color: rgba(255, 255, 255, 0.5);
+            }
+
+            a {
+                color: #c8d8ef;
+
+                &:hover {
+                    color: #ffffff;
+                }
+            }
+
+            code {
+                color: #ffb1ca;
+            }
+        }
+
+        .avatar {
+            &.bot {
+                outline: 1px solid #c0dbf4;
+                background: #c0dbf4;
+            }
+
+            &.user {
+                outline: 1px solid #f3b396;
+                background: #f3b396;
+            }
+        }
+
+        #welcome h4 {
+            color: #2c5282;
+        }
+
+        #chat-reset, #chat-submit {
+            &:hover {
+                background: #a80a1d;
+                border-color: #a80a1d;
+            }
+        }
+    }
+}

assets/icons/iconoir/microphone-solid.svg

@@ -9,6 +9,7 @@
         "ext-iconv": "*",
         "codewithkyrian/chromadb-php": "^0.3.0",
         "league/commonmark": "^2.6",
+        "php-llm/llm-chain": "dev-feat-whisper-support as 0.17.0",
         "php-llm/llm-chain-bundle": "^0.17",
         "runtime/frankenphp-symfony": "^0.2.0",
         "symfony/asset": "7.2.*",

assets/icons/iconoir/timer-solid.svg

@@ -37,6 +37,12 @@ llm_chain:
             system_prompt: 'Please answer the users question based on Wikipedia and provide a link to the article.'
             tools:
                 - 'PhpLlm\LlmChain\Chain\ToolBox\Tool\Wikipedia'
+        audio:
+            model:
+                name: 'GPT'
+                version: 'gpt-4o-mini'
+            system_prompt: 'You are a friendly chatbot that likes to have a conversation with users and asks them some questions.'
+            tools: false
     store:
         chroma_db:
             symfonycon:

assets/styles/audio.css

@@ -11,6 +11,13 @@ blog:
         template:  'chat.html.twig'
         context: { chat: 'blog' }
 
+audio:
+    path: '/audio'
+    controller: 'Symfony\Bundle\FrameworkBundle\Controller\TemplateController'
+    defaults:
+        template:  'chat.html.twig'
+        context: { chat: 'audio' }
+
 youtube:
     path: '/youtube'
     controller: 'Symfony\Bundle\FrameworkBundle\Controller\TemplateController'

composer.json

@@ -0,0 +1,72 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Audio;
+
+use PhpLlm\LlmChain\Bridge\OpenAI\Whisper;
+use PhpLlm\LlmChain\Bridge\OpenAI\Whisper\File;
+use PhpLlm\LlmChain\ChainInterface;
+use PhpLlm\LlmChain\Model\Message\Message;
+use PhpLlm\LlmChain\Model\Message\MessageBag;
+use PhpLlm\LlmChain\Model\Response\AsyncResponse;
+use PhpLlm\LlmChain\Model\Response\TextResponse;
+use PhpLlm\LlmChain\PlatformInterface;
+use Symfony\Component\DependencyInjection\Attribute\Autowire;
+use Symfony\Component\HttpFoundation\RequestStack;
+
+final class Chat
+{
+    private const SESSION_KEY = 'audio-chat';
+
+    public function __construct(
+        private readonly PlatformInterface $platform,
+        private readonly RequestStack $requestStack,
+        #[Autowire(service: 'llm_chain.chain.audio')]
+        private readonly ChainInterface $chain,
+    ) {
+    }
+
+    public function say(string $base64audio): void
+    {
+        // Convert base64 to temporary binary file
+        $path = tempnam(sys_get_temp_dir(), 'audio').'.wav';
+        file_put_contents($path, base64_decode($base64audio));
+
+        $response = $this->platform->request(new Whisper(), new File($path));
+        assert($response instanceof AsyncResponse);
+        $response = $response->unwrap();
+        assert($response instanceof TextResponse);
+
+        $this->submitMessage($response->getContent());
+    }
+
+    public function loadMessages(): MessageBag
+    {
+        return $this->requestStack->getSession()->get(self::SESSION_KEY, new MessageBag());
+    }
+
+    public function submitMessage(string $message): void
+    {
+        $messages = $this->loadMessages();
+
+        $messages->add(Message::ofUser($message));
+        $response = $this->chain->call($messages);
+
+        assert($response instanceof TextResponse);
+
+        $messages->add(Message::ofAssistant($response->getContent()));
+
+        $this->saveMessages($messages);
+    }
+
+    public function reset(): void
+    {
+        $this->requestStack->getSession()->remove(self::SESSION_KEY);
+    }
+
+    private function saveMessages(MessageBag $messages): void
+    {
+        $this->requestStack->getSession()->set(self::SESSION_KEY, $messages);
+    }
+}