feat: voice agent

apps/server/agents/conversational/conversational.py

@@ -89,6 +89,7 @@ def run(
         configs = agent_with_configs.configs
         audio_url = None
         if configs.response_mode.includes("Voice"):
+            # todo text to speech and generate audio url
             audio_url = text_to_speech(res, configs, voice_settings)
             pass
 

apps/server/models/config.py

@@ -299,11 +299,11 @@ def get_account_voice_settings(
         cls, session: Session, account_id: UUID
     ) -> AccountVoiceSettings:
         keys = [
-            "deepgram_api_key",
-            "azure_speech_key",
-            "azure_speech_region",
-            "play_ht_api_key",
-            "play_ht_user_id",
+            "DEEPGRAM_API_KEY",
+            "AZURE_SPEECH_KEY",
+            "AZURE_SPEECH_REGION",
+            "PLAY_HT_API_KEY",
+            "PLAY_HT_USER_ID",
             # todo add openai tsss
             # todo add 11labs
         ]
@@ -332,11 +332,11 @@ def get_account_voice_settings(
             )
 
         return AccountVoiceSettings(
-            deepgram_api_key=config.get("deepgram_api_key"),
-            azure_speech_key=config.get("azure_speech_key"),
-            azure_speech_region=config.get("azure_speech_region"),
-            play_ht_api_key=config.get("play_ht_api_key"),
-            play_ht_user_id=config.get("play_ht_user_id"),
+            DEEPGRAM_API_KEY=config.get("DEEPGRAM_API_KEY"),
+            AZURE_SPEECH_KEY=config.get("AZURE_SPEECH_KEY"),
+            AZURE_SPEECH_REGION=config.get("AZURE_SPEECH_REGION"),
+            PLAY_HT_API_KEY=config.get("PLAY_HT_API_KEY"),
+            PLAY_HT_USER_ID=config.get("PLAY_HT_USER_ID"),
         )
 
     @classmethod

apps/server/postgres.py

@@ -58,6 +58,7 @@ def messages(self) -> List[BaseMessage]:  # type: ignore
     def create_message(
         self, message, parent_id: Optional[str] = None, agent_id: Optional[UUID] = None
     ):
+        # todo store audio file
         # Append the message to the record in PostgreSQL
         chat_message = ChatMessage(
             sender_user_id=self.sender_user_id,

apps/server/services/chat.py

@@ -63,6 +63,9 @@ def create_user_message(body: ChatUserMessageInput, auth: UserAccount):
     prompt = body.prompt
 
     session_id = get_chat_session_id(user.id, account.id, agent_id, team_id)
+    if body.audio_data:
+        # todo Ismael need upload s3
+        pass
 
     process_chat_message(
         session_id=session_id,
@@ -77,7 +80,7 @@ def create_user_message(body: ChatUserMessageInput, auth: UserAccount):
         provider_account=provider_account,
         provider_user=provider_user,
         chat_id=None,
-        voice_url=None,
+        voice_url=body.voice_url,
     )
 
     return ""
@@ -116,7 +119,7 @@ def create_client_message(body: ChatMessageInput, auth: UserAccount):
     session_id = get_chat_session_id(user.id, account.id, agent_id, team_id, chat_id)
 
     if audio_data:
-        # need upload s3
+        # Ismael need upload s3
         pass
     voice_url = body.voice_url
 

apps/server/services/voice.py

@@ -26,6 +26,7 @@ async def text_to_speech(
     """
 
     # todo text to speech
+    # check which one you can use
     synthesizer = configs.synthesizer  # you can check with it later which synthesizer
     default_voice = configs.default_voice or None
 

apps/server/typings/chat.py

@@ -14,8 +14,8 @@ class ChatStatus(Enum):
 
 class ChatUserMessageInput(BaseModel):
     prompt: str
-    voice_url: str
-    audio_data: str
+    voice_url: Optional[str]
+    audio_data: Optional[str]
     local_chat_message_ref_id: Optional[str] = None
     agent_id: Optional[UUID] = None
     team_id: Optional[UUID] = None
@@ -24,8 +24,8 @@ class ChatUserMessageInput(BaseModel):
 
 class ChatMessageInput(BaseModel):
     prompt: str
-    voice_url: str
-    audio_data: str
+    voice_url: Optional[str]
+    audio_data: Optional[str]
     chat_id: Optional[UUID] = None
     # todo what it is?
     local_chat_message_ref_id: Optional[str] = None

apps/server/typings/config.py

@@ -60,8 +60,8 @@ class AccountSettings(BaseModel):
 
 
 class AccountVoiceSettings(BaseModel):
-    deepgram_api_key: Optional[str]
-    azure_speech_key: Optional[str]
-    azure_speech_region: Optional[str]
-    play_ht_api_key: Optional[str]
-    play_ht_user_id: Optional[str]
+    DEEPGRAM_API_KEY: Optional[str]
+    AZURE_SPEECH_KEY: Optional[str]
+    AZURE_SPEECH_REGION: Optional[str]
+    PLAY_HT_API_KEY: Optional[str]
+    PLAY_HT_USER_ID: Optional[str]

apps/ui/src/plugins/contact/pages/Voice/constants.ts

@@ -3,6 +3,7 @@ import playHTLogo from 'assets/voices/playHT.png'
 import azureLogo from 'assets/voices/azure.png'
 import googleVoiceLogo from 'assets/voices/googleVoice.png'
 import labsLogo from 'assets/voices/11labs.webp'
+import openAI from 'assets/models/openai.jpg'
 
 export const voiceLogos = [
   {
@@ -25,4 +26,8 @@ export const voiceLogos = [
     logoSrc: labsLogo,
     voiceName: '11Labs',
   },
+  {
+    logoSrc: openAI,
+    voiceName: 'OpenAI',
+  },
 ]