Refactor chat to use LlamaStack Conversations API

- Replace custom streaming adapter with direct LlamaStack SSE integration
- Add comprehensive TypeScript types for streaming events
- Implement batched updates with requestAnimationFrame for better performance
- Add empty response detection with user-friendly warnings
- Delete deprecated chat CRUD and adapter files
- Fix API endpoint trailing slash issue

Signed-off-by: Yuval Turgeman <[email protected]>
Assisted-by: Claude

Author: yuvalturg

backend/.pre-commit-config.yaml

@@ -18,7 +18,7 @@ repos:
     rev: 25.1.0
     hooks:
       - id: black
-        language_version: python3.12
+        language_version: python3.13
         exclude: '^(backend/migrations/|migrations/)'
 
   - repo: https://github.com/pycqa/isort

backend/app/api/llamastack.py

@@ -1,19 +1,17 @@
+import json
 import logging
 import os
-from typing import Optional
+from typing import Any, Optional
 
 import httpx
 from dotenv import load_dotenv
 from fastapi import Request
 from llama_stack_client import AsyncLlamaStackClient
 
-# TODO: Need to update this import when virtual_agents is refactored
-# from ..virtual_agents.agent_resource import EnhancedAgentResource
-
 load_dotenv()
 
 LLAMASTACK_URL = os.getenv("LLAMASTACK_URL", "http://localhost:8321")
-LLAMASTACK_TIMEOUT = float(os.getenv("LLAMASTACK_TIMEOUT", "300.0"))
+LLAMASTACK_TIMEOUT = float(os.getenv("LLAMASTACK_TIMEOUT", "180.0"))
 
 # Set up logging
 logger = logging.getLogger(__name__)
@@ -178,3 +176,28 @@ def get_sync_client() -> AsyncLlamaStackClient:
 
 # Create sync client instance
 sync_client = get_sync_client()
+
+
+def create_tool_call_trace_entry(item: Any) -> dict:
+    """Create trace entry for MCP tool call."""
+    args = item.arguments
+    if args and isinstance(args, str):
+        try:
+            args = json.loads(args)
+        except Exception:
+            pass
+
+    return {
+        "type": "tool_call",
+        "name": item.name or "tool",
+        "server_label": item.server_label,
+        "arguments": args,
+        "output": item.output,
+        "error": item.error,
+        "status": "failed" if item.error else "completed",
+    }
+
+
+ERROR_NO_RESPONSE_MESSAGE = (
+    "⚠️ Unable to generate a response. Please try rephrasing your question or try again."
+)

backend/app/api/v1/chat.py

@@ -48,22 +48,26 @@ async def chat(
         # Create chat service with the database session
         chat_service = ChatService(request, db, user_id=current_user.id)
 
-        async def chat_stream():
-            try:
-                async for chunk in chat_service.stream(
-                    chat_request.virtualAgentId,
-                    chat_request.sessionId or "default",
-                    chat_request.message.content,
-                ):
-                    yield f"data: {chunk}\n\n"
-
-                yield "data: [DONE]\n\n"
-
-            except Exception as e:
-                logger.error(f"Error in stream: {str(e)}")
-                yield f'data: {{"type":"error","content":"Error: {str(e)}"}}\n\n'
+        # Require a valid session ID
+        if not chat_request.sessionId:
+            raise HTTPException(
+                status_code=400,
+                detail="Session ID is required",
+            )
 
-        return StreamingResponse(chat_stream(), media_type="text/event-stream")
+        # Stream directly from service (SSE format)
+        return StreamingResponse(
+            chat_service.stream(
+                agent,  # Pass the agent object, not ID (already fetched above)
+                chat_request.sessionId,
+                chat_request.message.content,
+            ),
+            media_type="text/event-stream",
+            headers={
+                "Cache-Control": "no-cache",
+                "X-Accel-Buffering": "no",  # Disable nginx buffering
+            },
+        )
 
     except Exception as e:
         logger.error(f"Error in chat endpoint: {str(e)}")