Implement real streaming from LlamaStack

- Replace simulated streaming with actual LlamaStack streaming in chat service
- Handle chunk types: text deltas, tool execution status, completion events
- Add fallback formatting for tool outputs when no text is generated
- Increase timeout to 300s and max_infer_iters to 100
- Fix MCP discovery to use mcp.transport label instead of proxyMode
- Update logging for new app/ directory structure
- Remove unused Kubernetes service readiness code

Signed-off-by: Yuval Turgeman <[email protected]>
Assisted-by: Claude

Author: yuvalturg

backend/README.md

@@ -11,22 +11,41 @@ FastAPI backend for the AI Virtual Agent Quickstart project. For complete setup
 
 ```
 backend/
-├── main.py               # FastAPI app entrypoint, includes routers
-├── database.py           # Database connection and session
-├── models.py             # SQLAlchemy models
-├── schemas.py            # Pydantic schemas
-├── routes/               # API route modules
-│   ├── users.py          # User management endpoints
-│   ├── mcp_servers.py    # MCP server management
-│   ├── knowledge_bases.py # Knowledge base operations
-│   ├── virtual_assistants.py # Agent CRUD operations
-│   ├── chat_sessions.py  # Chat session management
-│   ├── tools.py          # Tool configuration endpoints
-│   └── guardrails.py     # Guardrail management
-├── utils/                # Utility modules
-│   └── logging_config.py # Centralized logging setup
-├── requirements.txt      # Python dependencies
-└── .env                  # Environment variables (not committed)
+├── main.py                    # FastAPI app entrypoint
+├── app/
+│   ├── api/v1/               # API route modules
+│   │   ├── chat.py           # Chat endpoints
+│   │   ├── chat_sessions.py  # Session management
+│   │   ├── virtual_agents.py # Agent CRUD operations
+│   │   ├── knowledge_bases.py # Knowledge base operations
+│   │   ├── tools.py          # Tool configuration endpoints
+│   │   ├── users.py          # User management endpoints
+│   │   ├── mcp_servers.py    # MCP server management
+│   │   └── ...               # Additional API endpoints
+│   ├── core/                 # Core configuration
+│   │   ├── logging_config.py # Centralized logging setup
+│   │   ├── auth.py           # Authentication utilities
+│   │   └── template_startup.py # Template initialization
+│   ├── crud/                 # Data access layer
+│   │   ├── virtual_agents.py # Agent data operations
+│   │   ├── chat_sessions.py  # Session data operations
+│   │   └── ...               # Additional CRUD operations
+│   ├── models/               # SQLAlchemy database models
+│   │   ├── agent.py          # Agent models
+│   │   ├── chat.py           # Chat models
+│   │   └── ...               # Additional models
+│   ├── schemas/              # Pydantic schemas for validation
+│   │   ├── agent.py          # Agent schemas
+│   │   ├── chat.py           # Chat schemas
+│   │   └── ...               # Additional schemas
+│   ├── services/             # Business logic layer
+│   │   ├── chat.py           # Chat service logic
+│   │   └── ...               # Additional services
+│   └── database.py           # Database connection and session
+├── migrations/               # Alembic database migrations
+├── agent_templates/          # Agent template YAML files
+├── requirements.txt          # Python dependencies
+└── .env                      # Environment variables (not committed)
 ```
 
 ## Environment Variables

backend/app/api/llamastack.py

@@ -13,7 +13,7 @@
 load_dotenv()
 
 LLAMASTACK_URL = os.getenv("LLAMASTACK_URL", "http://localhost:8321")
-LLAMASTACK_TIMEOUT = float(os.getenv("LLAMASTACK_TIMEOUT", "60.0"))
+LLAMASTACK_TIMEOUT = float(os.getenv("LLAMASTACK_TIMEOUT", "300.0"))
 
 # Set up logging
 logger = logging.getLogger(__name__)
@@ -70,7 +70,6 @@ def get_client(
         base_url=LLAMASTACK_URL,
         default_headers=headers or {},
         timeout=httpx.Timeout(LLAMASTACK_TIMEOUT),
-        max_retries=0,
     )
     if api_key:
         client.api_key = api_key

backend/app/api/v1/agent_templates.py

@@ -340,7 +340,7 @@ async def initialize_agent_from_template(
             top_p=0.95,
             max_tokens=4096,
             repetition_penalty=1.0,
-            max_infer_iters=10,
+            max_infer_iters=100,
             input_shields=[],
             output_shields=[],
             enable_session_persistence=False,

backend/app/core/logging_config.py

@@ -46,11 +46,14 @@ def setup_logging(
 
     # Ensure our backend loggers use DEBUG level
     logging.getLogger("backend").setLevel(logging.DEBUG)
-    logging.getLogger("backend.routes").setLevel(logging.DEBUG)
-    logging.getLogger("backend.routes.chat").setLevel(logging.DEBUG)
-    logging.getLogger("backend.routes.chat_sessions").setLevel(logging.DEBUG)
-    logging.getLogger("backend.routes.llama_stack").setLevel(logging.DEBUG)
-    logging.getLogger("backend.routes.knowledge_bases").setLevel(logging.DEBUG)
+    logging.getLogger("backend.app").setLevel(logging.DEBUG)
+    logging.getLogger("backend.app.api").setLevel(logging.DEBUG)
+    logging.getLogger("backend.app.api.v1").setLevel(logging.DEBUG)
+    logging.getLogger("backend.app.api.v1.chat").setLevel(logging.DEBUG)
+    logging.getLogger("backend.app.api.v1.chat_sessions").setLevel(logging.DEBUG)
+    logging.getLogger("backend.app.api.v1.llama_stack").setLevel(logging.DEBUG)
+    logging.getLogger("backend.app.api.v1.knowledge_bases").setLevel(logging.DEBUG)
+    logging.getLogger("backend.app.services.chat").setLevel(logging.DEBUG)
 
 
 def _get_handlers(log_file: Optional[str], format_string: str) -> list:

backend/app/schemas/agent.py

@@ -28,7 +28,7 @@ class VirtualAgentBase(BaseModel):
     top_k: Optional[int] = None
     max_tokens: Optional[int] = None
     repetition_penalty: Optional[float] = None
-    max_infer_iters: Optional[int] = None
+    max_infer_iters: Optional[int] = 100
 
 
 class VirtualAgentCreate(VirtualAgentBase):