Merge branch 'main' of github.com:l3vels/L3AGI

apps/server/controllers/agent.py

@@ -11,12 +11,11 @@
 from typings.auth import UserAccount
 from utils.agent import convert_agents_to_agent_list, convert_model_to_response
 from utils.auth import authenticate
+from utils.system_message import SystemMessageBuilder
 
 # Standard library imports
 
 
-
-
 router = APIRouter()
 
 
@@ -213,7 +212,10 @@ def get_agent_by_id(
             status_code=404, detail="Agent not found"
         )  # Ensure consistent case in error messages
 
-    return convert_model_to_response(db_agent, is_system_message)
+    agent_with_configs = convert_model_to_response(db_agent)
+    system_message = SystemMessageBuilder(agent_with_configs).build()
+    agent_with_configs.system_message = system_message
+    return agent_with_configs
 
 
 @router.get("/discover/{id}", response_model=AgentWithConfigsOutput)

apps/server/controllers/integrations.py

@@ -0,0 +1,18 @@
+from typing import List
+
+from fastapi import APIRouter
+
+from integrations.get_integrations import get_all_integration_providers
+from typings.integrations import IntegrationOutput
+
+router = APIRouter()
+
+
+@router.get("", response_model=List[IntegrationOutput])
+def get_integrations() -> List[IntegrationOutput]:
+    """
+    Get all integrations
+
+    """
+
+    return get_all_integration_providers()

apps/server/datasources/file/file_retriever.py

@@ -77,7 +77,7 @@ def __init__(
 
         self.index_persist_dir = f"{Config.AWS_S3_BUCKET}/account_{account_id}/index/datasource_{self.datasource_id}"
 
-    def get_vector_store(self):
+    def get_vector_store(self, is_retriever: bool = False):
         vector_store: VectorStore
 
         if self.vector_store == VectorStoreProvider.ZEP.value:
@@ -100,7 +100,10 @@ def get_vector_store(self):
                 api_key=self.settings.pinecone_api_key,
                 environment=self.settings.pinecone_environment,
             )
-            pinecone.create_index(index_name, dimension=1536, metric="cosine")
+
+            if not is_retriever:
+                pinecone.create_index(index_name, dimension=1536, metric="cosine")
+
             pinecone_index = pinecone.Index(index_name)
 
             vector_store = PineconeVectorStore(
@@ -134,10 +137,14 @@ def index_documents(self, file_urls: List[str]):
             self.datasource_path.resolve(), filename_as_id=True
         ).load_data()
 
+        # Remove tmp directory
+        shutil.rmtree(self.datasource_path)
+
         embed_model = LangchainEmbedding(
             OpenAIEmbeddings(
-                openai_api_key=self.settings.openai_api_key, show_progress_bar=True
-            )
+                openai_api_key=self.settings.openai_api_key,
+                show_progress_bar=True,
+            ),
         )
 
         service_context = ServiceContext.from_defaults(
@@ -179,11 +186,8 @@ def index_documents(self, file_urls: List[str]):
         # Persist index to S3
         self.index.storage_context.persist(persist_dir=self.index_persist_dir, fs=s3)
 
-        # Remove tmp directory
-        shutil.rmtree(self.datasource_path)
-
     def load_index(self):
-        vector_store = self.get_vector_store()
+        vector_store = self.get_vector_store(is_retriever=True)
         storage_context = StorageContext.from_defaults(
             persist_dir=self.index_persist_dir, fs=s3, vector_store=vector_store
         )

apps/server/integrations/base.py

@@ -0,0 +1,88 @@
+from abc import abstractmethod
+from enum import Enum
+from typing import Dict, List, Optional
+
+from pydantic import BaseModel, Field, validator
+
+from models.config import ConfigModel
+from typings.agent import AgentOutput, ConfigsOutput
+from typings.config import ConfigQueryParams
+
+
+class IntegrationEnvKeyType(Enum):
+    STRING = "string"
+    FILE = "file"
+    INT = "int"
+
+    def __str__(self):
+        return self.value
+
+
+class IntegrationEnvKey(BaseModel):
+    label: str = Field()
+    key: str = Field()
+    key_type: IntegrationEnvKeyType = Field(default=IntegrationEnvKeyType.STRING)
+    is_required: bool = Field(default=False)
+    is_secret: bool = Field(default=False)
+    default_value: str = Field(default=None)
+
+    @validator("is_secret", "is_required", pre=True, always=True)
+    def check_bool(cls, v):
+        """Check if the value is a boolean."""
+        if v is None:
+            return False
+        elif isinstance(v, bool):
+            return v
+        else:
+            raise ValueError("Value should be a boolean")
+
+    @validator("key_type", pre=True, always=True)
+    def check_key_type(cls, v):
+        """Check if the value is a boolean."""
+        if v is None:
+            return IntegrationEnvKeyType.STRING
+        elif isinstance(v, IntegrationEnvKeyType):
+            return v
+        else:
+            raise ValueError("key_type should be string/file/integer")
+
+
+class BaseIntegrationTools:
+    id: str
+    configs: Dict[str, str] = {}
+    settings: Optional[ConfigsOutput] = None
+    voice_slug: Optional[str] = None
+    agent: Optional[AgentOutput] = None
+
+    def get_env_key(self, key: str):
+        return self.configs.get(key)
+
+
+class BaseIntegration(BaseModel):
+    id: str
+    name: str
+    description: str
+    slug: str
+    is_active: bool = Field(default=True)
+
+    def get_tools_with_configs(
+        self, db, account, settings, agent_with_configs
+    ) -> List[BaseIntegrationTools]:
+        configs = ConfigModel.get_configs(
+            db=db, query=ConfigQueryParams(toolkit_id=self.id), account=account
+        )
+        config_dict = {config.key: config.value for config in configs}
+        tools = self.get_tools()
+
+        for tool in tools:
+            tool.configs = config_dict
+            tool.voice_slug = self.slug
+            tool.settings = settings
+            tool.account = account
+
+        return tools
+
+    @abstractmethod
+    def get_env_keys(self) -> List[IntegrationEnvKey]:
+        # Add file related config keys here
+        pass

apps/server/integrations/get_integrations.py

@@ -0,0 +1,77 @@
+from typing import List
+
+from integrations.base import BaseIntegration
+from integrations.telegram.telegram_integration import TelegramIntegration
+from typings.integrations import IntegrationOutput
+
+INTEGRATIONS: List[BaseIntegration] = [
+    TelegramIntegration(),
+]
+
+
+COMING_SOON = [
+    {
+        "is_public": True,
+        "is_active": False,
+        "name": "WhatsApp",
+        "description": "Connect L3AGI chat with WhatsApp for smooth communication between sessions.",
+    },
+    {
+        "is_public": True,
+        "is_active": False,
+        "name": "Facebook Messenger",
+        "description": "Connect L3AGI chat with Facebook's Messenger for smooth communication between sessions.",
+    },
+]
+
+
+def get_all_integration_providers():
+    """Return a list of all integrations."""
+    result = []
+
+    for integration in INTEGRATIONS:
+        result.append(
+            IntegrationOutput(
+                id=integration.id,
+                is_public=True,
+                is_active=integration.is_active,
+                name=integration.name,
+                slug=integration.slug,
+                description=integration.description,
+                fields=[
+                    {
+                        "label": env_key.label,
+                        "key": env_key.key,
+                        "type": str(env_key.key_type),
+                        "is_required": env_key.is_required,
+                        "is_secret": env_key.is_secret,
+                        "default_value": str(env_key.default_value)
+                        if env_key.default_value is not None
+                        else None,
+                    }
+                    for env_key in integration.get_env_keys()
+                ],
+                # tools=[
+                #     {
+                #         "tool_id": tool.tool_id,
+                #         "name": tool.name,
+                #         "description": tool.description,
+                #     }
+                #     for tool in voice.get_tools()
+                # ],
+            )
+        )
+
+    for integration in COMING_SOON:
+        result.append(
+            IntegrationOutput(
+                id=None,  # Update this if COMING_SOON has an 'id' field
+                is_public=integration["is_public"],
+                is_active=integration["is_active"],
+                name=integration["name"],
+                description=integration["description"],
+                fields=[],  # Update this if COMING_SOON has a 'fields' field
+            )
+        )
+
+    return result

apps/server/integrations/telegram/telegram_integration.py

@@ -0,0 +1,31 @@
+from abc import ABC
+from typing import List
+
+from integrations.base import (BaseIntegration, IntegrationEnvKey,
+                               IntegrationEnvKeyType)
+
+
+class TelegramIntegration(BaseIntegration, ABC):
+    name: str = "Telegram Bot"
+    description: str = "Connect L3AGI chat with a Telegram bot for smooth communication between sessions."
+    slug: str = "telegram"
+
+    id = "2c2849ab-c74e-485c-9603-b8a18280c7b1"
+
+    def get_env_keys(self) -> List[IntegrationEnvKey]:
+        return [
+            IntegrationEnvKey(
+                label="Bot API Token Key",
+                key="TELEGRAM_BOT_API_KEY",
+                key_type=IntegrationEnvKeyType.STRING,
+                is_required=True,
+                is_secret=True,
+            ),
+            IntegrationEnvKey(
+                label="Bot Username",
+                key="TELEGRAM_BOT_USERNAME",
+                key_type=IntegrationEnvKeyType.STRING,
+                is_required=False,
+                is_secret=False,
+            ),
+        ]

apps/server/main.py

@@ -18,6 +18,7 @@
 from controllers.datasource import router as datasource_router
 from controllers.file import router as file_router
 from controllers.fine_tuning import router as fine_tuning_router
+from controllers.integrations import router as integrations_router
 from controllers.llm import router as llm_router
 from controllers.model import router as model_router
 from controllers.run import router as run_router
@@ -110,6 +111,7 @@ def jwt_exception_handler(request: Request, exc: AuthJWTException):
 app.include_router(model_router, prefix="/model")
 app.include_router(schedule_router, prefix="/schedule")
 app.include_router(api_key_router, prefix="/api-key")
+app.include_router(integrations_router, prefix="/integrations")
 app.include_router(fine_tuning_router, prefix="/fine-tuning")
 
 

apps/server/models/agent.py

@@ -34,7 +34,7 @@ class AgentModel(BaseModel):
     # __abstract__ = True
     __tablename__ = "agent"
     id = Column(UUID, primary_key=True, index=True, default=uuid.uuid4)
-    name = Column(String)
+    name = Column(String, index=True)
     avatar = Column(String(300), default=None)
     role = Column(String)
     parent_id = Column(
@@ -249,6 +249,22 @@ def get_public_agents(cls, db):
         )
         return agents
 
+    @classmethod
+    def get_agent_by_name(cls, session, account_id: UUID, agent_name: str):
+        agent = (
+            session.query(AgentModel)
+            .outerjoin(UserModel, AgentModel.created_by == UserModel.id)
+            .filter(
+                AgentModel.account_id == account_id,
+                AgentModel.is_deleted.is_(False),
+                AgentModel.name.ilike(f"%{agent_name}%"),
+            )
+            .options(joinedload(AgentModel.configs))
+            .options(joinedload(AgentModel.creator))
+            .first()
+        )
+        return agent
+
     @classmethod
     def get_agent_by_id(cls, db, agent_id):
         """

apps/server/typings/agent.py

@@ -33,6 +33,7 @@ class ConfigInput(BaseModel):
     suggestions: Optional[List[str]]
     greeting: Optional[str]
     text: Optional[str]
+    integrations: Optional[List[dict]]
     source_flow: Optional[str]
     synthesizer: Optional[str]
     default_voice: Optional[str]
@@ -58,6 +59,7 @@ class ConfigsOutput(BaseModel):
     suggestions: Optional[List[str]]
     greeting: Optional[str]
     text: Optional[str]
+    integrations: Optional[List[dict]]
     source_flow: Optional[str]
     synthesizer: Optional[str]
     default_voice: Optional[str]

apps/server/typings/integrations.py

@@ -0,0 +1,22 @@
+from typing import List, Optional
+
+from pydantic import UUID4, BaseModel
+
+
+class IntegrationFieldOutput(BaseModel):
+    label: str
+    key: str
+    type: str
+    is_required: bool
+    is_secret: bool
+    default_value: Optional[str] = None
+
+
+class IntegrationOutput(BaseModel):
+    id: Optional[UUID4]
+    is_active: bool
+    is_public: bool
+    name: str
+    description: Optional[str]
+    slug: Optional[str]
+    fields: Optional[List[IntegrationFieldOutput]]