Merge pull request #22 from curieo-org/phoenix-tracing

phoenix tracing added for llm

backend/.env.template

@@ -83,5 +83,10 @@ POSTHOG_API_KEY=
 POSTHOG_HOST=
 
 # QDRANT API Configuration
+## need to be set before running the server
 QDRANT_API_KEY=
-QDRANT_TOP_K=
+QDRANT_TOP_K=
+
+# Phoenix Configuration
+## default values set in the code for local development
+PHOENIX_API_ENDPOINT=

backend/app/api/endpoints/search_endpoint.py

@@ -35,7 +35,7 @@
 @version(1, 0)
 async def get_search_results(
     query: str = "",
-    routecategory: RouteCategory = RouteCategory.NS
+    routecategory: RouteCategory = RouteCategory.PBW
 ) -> JSONResponse:
     if trace_transaction := sentry_sdk.Hub.current.scope.transaction:
         trace_transaction.set_tag("title", 'api_get_search_results')

backend/app/config.py

@@ -137,7 +137,10 @@
 CLINICAL_TRIALS_RESPONSE_REFINEMENT_PROGRAM: str = "app/dspy_integration/dspy_programs/clinical_trials_response_refinement.json"
 ORCHESRATOR_ROUTER_PROMPT_PROGRAM: str = "app/dspy_integration/dspy_programs/orchestrator_router_prompt.json"
 
+# Phoenix Configuration
+PHOENIX_API_ENDPOINT: str = config("PHOENIX_API_ENDPOINT", default="http://127.0.0.1:6007/v1/traces")
+
 #AI models
 ROUTER_MODEL: str = "gpt-3.5-turbo"
 SQL_GENERATION_MODEL: str =  "codellama/CodeLlama-13b-Instruct-hf"
-RESPONSE_SYNTHESIZER_MODEL: str = "NousResearch/Nous-Hermes-llama-2-7b"
+RESPONSE_SYNTHESIZER_MODEL: str = "NousResearch/Nous-Hermes-llama-2-7b"

backend/app/database/nebula_graph.py

@@ -60,7 +60,10 @@ def result_to_dict(self, result: ResultSet) -> dict[str, list]:
         for col_num in range(result.col_size()):
             col_name = columns[col_num]
             col_list = result.column_values(col_name)
-            result_dict[col_name] = [x.cast() for x in col_list]
+            if len(col_list) > 0:
+                result_dict[col_name] = [x.cast() for x in col_list]
+
+        assert len(result_dict) > 0
 
         return result_dict
 

backend/app/main.py

@@ -1,7 +1,8 @@
+import app.services.tracing # noqa
+
 from fastapi import FastAPI, HTTPException
 from fastapi.responses import RedirectResponse
 
-# from authx import AuthX, AuthXConfig
 from app.database.redis import Redis
 
 from app import config
@@ -11,9 +12,6 @@
 
 from app.middleware.process_time import ProcessTimeHeaderMiddleware
 
-from app.services.tracing import setup_tracing
-
-
 def get_application() -> FastAPI:
     application = FastAPI(
         title=config.PROJECT_NAME, debug=config.DEBUG, version=config.VERSION
@@ -27,9 +25,6 @@ def redirect_to_docs() -> RedirectResponse:  # pylint: disable=W0612
     async def startup():  # pylint: disable=W0612
         print()
 
-        # setup tracing
-        setup_tracing()
-
         # connect to redis
         cache = Redis()
         await cache.connect()

backend/app/router/orchestrator.py

@@ -47,7 +47,7 @@ def __init__(self, config):
     async def query_and_get_answer(
         self,
         search_text: str,
-        routecategory: RouteCategory = RouteCategory.NS
+        routecategory: RouteCategory = RouteCategory.PBW
     ) -> dict[str, str]:
         # search router call
         logger.info(

backend/app/services/tracing.py

@@ -5,7 +5,14 @@
 from sentry_sdk.integrations.starlette import StarletteIntegration
 from sentry_sdk.integrations.fastapi import FastApiIntegration
 
-from app.config import SENTRY_DSN, SENTRY_ENABLE_TRACING
+from openinference.instrumentation.llama_index import LlamaIndexInstrumentor
+from opentelemetry import trace as trace_api
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk import trace as trace_sdk
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+
+from app.config import SENTRY_DSN, SENTRY_ENABLE_TRACING, PHOENIX_API_ENDPOINT, ENVIRONMENT
 
 def setup_tracing():
     sentry_sdk.init(
@@ -24,4 +31,17 @@ def setup_tracing():
                 event_level=logging.WARNING
             ),
         ],
-    )
+    )
+
+
+    if ENVIRONMENT == 'production':
+        resource = Resource(attributes={})
+        tracer_provider = trace_sdk.TracerProvider(resource=resource)
+        span_exporter = OTLPSpanExporter(endpoint=PHOENIX_API_ENDPOINT)
+        span_processor = SimpleSpanProcessor(span_exporter=span_exporter)
+        tracer_provider.add_span_processor(span_processor=span_processor)
+        trace_api.set_tracer_provider(tracer_provider=tracer_provider)
+        LlamaIndexInstrumentor().instrument()
+
+
+setup_tracing()

backend/pyproject.toml

@@ -31,6 +31,9 @@ llama-index-llms-together = "^0.1.3"
 llama-index-postprocessor-cohere-rerank = "^0.1.2"
 together = "^0.2.11"
 llama-index-vector-stores-qdrant = "^0.1.4"
+openinference-semantic-conventions = "^0.1.5"
+openinference-instrumentation-llama-index = "^1.2.1"
+opentelemetry-exporter-otlp = "^1.23.0"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.1.1"