From 57acf6510474eeedcf091767bf6f39622b8a9594 Mon Sep 17 00:00:00 2001 From: = Enea_Gore Date: Fri, 24 Jan 2025 17:08:37 +0100 Subject: [PATCH 01/23] extend UI with Compile Analytics, create new endpoint, test place holder html --- athena/athena/__init__.py | 5 +- athena/athena/endpoints.py | 20 +++++- llm_core/llm_core/models/openai.py | 2 +- .../module_text_llm/interactive_chart.html | 14 ++++ .../module_text_llm/__main__.py | 9 ++- .../analytics/pre_processing.py | 10 +++ .../batch_module_experiment.tsx | 43 +++++++++++- .../conduct_experiment/index.tsx | 25 ++++++- playground/src/hooks/athena/send_results.ts | 67 +++++++++++++++++++ playground/src/hooks/athena_fetcher.ts | 4 +- .../src/hooks/batch_module_experiment.ts | 49 +++++++++++++- 11 files changed, 234 insertions(+), 14 deletions(-) create mode 100644 modules/text/module_text_llm/interactive_chart.html create mode 100644 modules/text/module_text_llm/module_text_llm/analytics/pre_processing.py create mode 100644 playground/src/hooks/athena/send_results.ts diff --git a/athena/athena/__init__.py b/athena/athena/__init__.py index f7eee0079..104e5a862 100644 --- a/athena/athena/__init__.py +++ b/athena/athena/__init__.py @@ -7,7 +7,7 @@ from .schemas import ExerciseType, GradingCriterion, StructuredGradingInstruction, StructuredGradingCriterion from .metadata import emit_meta, get_meta from .experiment import get_experiment_environment -from .endpoints import submission_selector, submissions_consumer, feedback_consumer, feedback_provider, config_schema_provider, evaluation_provider # type: ignore +from .endpoints import submission_selector, submissions_consumer, generate_statistics, feedback_consumer, feedback_provider, config_schema_provider, evaluation_provider # type: ignore @app.get("/") def module_health(): @@ -37,5 +37,6 @@ def run_module(): "ExerciseType", "GradingCriterion", "StructuredGradingInstruction", - "StructuredGradingCriterion" + "StructuredGradingCriterion", + "generate_statistics" ] diff --git a/athena/athena/endpoints.py b/athena/athena/endpoints.py index 6d259a2a2..7a4e56606 100644 --- a/athena/athena/endpoints.py +++ b/athena/athena/endpoints.py @@ -1,9 +1,10 @@ # type: ignore # too much weird behavior of mypy with decorators import inspect -from fastapi import Depends, BackgroundTasks, Body +from fastapi import Depends, BackgroundTasks, Body, Request from pydantic import BaseModel, ValidationError from typing import TypeVar, Callable, List, Union, Any, Coroutine, Type +from fastapi.responses import HTMLResponse from athena.app import app from athena.authenticate import authenticated from athena.metadata import with_meta @@ -196,6 +197,22 @@ async def wrapper(request: SubmissionSelectorRequest): return wrapper +def generate_statistics(func): + @app.post("/generate_statistics", response_class=HTMLResponse) + async def wrapper(request: Request): + try: + results = await request.json() + await func(results) + file_path = "interactive_chart.html" + with open("interactive_chart.html", "r", encoding="utf-8") as file: + html_content = file.read() + # Return the HTML content as a response + return html_content #HTMLResponse(content=html_content,media_type="text/html", status_code=200) + + except Exception as e: + return {"error": str(e)} + + return wrapper def feedback_consumer(func: Union[ Callable[[E, S, List[F]], None], @@ -234,7 +251,6 @@ def feedback_consumer(func: Union[ submission_type = inspect.signature(func).parameters["submission"].annotation feedback_type = inspect.signature(func).parameters["feedbacks"].annotation.__args__[0] module_config_type = inspect.signature(func).parameters["module_config"].annotation if "module_config" in inspect.signature(func).parameters else None - @app.post("/feedbacks", responses=module_responses) @authenticated @with_meta diff --git a/llm_core/llm_core/models/openai.py b/llm_core/llm_core/models/openai.py index 7bcc0f11f..ad33d4374 100644 --- a/llm_core/llm_core/models/openai.py +++ b/llm_core/llm_core/models/openai.py @@ -81,7 +81,7 @@ class OpenAIModelConfig(ModelConfig): We generally recommend altering this or `top_p` but not both.\ """) - top_p: float = Field(default=1, ge=0, le=1, description="""\ + top_p: float = Field(default=0, ge=0, le=1, description="""\ An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. \ So 0.1 means only the tokens comprising the top 10% probability mass are considered. diff --git a/modules/text/module_text_llm/interactive_chart.html b/modules/text/module_text_llm/interactive_chart.html new file mode 100644 index 000000000..01ca80c5d --- /dev/null +++ b/modules/text/module_text_llm/interactive_chart.html @@ -0,0 +1,14 @@ + + + +
+
+ + \ No newline at end of file diff --git a/modules/text/module_text_llm/module_text_llm/__main__.py b/modules/text/module_text_llm/module_text_llm/__main__.py index 0bfc6e41d..74d7dbe14 100644 --- a/modules/text/module_text_llm/module_text_llm/__main__.py +++ b/modules/text/module_text_llm/module_text_llm/__main__.py @@ -3,14 +3,14 @@ import nltk import tiktoken -from athena import app, submission_selector, submissions_consumer, feedback_consumer, feedback_provider, evaluation_provider +from athena import app, submission_selector, submissions_consumer, generate_statistics,feedback_consumer, feedback_provider, evaluation_provider from athena.text import Exercise, Submission, Feedback from athena.logger import logger - from module_text_llm.config import Configuration from module_text_llm.evaluation import get_feedback_statistics, get_llm_statistics from module_text_llm.generate_evaluation import generate_evaluation from module_text_llm.approach_controller import generate_suggestions +from module_text_llm.analytics.pre_processing import pre_processing @submissions_consumer def receive_submissions(exercise: Exercise, submissions: List[Submission]): @@ -27,6 +27,11 @@ def select_submission(exercise: Exercise, submissions: List[Submission]) -> Subm def process_incoming_feedback(exercise: Exercise, submission: Submission, feedbacks: List[Feedback]): logger.info("process_feedback: Received %d feedbacks for submission %d of exercise %d.", len(feedbacks), submission.id, exercise.id) +@generate_statistics +async def compile_analytics(results: dict): + logger.info("generate_statistics: Generating statistics") + print(pre_processing(results)) + @feedback_provider async def suggest_feedback(exercise: Exercise, submission: Submission, is_graded: bool, module_config: Configuration) -> List[Feedback]: logger.info("suggest_feedback: %s suggestions for submission %d of exercise %d were requested, with approach: %s", diff --git a/modules/text/module_text_llm/module_text_llm/analytics/pre_processing.py b/modules/text/module_text_llm/module_text_llm/analytics/pre_processing.py new file mode 100644 index 000000000..9b8aac71a --- /dev/null +++ b/modules/text/module_text_llm/module_text_llm/analytics/pre_processing.py @@ -0,0 +1,10 @@ +import json + +def pre_processing(data): + # data = json.loads(data) + + results = [] + for result in data['data']: + results.append(result) + return results + # return json.dumps(data) diff --git a/playground/src/components/view_mode/evaluation_mode/conduct_experiment/batch_module_experiment.tsx b/playground/src/components/view_mode/evaluation_mode/conduct_experiment/batch_module_experiment.tsx index 560d8b204..3a7be5ed1 100644 --- a/playground/src/components/view_mode/evaluation_mode/conduct_experiment/batch_module_experiment.tsx +++ b/playground/src/components/view_mode/evaluation_mode/conduct_experiment/batch_module_experiment.tsx @@ -13,6 +13,7 @@ import { FullScreenHandle } from "react-full-screen"; import useHealth from "@/hooks/health"; import useBatchModuleExperiment from "@/hooks/batch_module_experiment"; +import { useSendResults } from "@/hooks/athena/send_results"; import { ModuleProvider } from "@/hooks/module_context"; import { ExperimentIdentifiersProvider } from "@/hooks/experiment_identifiers_context"; import { ModuleConfiguration } from "../configure_modules"; @@ -39,6 +40,8 @@ type ConductBatchModuleExperimentProps = { export type ConductBatchModuleExperimentHandles = { importData: ReturnType["importData"]; exportData: ReturnType["exportData"]; + analyseData: ReturnType["analyseData"] + getResults: ReturnType["getResults"]; }; // ForwardRef is needed to expose the ref to the parent component @@ -66,7 +69,8 @@ const ConductBatchModuleExperiment = React.forwardRef< const [showProgress, setShowProgress] = useState(true); const [isConfigModalOpen, setConfigModalOpen] = useState(false); - + // Use the `useSendResults` hook + const { mutate: sendResultsMutate } = useSendResults(); function handleOpenModal() { document.body.style.overflow = "hidden"; // Prevent scrolling setConfigModalOpen(true); @@ -80,10 +84,45 @@ const ConductBatchModuleExperiment = React.forwardRef< if (didStartExperiment) { moduleExperiment.startExperiment(); } - + // Function to analyze and send data + const analyseData = () => { + const payload = { + results: { + type: "results", + runId: moduleExperiment.data.runId, + experimentId: experiment.id, + moduleConfigurationId: moduleConfiguration.id, + step: moduleExperiment.data.step, + didSendSubmissions: moduleExperiment.data.didSendSubmissions, + sentTrainingSubmissions: moduleExperiment.data.sentTrainingSubmissions, + submissionsWithFeedbackSuggestions: Object.fromEntries( + moduleExperiment.data.submissionsWithFeedbackSuggestions + ), + }, + }; + + console.log( + "Analyzing data and preparing to send results to the backend...", + payload + ); + + // Send the payload directly without wrapping it in an additional object + sendResultsMutate(payload, { + onSuccess: () => { + console.log("Data analysis sent successfully!"); + }, + onError: (error) => { + console.error("Error sending data analysis to the backend:", error); + }, + }); + + return payload; // Return the payload for reference + }; useImperativeHandle(ref, () => ({ importData: moduleExperiment.importData, exportData: moduleExperiment.exportData, + analyseData: moduleExperiment.analyseData, + getResults: moduleExperiment.getResults, })); useEffect(() => { diff --git a/playground/src/components/view_mode/evaluation_mode/conduct_experiment/index.tsx b/playground/src/components/view_mode/evaluation_mode/conduct_experiment/index.tsx index 3c78e1ecb..33e02ab48 100644 --- a/playground/src/components/view_mode/evaluation_mode/conduct_experiment/index.tsx +++ b/playground/src/components/view_mode/evaluation_mode/conduct_experiment/index.tsx @@ -37,7 +37,6 @@ export default function ConductExperiment({ const [didStartExperiment, setDidStartExperiment] = useState(false); const [modulesStep, setModulesStep] = useState([]); - const [viewSubmissionIndex, setViewSubmissionIndex] = useState(0); const [moduleRenderOrder, setModuleRenderOrder] = useState( moduleConfigurations.map((_, index) => index) @@ -47,6 +46,21 @@ export default function ConductExperiment({ [] ); + // I have no idea how to use React. + // Data for analytics is aggregated and send to the last Ref to send to the backend because i cannot call the fetcher here. + const handleAnalysis = () => { + let aggregatedData: any[] = [] + let lastRef + moduleViewRefs.current.flatMap((moduleViewRef, index) => { + const data = moduleViewRef?.getResults(); + + console.log(data) + aggregatedData.push(data) + lastRef = moduleViewRef + }) + lastRef!.analyseData(aggregatedData) + } + const handleExport = () => { downloadJSONFiles( moduleViewRefs.current.flatMap((moduleViewRef, index) => { @@ -186,7 +200,7 @@ export default function ConductExperiment({ onClick={handleExport} > Export - +