added more examples for annotation (#16)

Co-authored-by: Shahules786 <[email protected]>

Author: jjmachan

RAG-Experiments/gitlab-handbook/baseline.ipynb renamed to RAG-Experiments/gitlab-handbook/baseline_llamaindex.ipynb

@@ -0,0 +1,143 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "22d60861-5bb3-48b0-a6a1-c3008c63c14b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/jjmachan/.pyenv/versions/notes/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n",
+      "/home/jjmachan/.pyenv/versions/notes/lib/python3.12/site-packages/pysbd/segmenter.py:66: SyntaxWarning: invalid escape sequence '\\s'\n",
+      "  for match in re.finditer('{0}\\s*'.format(re.escape(sent)), self.original_text):\n",
+      "/home/jjmachan/.pyenv/versions/notes/lib/python3.12/site-packages/pysbd/lang/arabic.py:29: SyntaxWarning: invalid escape sequence '\\.'\n",
+      "  txt = re.sub('(?<={0})\\.'.format(am), '∯', txt)\n",
+      "/home/jjmachan/.pyenv/versions/notes/lib/python3.12/site-packages/pysbd/lang/persian.py:29: SyntaxWarning: invalid escape sequence '\\.'\n",
+      "  txt = re.sub('(?<={0})\\.'.format(am), '∯', txt)\n"
+     ]
+    },
+    {
+     "ename": "ImportError",
+     "evalue": "nltk is required for bleu score. Please install it using `pip install nltk`",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
+      "File \u001b[0;32m~/.pyenv/versions/notes/lib/python3.12/site-packages/ragas/metrics/_bleu_score.py:23\u001b[0m, in \u001b[0;36mBleuScore.__post_init__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 23\u001b[0m     \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mnltk\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mtokenize\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m word_tokenize\n\u001b[1;32m     24\u001b[0m     \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mnltk\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mtranslate\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mbleu_score\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m corpus_bleu\n",
+      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'nltk'",
+      "\nDuring handling of the above exception, another exception occurred:\n",
+      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[1], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mdatasets\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m load_dataset\n\u001b[0;32m----> 2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mragas\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m evaluate, EvaluationDataset\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mragas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmetrics\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m AspectCritic\n\u001b[1;32m      4\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mlangchain_openai\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mchat_models\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m ChatOpenAI\n",
+      "File \u001b[0;32m~/.pyenv/versions/notes/lib/python3.12/site-packages/ragas/__init__.py:2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mragas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdataset_schema\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m EvaluationDataset, MultiTurnSample, SingleTurnSample\n\u001b[0;32m----> 2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mragas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mevaluation\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m evaluate\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mragas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mrun_config\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m RunConfig\n\u001b[1;32m      5\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n",
+      "File \u001b[0;32m~/.pyenv/versions/notes/lib/python3.12/site-packages/ragas/evaluation.py:29\u001b[0m\n\u001b[1;32m     27\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mragas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mllms\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m llm_factory\n\u001b[1;32m     28\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mragas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mllms\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mbase\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m BaseRagasLLM, LangchainLLMWrapper\n\u001b[0;32m---> 29\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mragas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmetrics\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m AspectCritic\n\u001b[1;32m     30\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mragas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmetrics\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_answer_correctness\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m AnswerCorrectness\n\u001b[1;32m     31\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mragas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmetrics\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mbase\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[1;32m     32\u001b[0m     Metric,\n\u001b[1;32m     33\u001b[0m     MetricWithEmbeddings,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     37\u001b[0m     is_reproducable,\n\u001b[1;32m     38\u001b[0m )\n",
+      "File \u001b[0;32m~/.pyenv/versions/notes/lib/python3.12/site-packages/ragas/metrics/__init__.py:16\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mragas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmetrics\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_answer_similarity\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[1;32m     11\u001b[0m     AnswerSimilarity,\n\u001b[1;32m     12\u001b[0m     SemanticSimilarity,\n\u001b[1;32m     13\u001b[0m     answer_similarity,\n\u001b[1;32m     14\u001b[0m )\n\u001b[1;32m     15\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mragas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmetrics\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_aspect_critic\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m AspectCritic\n\u001b[0;32m---> 16\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mragas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmetrics\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_bleu_score\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m BleuScore\n\u001b[1;32m     17\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mragas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmetrics\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_context_entities_recall\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[1;32m     18\u001b[0m     ContextEntityRecall,\n\u001b[1;32m     19\u001b[0m     context_entity_recall,\n\u001b[1;32m     20\u001b[0m )\n\u001b[1;32m     21\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mragas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmetrics\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_context_precision\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[1;32m     22\u001b[0m     ContextPrecision,\n\u001b[1;32m     23\u001b[0m     ContextUtilization,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     26\u001b[0m     context_precision,\n\u001b[1;32m     27\u001b[0m )\n",
+      "File \u001b[0;32m~/.pyenv/versions/notes/lib/python3.12/site-packages/ragas/metrics/_bleu_score.py:54\u001b[0m\n\u001b[1;32m     50\u001b[0m     \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_ascore\u001b[39m(\u001b[38;5;28mself\u001b[39m, row: t\u001b[38;5;241m.\u001b[39mDict, callbacks: Callbacks) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mfloat\u001b[39m:\n\u001b[1;32m     51\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_single_turn_ascore(SingleTurnSample(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mrow), callbacks)\n\u001b[0;32m---> 54\u001b[0m bleu_score \u001b[38;5;241m=\u001b[39m \u001b[43mBleuScore\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m<string>:7\u001b[0m, in \u001b[0;36m__init__\u001b[0;34m(self, _required_columns, name, weights, sentence_segmenter)\u001b[0m\n",
+      "File \u001b[0;32m~/.pyenv/versions/notes/lib/python3.12/site-packages/ragas/metrics/_bleu_score.py:26\u001b[0m, in \u001b[0;36mBleuScore.__post_init__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     24\u001b[0m     \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mnltk\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mtranslate\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mbleu_score\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m corpus_bleu\n\u001b[1;32m     25\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m:\n\u001b[0;32m---> 26\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(\n\u001b[1;32m     27\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnltk is required for bleu score. Please install it using `pip install nltk`\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     28\u001b[0m     )\n\u001b[1;32m     29\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msegmenter \u001b[38;5;241m=\u001b[39m get_segmenter()\n\u001b[1;32m     30\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mword_tokenizer \u001b[38;5;241m=\u001b[39m word_tokenize\n",
+      "\u001b[0;31mImportError\u001b[0m: nltk is required for bleu score. Please install it using `pip install nltk`"
+     ]
+    }
+   ],
+   "source": [
+    "from datasets import load_dataset\n",
+    "from ragas import evaluate, EvaluationDataset\n",
+    "from ragas.metrics import AspectCritic\n",
+    "from langchain_openai.chat_models import ChatOpenAI\n",
+    "from ragas.llms import LangchainLLMWrapper\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0dab6de2-680a-4178-823f-8889a144a5d4",
+   "metadata": {},
+   "source": [
+    "## Dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "a2e9130c-b934-4331-a0c1-ce63089dcdf8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dataset = load_dataset(\"explodinggradients/aspect_critic_answer_correctness\",split=\"train\")\n",
+    "eval_dataset = EvaluationDataset.from_hf_dataset(dataset)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ffe5b8b9-8b1b-4ce3-95ce-51dab58458d0",
+   "metadata": {},
+   "source": [
+    "## Set Model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "2c0d7d0c-d7e7-4c50-b2a5-a7336744288e",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "EvaluationDataset(features=['user_input', 'response', 'reference'], len=50)"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "\n",
+    "llm_4o = LangchainLLMWrapper(ChatOpenAI(model=\"gpt-4o\"))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cb91f37f-3eb0-425a-8a47-7ca6729e498e",
+   "metadata": {},
+   "source": [
+    "## Evaluate"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2e52e40c-10c6-4cb0-8815-c01614225b2e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "critic = AspectCritic(name=\"answer_correctness\",definition=\"Given the user_input, reference and response. Is the response correct compared with the reference\")\n",
+    "results = evaluate(eval_dataset,metrics=[critic],llm=llm_4o)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "notes",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}