00-playground/36-langchain/L5-Evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "52824b89-532a-4e54-87e9-1410813cd39e",
   "metadata": {},
   "source": [
    "# LangChain: Evaluation\n",
    "\n",
    "## Outline:\n",
    "\n",
    "* Example generation\n",
    "* Manual evaluation (and debuging)\n",
    "* LLM-assisted evaluation\n",
    "* LangChain evaluation platform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7ed03ed-1322-49e3-b2a2-33e94fb592ef",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "from dotenv import load_dotenv, find_dotenv\n",
    "_ = load_dotenv(find_dotenv()) # read local .env file"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "945d8abb-ba55-40a4-a3c5-6ad0dab73e3e",
   "metadata": {},
   "source": [
    "Note: LLM's do not always produce the same results. When executing the code in your notebook, you may get slightly different answers that those in the video."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24ff81cd-dce0-4344-8d45-4a98fd3a87c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# account for deprecation of LLM model\n",
    "import datetime\n",
    "# Get the current date\n",
    "current_date = datetime.datetime.now().date()\n",
    "\n",
    "# Define the date after which the model should be set to \"gpt-3.5-turbo\"\n",
    "target_date = datetime.date(2024, 6, 12)\n",
    "\n",
    "# Set the model variable based on the current date\n",
    "if current_date > target_date:\n",
    "    llm_model = \"gpt-3.5-turbo\"\n",
    "else:\n",
    "    llm_model = \"gpt-3.5-turbo-0301\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28008949",
   "metadata": {},
   "source": [
    "## Create our QandA application"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "974acf8e-8f88-42de-88f8-40a82cb58e8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.chains import RetrievalQA\n",
    "from langchain.chat_models import ChatOpenAI\n",
    "from langchain.document_loaders import CSVLoader\n",
    "from langchain.indexes import VectorstoreIndexCreator\n",
    "from langchain.vectorstores import DocArrayInMemorySearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ec1106d",
   "metadata": {},
   "outputs": [],
   "source": [
    "file = 'OutdoorClothingCatalog_1000.csv'\n",
    "loader = CSVLoader(file_path=file)\n",
    "data = loader.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b31c218f",
   "metadata": {},
   "outputs": [],
   "source": [
    "index = VectorstoreIndexCreator(\n",
    "    vectorstore_cls=DocArrayInMemorySearch\n",
    ").from_loaders([loader])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2006054",
   "metadata": {},
   "outputs": [],
   "source": [
    "llm = ChatOpenAI(temperature = 0.0, model=llm_model)\n",
    "qa = RetrievalQA.from_chain_type(\n",
    "    llm=llm, \n",
    "    chain_type=\"stuff\", \n",
    "    retriever=index.vectorstore.as_retriever(), \n",
    "    verbose=True,\n",
    "    chain_type_kwargs = {\n",
    "        \"document_separator\": \"<<<<>>>>>\"\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "791ebd73",
   "metadata": {},
   "source": [
    "### Coming up with test datapoints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fb04a0f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "data[10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe4a88c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "data[11]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d548aef",
   "metadata": {},
   "source": [
    "### Hard-coded examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c2d59bf2",
   "metadata": {},
   "outputs": [],
   "source": [
    "examples = [\n",
    "    {\n",
    "        \"query\": \"Do the Cozy Comfort Pullover Set\\\n",
    "        have side pockets?\",\n",
    "        \"answer\": \"Yes\"\n",
    "    },\n",
    "    {\n",
    "        \"query\": \"What collection is the Ultra-Lofty \\\n",
    "        850 Stretch Down Hooded Jacket from?\",\n",
    "        \"answer\": \"The DownTek collection\"\n",
    "    }\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c7ce3e4f",
   "metadata": {},
   "source": [
    "### LLM-Generated examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d44f8376",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.evaluation.qa import QAGenerateChain\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34e87816",
   "metadata": {},
   "outputs": [],
   "source": [
    "example_gen_chain = QAGenerateChain.from_llm(ChatOpenAI(model=llm_model))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31fb5bd0-fc2a-478c-9a09-6fe7e3307241",
   "metadata": {},
   "outputs": [],
   "source": [
    "# the warning below can be safely ignored"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62abae09",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_examples = example_gen_chain.apply_and_parse(\n",
    "    [{\"doc\": t} for t in data[:5]]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97ab28b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_examples[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0ebe4228",
   "metadata": {},
   "outputs": [],
   "source": [
    "data[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "faf25f2f",
   "metadata": {},
   "source": [
    "### Combine examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ada2a3fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "examples += new_examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9cdf5cf5",
   "metadata": {},
   "outputs": [],
   "source": [
    "qa.run(examples[0][\"query\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "63f3cb08",
   "metadata": {},
   "source": [
    "## Manual Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fcaf622e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import langchain\n",
    "langchain.debug = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8a142638",
   "metadata": {},
   "outputs": [],
   "source": [
    "qa.run(examples[0][\"query\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3d6bef0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Turn off the debug mode\n",
    "langchain.debug = False"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5bdbdce",
   "metadata": {},
   "source": [
    "## LLM assisted evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4dca05a",
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = qa.apply(examples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6012a3e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.evaluation.qa import QAEvalChain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "724b1c0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "llm = ChatOpenAI(temperature=0, model=llm_model)\n",
    "eval_chain = QAEvalChain.from_llm(llm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b46ae55",
   "metadata": {},
   "outputs": [],
   "source": [
    "graded_outputs = eval_chain.evaluate(examples, predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3437cfbe",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, eg in enumerate(examples):\n",
    "    print(f\"Example {i}:\")\n",
    "    print(\"Question: \" + predictions[i]['query'])\n",
    "    print(\"Real Answer: \" + predictions[i]['answer'])\n",
    "    print(\"Predicted Answer: \" + predictions[i]['result'])\n",
    "    print(\"Predicted Grade: \" + graded_outputs[i]['text'])\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d95133bb-43b6-441d-9ba5-00ef5609ccc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "graded_outputs[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fad0ddd1",
   "metadata": {},
   "source": [
    "## LangChain evaluation platform"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef63bb24",
   "metadata": {},
   "source": [
    "The LangChain evaluation platform, LangChain Plus, can be accessed here https://www.langchain.plus/.  \n",
    "Use the invite code `lang_learners_2023`"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "35b95a2e-3bc7-429a-83af-387239e7f2a1",
   "metadata": {},
   "source": [
    "Reminder: Download your notebook to you local computer to save your work."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be5b2aae",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "319798ba",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89a504ad",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dedd758b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36885b20",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65c6cfb6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ad3c7cc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26ac493e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f94cdacd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "langchain-c1",
   "language": "python",
   "name": "langchain-c1"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.17"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}