notebooks/calamita_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import json\n",
    "import pandas as pd\n",
    "\n",
    "regex_word_guess = '[\\d\\.|-] \\[.* = (.*)'\n",
    "regex_firstpass = 'Prima lettura: (.*)'\n",
    "regex_solution_word = \"\\d+ = (.*)\"\n",
    "regex_solution = \"Soluzione: (.*)\"\n",
    "\n",
    "def parse_generation(doc_id, resp):\n",
    "    try:\n",
    "        word_guesses = \";\".join(re.findall(regex_word_guess, resp))\n",
    "    except:\n",
    "        word_guesses = \"\"\n",
    "    try:\n",
    "        first_pass = re.findall(regex_firstpass, resp)[0]\n",
    "    except:\n",
    "        first_pass = \"\"\n",
    "    try:\n",
    "        solution_words = \";\".join(re.findall(regex_solution_word, resp))\n",
    "    except:\n",
    "        solution_words = \"\"\n",
    "    try:\n",
    "        solution = re.findall(regex_solution, resp)[0]\n",
    "    except:\n",
    "        solution = \"\"\n",
    "    return {\n",
    "        \"idx\": doc_id,\n",
    "        \"word_guesses\": word_guesses,\n",
    "        \"first_pass\": first_pass,\n",
    "        \"solution_words\": solution_words,\n",
    "        \"solution\": solution,\n",
    "    }\n",
    "\n",
    "def parse_calamita_outputs(\n",
    "    setting: str = \"eureka_original\",\n",
    "    model: str = \"llama3.1_8b\",\n",
    "    do_gold: bool = False\n",
    ") -> pd.DataFrame:\n",
    "    examples = []\n",
    "    with open(f\"../eureka-rebus-calamita-2024/outputs/{setting}/{model}.jsonl\") as f:\n",
    "        lines = f.readlines()\n",
    "        for line in lines:\n",
    "            entry = json.loads(line)\n",
    "            ex_idx = entry[\"doc_id\"]\n",
    "            if do_gold:\n",
    "                example = {\n",
    "                    \"idx\": entry[\"doc_id\"],\n",
    "                    \"word_guesses\": entry[\"doc\"][\"word_guesses\"],\n",
    "                    \"first_pass\": entry[\"doc\"][\"first_pass\"],\n",
    "                    \"solution_words\": entry[\"doc\"][\"solution_words\"],\n",
    "                    \"solution\": entry[\"doc\"][\"solution\"],\n",
    "                }\n",
    "            else:\n",
    "                example = parse_generation(ex_idx, entry[\"filtered_resps\"][0])\n",
    "            examples.append(example)\n",
    "        examples = sorted(examples, key=lambda x: x[\"idx\"])\n",
    "        return pd.DataFrame(examples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "gold_df = parse_calamita_outputs(do_gold=True)\n",
    "gold_df.to_csv(\"calamita_gold.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "MODELS = [\n",
    "    \"llama3.1_8b\",\n",
    "    \"llama3.1_70b\",\n",
    "    \"llamantino3_8b\",\n",
    "    \"minerva_7b\"\n",
    "]\n",
    "\n",
    "SETTINGS = [\"eureka_hints\", \"eureka_original\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for model in MODELS:\n",
    "    for setting in SETTINGS:\n",
    "        df = parse_calamita_outputs(setting=setting, model=model)\n",
    "        df.to_csv(f\"../eureka-rebus-calamita-2024/outputs/{setting}/{model}_results.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==========\n",
      "Evaluating llama3.1_8b\n",
      "EUREKA_HINTS\n",
      "Word Guess Accuracy: 0.07\n",
      "Word Guess Length: 0.16\n",
      "Error: nan legno abete cicno pi abietina cicuta sasso cicero cicale\n",
      "First Pass Exact Match:: 0.0\n",
      "Solution Word Accuracy: 0.01\n",
      "Solution Word Lengths: 0.09\n",
      "Solution Exact Match: 0.0\n",
      "\n",
      "\n",
      "EUREKA_ORIGINAL\n",
      "Word Guess Accuracy: 0.09\n",
      "Word Guess Length: 0.18\n",
      "First Pass Exact Match:: 0.0\n",
      "Solution Word Accuracy: 0.02\n",
      "Solution Word Lengths: 0.17\n",
      "Solution Exact Match: 0.0\n",
      "\n",
      "\n",
      "==========\n",
      "Evaluating llama3.1_70b\n",
      "EUREKA_HINTS\n",
      "Word Guess Accuracy: 0.3\n",
      "Word Guess Length: 0.49\n",
      "First Pass Exact Match:: 0.04\n",
      "Solution Word Accuracy: 0.06\n",
      "Solution Word Lengths: 0.21\n",
      "Solution Exact Match: 0.01\n",
      "\n",
      "\n",
      "EUREKA_ORIGINAL\n",
      "Word Guess Accuracy: 0.34\n",
      "Word Guess Length: 0.42\n",
      "First Pass Exact Match:: 0.07\n",
      "Solution Word Accuracy: 0.08\n",
      "Solution Word Lengths: 0.26\n",
      "Solution Exact Match: 0.0\n",
      "\n",
      "\n",
      "==========\n",
      "Evaluating llamantino3_8b\n",
      "EUREKA_HINTS\n",
      "Word Guess Accuracy: 0.0\n",
      "Word Guess Length: 0.01\n",
      "First Pass Exact Match:: 0.0\n",
      "Solution Word Accuracy: 0.0\n",
      "Solution Word Lengths: 0.0\n",
      "Solution Exact Match: 0.0\n",
      "\n",
      "\n",
      "EUREKA_ORIGINAL\n",
      "Word Guess Accuracy: 0.0\n",
      "Word Guess Length: 0.01\n",
      "First Pass Exact Match:: 0.0\n",
      "Solution Word Accuracy: 0.0\n",
      "Solution Word Lengths: 0.0\n",
      "Solution Exact Match: 0.0\n",
      "\n",
      "\n",
      "==========\n",
      "Evaluating minerva_7b\n",
      "EUREKA_HINTS\n",
      "Word Guess Accuracy: 0.0\n",
      "Word Guess Length: 0.0\n",
      "First Pass Exact Match:: 0.0\n",
      "Solution Word Accuracy: 0.0\n",
      "Solution Word Lengths: 0.0\n",
      "Solution Exact Match: 0.0\n",
      "\n",
      "\n",
      "EUREKA_ORIGINAL\n",
      "Word Guess Accuracy: 0.0\n",
      "Word Guess Length: 0.0\n",
      "First Pass Exact Match:: 0.0\n",
      "Solution Word Accuracy: 0.0\n",
      "Solution Word Lengths: 0.01\n",
      "Solution Exact Match: 0.0\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for model in MODELS:\n",
    "    print(\"=\" * 10 + f\"\\nEvaluating {model}\")\n",
    "    for setting in SETTINGS:\n",
    "        print(f\"{setting.upper()}\")\n",
    "        !python ../scripts/evaluate.py \\\n",
    "        --predicted_outputs ../eureka-rebus-calamita-2024/outputs/{setting}/{model}_results.csv \\\n",
    "        --gold_outputs ../eureka-rebus-calamita-2024/outputs/calamita_gold.csv \\\n",
    "        --word_frequencies ../outputs/word_frequencies_paisa.json \\\n",
    "        --word_frequencies_fp_train ../eureka-rebus/word_frequencies_fp_train.json \\\n",
    "        --word_frequencies_solution_train ../eureka-rebus/word_frequencies_solution_train.json\n",
    "        print(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}