llama_finetune_honaz.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "b197bec6-8ba7-4e1f-960d-67136636ae99",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from trl import SFTTrainer\n",
    "from datasets import load_dataset, concatenate_datasets\n",
    "from transformers import TrainingArguments, TextStreamer\n",
    "from unsloth import FastLanguageModel, is_bfloat16_supported\n",
    "from evaluate import load\n",
    "#\n",
    "from dotenv import load_dotenv\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "import os\n",
    "import wandb\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e3f82530-c314-4d26-96c6-fa016db66836",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Using wandb-core as the SDK backend.  Please refer to https://wandb.me/wandb-core for more information.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using GPU: NVIDIA A100-SXM4-40GB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33merkara\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m If you're specifying your api key in code, ensure this code is not shared publicly.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Consider setting the WANDB_API_KEY environment variable, or running `wandb login` from the command line.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /home/ubuntu/.netrc\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.19.0"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/home/ubuntu/wandb/run-20241209_201522-p4zapx3l</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/erkara/llama_custom_towndata/runs/p4zapx3l' target=\"_blank\">rose-blaze-7</a></strong> to <a href='https://wandb.ai/erkara/llama_custom_towndata' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/developer-guide' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/erkara/llama_custom_towndata' target=\"_blank\">https://wandb.ai/erkara/llama_custom_towndata</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/erkara/llama_custom_towndata/runs/p4zapx3l' target=\"_blank\">https://wandb.ai/erkara/llama_custom_towndata/runs/p4zapx3l</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<button onClick=\"this.nextSibling.style.display='block';this.style.display='none';\">Display W&B run</button><iframe src='https://wandb.ai/erkara/llama_custom_towndata/runs/p4zapx3l?jupyter=true' style='border:none;width:100%;height:420px;display:none;'></iframe>"
      ],
      "text/plain": [
       "<wandb.sdk.wandb_run.Run at 0x70ce89969900>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Set environment variables and device\n",
    "device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "if torch.cuda.is_available():\n",
    "    print(f\"Using GPU: {torch.cuda.get_device_name()}\")\n",
    "else:\n",
    "    print(f\"CUDA not found\")\n",
    "\n",
    "\n",
    "# ADD YOUR KEY IN THIS FILE\n",
    "load_dotenv(\"all_keys.txt\")\n",
    "\n",
    "# Register HuggingFace\n",
    "hf_token = os.getenv(\"HF_TOKEN\")\n",
    "\n",
    "# initizalie wandb with gradient info as well.\n",
    "wandb_api_key = os.getenv(\"WANDB_API_KEY\")\n",
    "wandb.login(key=wandb_api_key)\n",
    "wandb.init(project=\"llama_custom_towndata\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5ea9feb0-308d-4860-a844-c57c00377767",
   "metadata": {},
   "source": [
    "# Instruction Fine-Tuning with Unsloth"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ecefbd9-4fe6-4dcb-8bae-eff8f5679db4",
   "metadata": {},
   "source": [
    "I set out to create a niche dataset all about my tiny home town (not city), [Honaz](https://en.wikipedia.org/wiki/Honaz) in Turkey. This dataset was built entirely from three `Turkish` articles in the [DergiPark](https://dergipark.org.tr/) system. The idea here is simple: take rich, detailed information about a small town in Turkish (because that's the language of the original sources), and see if a model can actually learn from it. Why bother creating such a niche dataset in Turkish? Well, fine-tuning models on localized, specific datasets lets us test if they can adapt to truly unique and specialized topics. This is good because we can easily assess if the model really learns something. \n",
    "Here’s the plan:\n",
    "\n",
    "1. Data Creation: The dataset creation process is detailed in another notebook, where I carefully compiled and cleaned the articles.\n",
    "   \n",
    "2. Fine-Tuning: Using the Unsloth library, I fine-tuned Llama-3.2-1B-Instruct on this dataset. The goal? To see if the model can genuinely learn and retain the knowledge in those articles.\n",
    "\n",
    "3. Evaluation: I put the fine-tuned model to the test with a curated set of questions about Honaz, comparing it against the original model. This is the best way of testing to see if finetuning does anyhting good. In addition, I provide a script to evaluate the BLUE scores on the testing data as well.\n",
    "\n",
    "Both the dataset and finetuned model can be accessed at:\n",
    "\n",
    "Dataset: [https://huggingface.co/datasets/erdi28/honaz_instruction_dataset](https://huggingface.co/datasets/erdi28/honaz_instruction_dataset)\n",
    "\n",
    "Fine-tuned model: [https://huggingface.co/erdi28/finetune_llama_honaz](https://huggingface.co/erdi28/finetune_llama_honaz)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7579d4cc-d5d0-4629-a814-dcf53e698485",
   "metadata": {},
   "source": [
    "## Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ebf0c36d-8805-4a11-9210-f646ae32a4c9",
   "metadata": {},
   "source": [
    "You can find the details in my hub but here is what I simply did. I found three niche articles in Turkish about Honaz and translated them to Turkish via OpenAI APIs, more specifically(GPT4-4o-mini). Articles are about history and local vegitation of Honaz as well as a population exhance event that deeply effected the town in 1920s. Then created an instruction dataset from it based on carefully curated instructions. This process gave us around 1000 instruction-response pairs\n",
    "\n",
    "Since 1K is not really sufficient to train an LLM, I added 5K more pairs from a public dataset and merged with mine to finalize the dataset. See below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4169e015-179d-4b79-8f19-a254fb5ef6ce",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'instruction': 'Answer a question based on the following content. The vegetation of Honaz Mountain and its surroundings generally consists of dry forests dominated by red pines at lower elevations and black pines at higher elevations. The northern slopes of the Honaz massif are influenced by the Mediterranean climate that penetrates along the Büyük Menderes valley, while the interior areas and southern slopes are under the influence of a continental climate. As a result, the vegetation on the northern and southern slopes of the massif differs. On the more humid northern slopes, a richer and more diverse maquis formation has developed, whereas on the southern slopes, a garigue formation consisting of only the most drought-resistant maquis species is prevalent.',\n",
       " 'output': 'What types of vegetation are found on the northern and southern slopes of Honaz Mountain?\\n\\nThe northern slopes of Honaz Mountain feature a richer and more diverse maquis formation due to the influence of the Mediterranean climate, while the southern slopes are dominated by a garigue formation consisting of drought-resistant maquis species, influenced by the continental climate.'}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "honaz_dataset = load_dataset(\"erdi28/honaz_instruction_dataset\")\n",
    "honaz_train = honaz_dataset[\"train\"]\n",
    "honaz_train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "def63ea2-f30b-44f9-9c63-d7d53e2a2886",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'instruction': 'Write Python code to solve the task:\\nIn programming, hexadecimal notation is often used.\\n\\nIn hexadecimal notation, besides the ten digits 0, 1, ..., 9, the six letters `A`, `B`, `C`, `D`, `E` and `F` are used to represent the values 10, 11, 12, 13, 14 and 15, respectively.\\n\\nIn this problem, you are given two letters X and Y. Each X and Y is `A`, `B`, `C`, `D`, `E` or `F`.\\n\\nWhen X and Y are seen as hexadecimal numbers, which is larger?\\n\\nConstraints\\n\\n* Each X and Y is `A`, `B`, `C`, `D`, `E` or `F`.\\n\\nInput\\n\\nInput is given from Standard Input in the following format:\\n\\n\\nX Y\\n\\n\\nOutput\\n\\nIf X is smaller, print `<`; if Y is smaller, print `>`; if they are equal, print `=`.\\n\\nExamples\\n\\nInput\\n\\nA B\\n\\n\\nOutput\\n\\n<\\n\\n\\nInput\\n\\nE C\\n\\n\\nOutput\\n\\n>\\n\\n\\nInput\\n\\nF F\\n\\n\\nOutput\\n\\n=',\n",
       " 'output': \"Step 1:  To compare two hexadecimal numbers represented by the letters X and Y, we need to understand the hexadecimal number system and how to compare hexadecimals.\\nStep 2:  Since the possible letters are `A`, `B`, `C`, `D`, `E`, and `F`, we can directly compare the letters as hexadecimal numbers.\\nStep 3:  We can then print the comparison result based on whether X is smaller, Y is smaller, or they are equal.\\n```python\\n# Step 4:  Read the input as a string and split it into two letters X and Y.\\na, b = input().split()\\n# Step 5:  Compare the two letters as hexadecimal numbers and print the result based on the comparison.\\nif a == b:\\n    print('=')\\nif a < b:\\n    print('<')\\nif a > b:\\n    print('>')\\n\\n```\"}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load the Alpaca dataset\n",
    "N = 5000\n",
    "alpaca_dataset = load_dataset(\"mlabonne/FineTome-Alpaca-100k\",split=f\"train[:{N}]\")\n",
    "alpaca_train = alpaca_dataset.remove_columns(['score', 'source'])\n",
    "alpaca_train[10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5db702ff-a6d1-4a87-b2b4-085f76ce80f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "#combine both datassets\n",
    "dataset = concatenate_datasets([honaz_train, alpaca_train])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "115474a0-2552-44a6-9b3b-5bf3a32e7a7a",
   "metadata": {},
   "source": [
    "## Model Tranining"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c32235f-36aa-4008-981f-de308db9e44d",
   "metadata": {},
   "source": [
    "Unsloth can train 2-5X faster with custom kernels. We will start with it. However since it does not support multi-gpu training, later on, we plan to switch to Axolotl. We will load the model with 4bit quantization to reduce memory usage."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3f735661-d11b-4588-81de-6f043213b1a4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==((====))==  Unsloth 2024.12.4: Fast Llama patching. Transformers:4.46.3.\n",
      "   \\\\   /|    GPU: NVIDIA A100-SXM4-40GB. Max memory: 39.381 GB. Platform: Linux.\n",
      "O^O/ \\_/ \\    Torch: 2.5.1+cu124. CUDA: 8.0. CUDA Toolkit: 12.4. Triton: 3.1.0\n",
      "\\        /    Bfloat16 = TRUE. FA [Xformers = 0.0.28.post3. FA2 = False]\n",
      " \"-____-\"     Free Apache license: http://github.com/unslothai/unsloth\n",
      "Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!\n"
     ]
    }
   ],
   "source": [
    "max_seq_length = 2048\n",
    "ref_model, tokenizer = FastLanguageModel.from_pretrained(model_name = \"unsloth/Llama-3.2-1B-Instruct\",\n",
    "                                                     max_seq_length = max_seq_length,\n",
    "                                                     dtype = None,                          # auto detection\n",
    "                                                     load_in_4bit = True)                   # check if "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ead4a81b-e087-41fd-99ba-a7ddc1f3926c",
   "metadata": {},
   "source": [
    "Let's go ahead and see if our baseline model knows something about our data. It is clear that it has no idea about it, which is what we want at this point."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4d8f9e1c-ff77-4a80-8af2-9e4cd3cd6a3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the Alpaca prompt template ( we dont have \"Input\" field)\n",
    "alpaca_prompt = \"\"\"Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
    "\n",
    "### Instruction:\n",
    "{}\n",
    "\n",
    "### Response:\n",
    "{}\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "aaa176d4-cb6c-4ab1-a8d9-0a5b60f4087f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<|begin_of_text|>Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
      "\n",
      "### Instruction:\n",
      "How did the population of Honaz evolve during the late 19th century?\n",
      "\n",
      "### Response:\n",
      "The population of Honaz, a small town in Afghanistan, likely grew significantly during the late 19th century due to an increase in agricultural production and trade. As Afghanistan was under British influence, there was an influx of British settlers and traders in the region. The British also established trade routes and missions in the area, which led to an expansion of local agriculture. This led to a surge in population as the local population grew to accommodate the increased agricultural production and trade. Additionally, the town's strategic\n"
     ]
    }
   ],
   "source": [
    "def generate_streaming_text(model, tokenizer, prompt, max_new_tokens=256, prompt_template=alpaca_prompt):\n",
    "    \"\"\"\n",
    "    Generates text from a model with streaming output.\n",
    "    \"\"\"\n",
    "    # format the input and set up the stremaer\n",
    "    message = prompt_template.format(prompt, \"\")\n",
    "    inputs = tokenizer([message], return_tensors=\"pt\").to(device)\n",
    "    text_streamer = TextStreamer(tokenizer)\n",
    "    \n",
    "    # Generate text with streaming\n",
    "    _ = model.generate(\n",
    "        **inputs, \n",
    "        streamer=text_streamer, \n",
    "        max_new_tokens=max_new_tokens, \n",
    "        use_cache=True\n",
    "    )\n",
    "\n",
    "# test\n",
    "ref_model = FastLanguageModel.for_inference(ref_model)\n",
    "prompt = \"How did the population of Honaz evolve during the late 19th century?\"\n",
    "generate_streaming_text(ref_model, tokenizer, prompt, max_new_tokens=100)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "fcaf01cb-16da-4fa8-91fc-769c981b46ae",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<|begin_of_text|>Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
      "\n",
      "### Instruction:\n",
      "What are the climatic influences on Honaz Mountain’s vegetation?\n",
      "\n",
      "### Response:\n",
      "The climatic influences on Honaz Mountain's vegetation are primarily influenced by temperature and precipitation. The mountain's high elevation, with an average elevation of 4,500 meters, results in cooler temperatures. The cooler temperatures, combined with the cloud cover and humidity, support a diverse range of vegetation, including alpine meadows and forests. The precipitation patterns are also crucial, with the mountain receiving significant snowfall during the winter months. This snowpack supports the growth of certain plant species that are adapted to\n"
     ]
    }
   ],
   "source": [
    "prompt = \"What are the climatic influences on Honaz Mountain’s vegetation?\"\n",
    "generate_streaming_text(ref_model, tokenizer, prompt, device, max_new_tokens=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0044ae03-4d57-42bc-95cb-7a500cce854a",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "f86efe16-2f9d-478a-91b0-438579ecd86b",
   "metadata": {},
   "source": [
    "- We would like to use LORA for fine-tuning. Unsloth supports several accelerated options, see below. We target all linear layers in LLama model. Note that this is model spesific, depending on the arthitecture, we may need to adjust it. However, we are particularly interested in layers that are part of the attention mechanism (e.g., q_proj for query projection, k_proj for key projection, etc.) and feed-forward layers (up_proj, down_proj)\n",
    "- `Gradient checkpointing` saves GPU memory by not storing all intermediate activations during the forward pass and instead recomputes them on-the-fly during the backward pass, trading memory for a bit more computation. Unsloth optimizes this further for super long sequences with some strategy to where to apply them.\n",
    "- `LoftQ (LoRA with Quantization)` applies additional quantization optimizations *specifically to the LoRA* adapter weights during training, even if the main model is already loaded in 4-bit precision. In other words, matrices introduced by LoRA are stored in 4-bit.\n",
    "- Note that some of these options (i.e use_gradient_checkpointing ) will **override TrainingArguments class** below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "6ee02d47-4b3b-440a-8a72-c71950c7f319",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unsloth 2024.12.4 patched 16 layers with 16 QKV layers, 16 O layers and 16 MLP layers.\n"
     ]
    }
   ],
   "source": [
    "model = FastLanguageModel.get_peft_model(\n",
    "    ref_model,\n",
    "    r = 32,            \n",
    "    target_modules = [\"q_proj\", \"k_proj\", \"v_proj\", \"o_proj\",\n",
    "                      \"gate_proj\", \"up_proj\", \"down_proj\",],\n",
    "    lora_alpha = 32,\n",
    "    lora_dropout = 0,         # dropout after adapter, \"0\" is optimized\n",
    "    bias = \"none\",            # biases in the model remain frozen (not updated), \"none\" is optimized\n",
    "    # [NEW] \"unsloth\" uses 30% less VRAM, fits 2x larger batch sizes!\n",
    "    use_gradient_checkpointing = \"unsloth\", # True or \"unsloth\" for very long context\n",
    "    use_rslora = False,     # rank stabilized LoRA\n",
    "    loftq_config = None,    # And LoftQ\n",
    "    random_state = 1234,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "402eb8dc-d9f3-44d3-b50a-59c606fe321a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Mapping function to format the dataset\n",
    "EOS_TOKEN = tokenizer.eos_token\n",
    "def formatting_prompts_func(examples):\n",
    "    instructions = examples[\"instruction\"]\n",
    "    outputs = examples[\"output\"]\n",
    "    texts = []\n",
    "    for instruction, output in zip(instructions, outputs):\n",
    "        text = alpaca_prompt.format(instruction, output) + EOS_TOKEN\n",
    "        texts.append(text)\n",
    "    return {\"text\": texts}\n",
    "\n",
    "# Apply the mapping function to the dataset\n",
    "dataset = dataset.map(formatting_prompts_func, batched=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "32eaac3c-7e96-4316-8d3b-f2e9090a3b36",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Answer a question based on the following content. The vegetation of Honaz Mountain and its surroundings generally consists of dry forests dominated by red pines at lower elevations and black pines at higher elevations. The northern slopes of the Honaz massif are influenced by the Mediterranean climate that penetrates along the Büyük Menderes valley, while the interior areas and southern slopes are under the influence of a continental climate. As a result, the vegetation on the northern and southern slopes of the massif differs. On the more humid northern slopes, a richer and more diverse maquis formation has developed, whereas on the southern slopes, a garigue formation consisting of only the most drought-resistant maquis species is prevalent.\n",
      "=====================================================\n",
      "What types of vegetation are found on the northern and southern slopes of Honaz Mountain?\n",
      "\n",
      "The northern slopes of Honaz Mountain feature a richer and more diverse maquis formation due to the influence of the Mediterranean climate, while the southern slopes are dominated by a garigue formation consisting of drought-resistant maquis species, influenced by the continental climate.\n",
      "=====================================================\n",
      "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
      "\n",
      "### Instruction:\n",
      "Answer a question based on the following content. The vegetation of Honaz Mountain and its surroundings generally consists of dry forests dominated by red pines at lower elevations and black pines at higher elevations. The northern slopes of the Honaz massif are influenced by the Mediterranean climate that penetrates along the Büyük Menderes valley, while the interior areas and southern slopes are under the influence of a continental climate. As a result, the vegetation on the northern and southern slopes of the massif differs. On the more humid northern slopes, a richer and more diverse maquis formation has developed, whereas on the southern slopes, a garigue formation consisting of only the most drought-resistant maquis species is prevalent.\n",
      "\n",
      "### Response:\n",
      "What types of vegetation are found on the northern and southern slopes of Honaz Mountain?\n",
      "\n",
      "The northern slopes of Honaz Mountain feature a richer and more diverse maquis formation due to the influence of the Mediterranean climate, while the southern slopes are dominated by a garigue formation consisting of drought-resistant maquis species, influenced by the continental climate.<|eot_id|>\n"
     ]
    }
   ],
   "source": [
    "idx = 0\n",
    "print(dataset[0]['instruction'])\n",
    "print(\"=====================================================\")\n",
    "print(dataset[0]['output'])\n",
    "print(\"=====================================================\")\n",
    "print(dataset[0]['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7cb0e052-45fa-434e-b4fa-f7ee08fb2be3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "67ed5992-384f-4e20-b3d3-815771d17121",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5409\n",
      "601\n"
     ]
    }
   ],
   "source": [
    "# do train-test split\n",
    "dataset = dataset.train_test_split(test_size=0.10)\n",
    "print(dataset[\"train\"].num_rows)\n",
    "print(dataset[\"test\"].num_rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "42774c1d-66d7-40d9-9b50-44ed32f417f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "#wandb.watch(model, log=\"all\", log_freq=2) % this slows down the tranining considerably  "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "36d7c073-8b7a-4363-8f10-6aa78256df03",
   "metadata": {},
   "source": [
    "This is the main driver function pretty much all transformers based supervised-fine tuning routines nowadays. The list of arguments are overwhelming but let's touch some of the interesting/less obvious ones:\n",
    "\n",
    "-  **`dataset_num_proc`**: number of parallel processes(CPU cores) to use for preprocessing (i.e tokenization, formatting the input/output pairs, truncation etc.)\n",
    "  \n",
    "-  **`packing`**: combines multiple shorter examples into a single sequence (up to max_seq_length), reducing the amount of padding caused by short sequences. There is a nice [blog post](https://wandb.ai/capecape/alpaca_ft/reports/How-to-Fine-tune-an-LLM-Part-3-The-HuggingFace-Trainer--Vmlldzo1OTEyNjMy) you can check out. This is doing `ConstantLengthDataset` behind the scene.\n",
    "\n",
    "-  **`per_device_train_batch_size`**: number of samples processed per GPU during training in each forward pass.\n",
    "\n",
    "-  **`gradient_accumulation_steps`**: accumulates gradients over multiple steps before performing an optimizer update. Basically updates happen in every\n",
    "\n",
    "                       effective batch size = per_device_train_batch_size * gradient_accumulation_steps\n",
    "\n",
    "Other arguments are annoated below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "4feb3904-d763-4284-a74b-9cef1205cec2",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer = SFTTrainer(\n",
    "    model = model,\n",
    "    tokenizer = tokenizer,\n",
    "    train_dataset = dataset[\"train\"],\n",
    "    eval_dataset = dataset[\"test\"],\n",
    "    dataset_text_field = \"text\",\n",
    "    max_seq_length = max_seq_length,\n",
    "    dataset_num_proc = 2,                 # number of parallel proceses for data preprocessing\n",
    "    packing = True,                       # Combine short sequences\n",
    "    args = TrainingArguments(\n",
    "         # Training hyperparameters\n",
    "        num_train_epochs=3,               # Train for one epoch\n",
    "        per_device_train_batch_size=2,    # Batch size per device during training\n",
    "        per_device_eval_batch_size=2,     # Batch size per device during evaluation\n",
    "        gradient_accumulation_steps=8,    # Accumulate gradients for larger effective batch size\n",
    "        gradient_checkpointing=True,      # Save memory by recomputing activations in backprop\n",
    "        \n",
    "        # Optimization settings\n",
    "        learning_rate = 3e-4,\n",
    "        optim = \"adamw_8bit\",\n",
    "        weight_decay = 0.01, \n",
    "        lr_scheduler_type = \"linear\",      \n",
    "        warmup_steps=10,\n",
    "        \n",
    "        # Precision settings\n",
    "        fp16 = not is_bfloat16_supported(),     # Disable FP16 precision (set True if supported)\n",
    "        bf16 = is_bfloat16_supported(),         # Disable BF16 precision (use True on A100 GPUs)\n",
    "        \n",
    "       # Logging and checkpoints\n",
    "        save_steps=100,                   # Save checkpoint every 100 steps\n",
    "        save_total_limit=1,               # Keep only the most recent checkpoint\n",
    "        logging_steps=20,                 # Log training progress every 10 steps\n",
    "        eval_strategy=\"steps\",            # Run evaluation at regular intervals, dont wait epochs\n",
    "        eval_steps=10,                    # Evaluate in every such steps\n",
    "        output_dir= \"output\",\n",
    "        run_name=\"llama_fine_tune\",\n",
    "        report_to=\"wandb\",                # Report metrics to Weights and Biases\n",
    "        ),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "1883f9dd-3548-41d4-b416-5afb3752eacf",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "==((====))==  Unsloth - 2x faster free finetuning | Num GPUs = 1\n",
      "   \\\\   /|    Num examples = 1,185 | Num Epochs = 3\n",
      "O^O/ \\_/ \\    Batch size per device = 2 | Gradient Accumulation steps = 8\n",
      "\\        /    Total batch size = 16 | Total steps = 222\n",
      " \"-____-\"     Number of trainable parameters = 22,544,384\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='222' max='222' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [222/222 10:36, Epoch 2/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.295812</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>1.312000</td>\n",
       "      <td>1.199491</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30</td>\n",
       "      <td>1.312000</td>\n",
       "      <td>1.137736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40</td>\n",
       "      <td>1.130500</td>\n",
       "      <td>1.100059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>1.130500</td>\n",
       "      <td>1.069004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>60</td>\n",
       "      <td>1.071000</td>\n",
       "      <td>1.041719</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>70</td>\n",
       "      <td>1.071000</td>\n",
       "      <td>1.017723</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>80</td>\n",
       "      <td>1.066500</td>\n",
       "      <td>1.004602</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>90</td>\n",
       "      <td>1.066500</td>\n",
       "      <td>0.988280</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>0.923900</td>\n",
       "      <td>0.967951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>110</td>\n",
       "      <td>0.923900</td>\n",
       "      <td>0.949770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>120</td>\n",
       "      <td>0.920000</td>\n",
       "      <td>0.931849</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>130</td>\n",
       "      <td>0.920000</td>\n",
       "      <td>0.914590</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>140</td>\n",
       "      <td>0.872800</td>\n",
       "      <td>0.900389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>150</td>\n",
       "      <td>0.872800</td>\n",
       "      <td>0.894003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>160</td>\n",
       "      <td>0.865000</td>\n",
       "      <td>0.891151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>170</td>\n",
       "      <td>0.865000</td>\n",
       "      <td>0.883936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>180</td>\n",
       "      <td>0.773300</td>\n",
       "      <td>0.877599</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>190</td>\n",
       "      <td>0.773300</td>\n",
       "      <td>0.871471</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>200</td>\n",
       "      <td>0.787000</td>\n",
       "      <td>0.868901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>210</td>\n",
       "      <td>0.787000</td>\n",
       "      <td>0.866788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>220</td>\n",
       "      <td>0.769300</td>\n",
       "      <td>0.865117</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=222, training_loss=0.9518617015701156, metrics={'train_runtime': 639.9751, 'train_samples_per_second': 5.555, 'train_steps_per_second': 0.347, 'total_flos': 4.34344090927104e+16, 'train_loss': 0.9518617015701156, 'epoch': 2.9949409780775715})"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "103b30f8-68be-4b6c-a977-fee69f0b6e0b",
   "metadata": {},
   "source": [
    "## Inference and Testing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "032ed01a-446a-442b-9939-ec03d9f8710a",
   "metadata": {},
   "source": [
    "As we see from the responses below(compare with the baseline model above), the model learned great deal about Honaz. It is not making things up anymore. Also carefully inspect the answers from both model in the curated QA samples I created below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "d9a80612-0526-46aa-9215-a9498b198957",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<|begin_of_text|>Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
      "\n",
      "### Instruction:\n",
      "How did the population of Honaz evolve during the late 19th century?\n",
      "\n",
      "### Response:\n",
      "The population of Honaz, as with many other areas in the Ottoman Empire, was influenced by a variety of factors, including economic opportunities, social and cultural ties, and the impact of external events such as the Greek population exchange after World War I. \n",
      "\n",
      "During the late 19th century, Honaz, like many other regions in the Ottoman Empire, experienced significant economic growth. The Ottoman Empire was characterized by a centralized economy, with extensive infrastructure and trade networks. This led to increased agricultural production,\n"
     ]
    }
   ],
   "source": [
    "model = FastLanguageModel.for_inference(model)\n",
    "prompt = \"How did the population of Honaz evolve during the late 19th century?\"\n",
    "generate_streaming_text(ref_model, tokenizer, prompt, device, max_new_tokens=100)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "62b1877f-0d3d-4a04-a3fc-086e1f13b8a0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<|begin_of_text|>Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
      "\n",
      "### Instruction:\n",
      "What are the climatic influences on Honaz Mountain’s vegetation?\n",
      "\n",
      "### Response:\n",
      "Honaz Mountain, located in the Aegean region of Turkey, experiences a Mediterranean climate characterized by mild winters and hot, dry summers. The climatic influences on its vegetation are primarily driven by temperature, precipitation, and vegetation patterns.\n",
      "\n",
      "1. Temperature: The average temperature ranges from 12°C in winter to 26°C in summer, with spring and autumn experiencing mild variations. The temperature influences the distribution of plant species, with the majority being Mediterranean shrubs and trees.\n",
      "\n",
      "2. Precipitation\n"
     ]
    }
   ],
   "source": [
    "prompt = \"What are the climatic influences on Honaz Mountain’s vegetation?\"\n",
    "generate_streaming_text(ref_model, tokenizer, prompt, device, max_new_tokens=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ebce5b7-ef7a-43ca-a27d-0f403fad8111",
   "metadata": {},
   "source": [
    "**Note**: `Since I spent my 18 years there, I know the temperature values above are pretty accurate(:`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a0c700e-e873-42ee-9765-2ff408d0d910",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "87addcd0-8987-437e-b036-753749865ee3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compare_models(csv_path, model, ref_model, tokenizer, max_new_tokens=256, prompt_template=alpaca_prompt):\n",
    "    \"\"\"\n",
    "    compare both models on QA data\n",
    "    \"\"\"\n",
    "\n",
    "    # get QA dataset\n",
    "    df = pd.read_csv(csv_path)\n",
    "    model_answers = []\n",
    "    ref_model_answers = []\n",
    "\n",
    "    # Process each question \n",
    "    for index, row in tqdm(df.iterrows(), total=len(df), desc=\"Processing Questions\"):\n",
    "        #  Generate the response from the trained model\n",
    "        question = row['question']\n",
    "        message = prompt_template.format(question, \"\")\n",
    "        inputs = tokenizer([message], return_tensors=\"pt\").to(device)\n",
    "        outputs = model.generate(\n",
    "            **inputs,\n",
    "            max_new_tokens=max_new_tokens,\n",
    "            use_cache=True\n",
    "        )\n",
    "        trained_response = tokenizer.decode(outputs[0], skip_special_tokens=True)\n",
    "        trained_response = trained_response.split(\"### Response:\")[1].strip() if \"### Response:\" in trained_response else trained_response\n",
    "        model_answers.append(trained_response)\n",
    "\n",
    "        # Generate the response from the reference model\n",
    "        ref_outputs = ref_model.generate(\n",
    "            **inputs,\n",
    "            max_new_tokens=max_new_tokens,\n",
    "            use_cache=True\n",
    "        )\n",
    "        ref_response = tokenizer.decode(ref_outputs[0], skip_special_tokens=True)\n",
    "        ref_response = ref_response.split(\"### Response:\")[1].strip() if \"### Response:\" in ref_response else ref_response\n",
    "        ref_model_answers.append(ref_response)\n",
    "\n",
    "    # Append the responses\n",
    "    df['finetuned_model'] = model_answers\n",
    "    df['original_model'] = ref_model_answers\n",
    "\n",
    "    return df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "242eab39-99df-460c-93c3-c45dbe15faca",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==((====))==  Unsloth 2024.12.4: Fast Llama patching. Transformers:4.46.3.\n",
      "   \\\\   /|    GPU: NVIDIA A100-SXM4-40GB. Max memory: 39.381 GB. Platform: Linux.\n",
      "O^O/ \\_/ \\    Torch: 2.5.1+cu124. CUDA: 8.0. CUDA Toolkit: 12.4. Triton: 3.1.0\n",
      "\\        /    Bfloat16 = TRUE. FA [Xformers = 0.0.28.post3. FA2 = False]\n",
      " \"-____-\"     Free Apache license: http://github.com/unslothai/unsloth\n",
      "Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!\n"
     ]
    }
   ],
   "source": [
    "#load the base model again to be sure we start fresh--> I cannot confirm exacly but Unsloth may be modifying the base model in some way\n",
    "ref_model, tokenizer = FastLanguageModel.from_pretrained(\n",
    "    model_name=\"unsloth/Llama-3.2-1B-Instruct\",\n",
    "    max_seq_length=max_seq_length,\n",
    "    dtype=None,\n",
    "    load_in_4bit=True\n",
    ")\n",
    "ref_model = FastLanguageModel.for_inference(ref_model)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "b627515c-d51c-4004-bd2c-86ee34765d79",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing Questions: 100%|██████████| 9/9 [01:16<00:00,  8.49s/it]\n"
     ]
    }
   ],
   "source": [
    "csv_path = \"honaz_questions_answers.csv\"\n",
    "df = compare_models(\n",
    "    csv_path,\n",
    "    model=model,  \n",
    "    ref_model=ref_model,   \n",
    "    tokenizer=tokenizer,\n",
    "    max_new_tokens=200\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "5d6ebaac-003b-4f68-879b-19c160a0f7c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def inspect_answers(df,idx=1):\n",
    "    \n",
    "    # inspect the first question-answer pair\n",
    "    print(f\"Instruction:\\n\\n{df.iloc[idx]['question']}\")\n",
    "    \n",
    "    print(\"=========================================\")\n",
    "    print(f\"Reference Answer:\\n\\n{df.iloc[idx]['answer']}\")\n",
    "    \n",
    "    print(\"=========================================\")\n",
    "    print(f\"Fine Tuned Model:\\n\\n{df.iloc[idx]['finetuned_model']}\")\n",
    "\n",
    "    print(\"=========================================\")\n",
    "    print(f\"Original Model:\\n\\n{df.iloc[idx]['original_model']}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "3acba17a-5e69-432f-be16-da8f8f8e5772",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Instruction:\n",
      "\n",
      "What roles did Honaz historically play in the economic, military, and cultural life of the region? Please provide specific examples.\n",
      "=========================================\n",
      "Reference Answer:\n",
      "\n",
      "Honaz served as a vital center for trade and military defense due to its strategic location in the Menderes basin. During the Byzantine period, it was a military stronghold overseeing critical routes. Culturally, it was notable for religious significance, with prominent structures like the Church of Archangel Michael.\n",
      "=========================================\n",
      "Fine Tuned Model:\n",
      "\n",
      "Honaz has been a significant economic, military, and cultural center in the region for centuries. Its strategic location on the Aegean Sea, coupled with its proximity to the Menderes Valley and the Çürüksu River, made it an important hub for trade and commerce.\n",
      "\n",
      "Historically, Honaz served as a key military location, particularly during the Ottoman era. The city was strategically situated to control the passage of the Çürüksu River to the Aegean Sea, making it an essential location for the Ottoman Empire's military operations.\n",
      "\n",
      "In terms of economy, Honaz was a major center for the production and trade of agricultural goods, such as wheat, barley, and olive oil. The city's proximity to the fertile valleys of Western Anatolia and its access to the Aegean Sea facilitated the growth of a thriving agricultural economy.\n",
      "\n",
      "Culturally, Honaz was a significant center for the arts and architecture of the region. The city was home to the famous H\n",
      "=========================================\n",
      "Original Model:\n",
      "\n",
      "Historically, Honaz played significant roles in the economic, military, and cultural life of the region. Here are some examples:\n",
      "\n",
      "- **Economic Role:** Honaz was an important trading center in the region, with a significant economy based on the export of cotton, silk, and other goods. The region was also known for its textile industry, with many skilled weavers and textile manufacturers operating in the area.\n",
      "\n",
      "- **Military Role:** Historically, Honaz was an important military outpost. The region was strategically located, and the local rulers and the state of Sindh used it as a base for military operations in the surrounding areas. The fort of Honaz was an important military stronghold, which played a significant role in the defense of the region.\n",
      "\n",
      "- **Cultural Role:** Honaz was a significant cultural center in the region. The region was known for its rich cultural heritage, with a blend of Persian, Turkish, and Indian influences. The local rulers and artisans used traditional crafts like pottery\n"
     ]
    }
   ],
   "source": [
    "inspect_answers(df=df,idx=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "0f3ac345-063c-4313-a851-f834033e3867",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Instruction:\n",
      "\n",
      "How did the early 20th-century population exchange, involving specific agreements between communities, affect Honaz’s social and economic structure? Be specific about its consequences.\n",
      "=========================================\n",
      "Reference Answer:\n",
      "\n",
      "The population exchange (1924–1930) resettled 70 households in Honaz, replacing the departing Greek population. These new settlers introduced agricultural techniques, gradually rebuilt homes, and contributed to the homogenization of the town’s community structure.\n",
      "=========================================\n",
      "Fine Tuned Model:\n",
      "\n",
      "The early 20th-century population exchange, which involved specific agreements between communities in Turkey and Greece, had significant effects on Honaz's social and economic structure. The exchange was part of a broader effort to reduce ethnic and religious tensions in the region and to achieve demographic balance.\n",
      "\n",
      "One of the primary consequences of the population exchange was the shift in the demographic makeup of Honaz. Historically, Honaz was predominantly Muslim, while the Greek population was larger. However, following the population exchange, the Greek population in Honaz decreased significantly, as many of its Greek residents were forced to leave the area. This demographic change had a profound impact on the social and economic fabric of Honaz.\n",
      "\n",
      "The decrease in the Greek population in Honaz led to a shift in the cultural and social dynamics of the community. The remaining Greek residents in Honaz were likely to be older and of a different generation, having been born and raised in the area during the population exchange. This demographic shift resulted in a loss of cultural\n",
      "=========================================\n",
      "Original Model:\n",
      "\n",
      "The early 20th-century population exchange between Honaz and another community, which was agreed upon through specific agreements, had a profound impact on the social and economic structure of Honaz. These agreements led to a significant influx of people from Honaz to the other community, resulting in a shift in the demographic composition of Honaz.\n",
      "\n",
      "The specific agreements involved the exchange of laborers, artisans, and skilled workers, which led to a shortage of certain skills and expertise in Honaz. As a result, Honaz's economy became increasingly dependent on the skills and expertise provided by the other community. This shift led to a decrease in the economic autonomy of Honaz, as it became more reliant on external labor and expertise.\n",
      "\n",
      "The social structure of Honaz was also significantly affected by the population exchange. The influx of people from the other community led to a significant increase in the population of Honaz, resulting in a more crowded and densely populated urban area. This led to increased competition for resources, housing, and\n"
     ]
    }
   ],
   "source": [
    "inspect_answers(df=df,idx=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "68aa3942-3486-4398-aa70-f58066be3e7d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Instruction:\n",
      "\n",
      "How do the current ecological features of Honaz Mountain reflect historical climatic periods? Provide specific examples tied to changes over time.\n",
      "=========================================\n",
      "Reference Answer:\n",
      "\n",
      "The presence of Black Sea floristic elements in the northern valleys of Honaz Mountain indicates remnants of humid vegetation from the Pleistocene glacial period, reflecting adaptations to historical climatic conditions.\n",
      "=========================================\n",
      "Fine Tuned Model:\n",
      "\n",
      "Honaz Mountain, located in the Aegean Region of Turkey, has undergone significant ecological changes over the centuries due to shifts in climate. These changes are evident in the vegetation, soil composition, and other ecological features of the area.\n",
      "\n",
      "One of the earliest changes is the increase in Mediterranean species. These species are adapted to the warmer and drier conditions of the Mediterranean climate. For example, the iconic Mediterranean olive tree (Olivia muscarellifolia) is now widespread in the region. This shift is linked to the warming of the climate, which has allowed for the growth of these species.\n",
      "\n",
      "Another example is the rise in Black Sea species. The Black Sea is a large inland body of water that originated from a supercontinent called Laurasia. As the climate warmed, the Black Sea's water level rose, and it became a significant habitat for many Black Sea species. For instance, the Black Sea bird (Acanthiza melanocephala) is now found in the region, which\n",
      "=========================================\n",
      "Original Model:\n",
      "\n",
      "The current ecological features of Honaz Mountain reflect historical climatic periods through the presence of specific plant species and their adaptations to the changing environmental conditions. Specifically, the mountain's unique flora is influenced by the mountain's position in the Himalayan range, where the climate has shifted significantly over time.\n",
      "\n",
      "#### Changes Over Time:\n",
      "- **Early Holocene (10,000 - 5,000 years ago):** The mountain's vegetation has shifted towards more temperate species. This includes the presence of evergreen conifers and deciduous broad-leaved trees. The mountain's forest cover has been expanding in recent centuries, with the introduction of species like the Himalayan red pine (Pinus roxburghii) and the Himalayan balsam fir (Pseudolaris chinensis), which are adapted to the warmer temperatures and more abundant rainfall of the Holocene period.\n",
      "  \n",
      "- **Last Glacial Maximum (20,000 - 10,000 years ago):** The mountain's ecosystem has\n"
     ]
    }
   ],
   "source": [
    "inspect_answers(df=df,idx=7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f529bc31-f2d7-41de-94ce-29649584ff47",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "84750bf8-a00f-4289-8fec-49d3edd3c30f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unsloth: Merging 4bit and LoRA weights to 16bit...\n",
      "Unsloth: Will use up to 154.55 out of 216.26 RAM for saving.\n",
      "Unsloth: Saving model... This might take 5 minutes ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:00<00:00, 115.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unsloth: Saving tokenizer..."
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Done.\n",
      "Done.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1edf2ebb11fc4e38be4867af1e29d739",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "README.md:   0%|          | 0.00/595 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9f4c4703fc27409498af05257c7bdd15",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "adapter_model.safetensors:   0%|          | 0.00/90.2M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved model to https://huggingface.co/erdi28/finetune_llama_honaz\n"
     ]
    }
   ],
   "source": [
    "# Save the whole model and push to HuggingFace for further usage\n",
    "model.save_pretrained_merged(\"finetune_llama_honaz\", tokenizer,save_method=\"merged_16bit\")\n",
    "model.push_to_hub(\"erdi28/finetune_llama_honaz\", tokenizer,save_method=\"merged_16bit\") "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55ddd6af-36a0-4775-986e-b300edb974c5",
   "metadata": {},
   "source": [
    "Now lets calculate the infamous metric BLEU(Bilingual Evaluation Understudy). It measures how well n-grams in the prediction match the reference. I am not a big fan of this metric or any other metric in LLMs since I prefer simple human evaluation, we can nevertheless see that the fine-tuned model does a way better job on testing data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "1962b617-edf1-4163-8fa2-1d0c8cd619eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_bleu_scores(model, tokenizer, test_data, alpaca_prompt, num_test_points=10, max_new_tokens=128):\n",
    "    \"\"\"\n",
    "    Compute BLEU scores for a specified number of test data points using Unsloth's inference.\n",
    "    \"\"\"\n",
    "    # Enable optimized inference for the model\n",
    "    FastLanguageModel.for_inference(model)\n",
    "    \n",
    "    # Load BLEU metric\n",
    "    bleu = load(\"bleu\")\n",
    "    bleu_scores = []\n",
    "\n",
    "    # Randomly sample num_test_points from the test_data-\n",
    "    total_data_points = len(test_data)\n",
    "    sampled_indices = random.sample(range(total_data_points), min(num_test_points, total_data_points))\n",
    "    sampled_test_data = test_data.select(sampled_indices)\n",
    "\n",
    "    # Iterate over test data with progress bar\n",
    "    for i in tqdm(range(len(sampled_test_data)), desc=\"Evaluating BLEU\"):\n",
    "        # Get the test example\n",
    "        example = test_data[i]\n",
    "        instruction = example[\"instruction\"]\n",
    "        reference_output = example[\"output\"]\n",
    "\n",
    "        # Prepare the input prompt\n",
    "        input_text = alpaca_prompt.format(instruction, \"\")\n",
    "        inputs = tokenizer([input_text], return_tensors=\"pt\").to(\"cuda\")\n",
    "\n",
    "        # Generate predictions using Unsloth's fast inference\n",
    "        with torch.no_grad():\n",
    "            output_ids = model.generate(**inputs, max_new_tokens=max_new_tokens, use_cache=True)\n",
    "        generated_output = tokenizer.decode(output_ids[0], skip_special_tokens=True)\n",
    "        \n",
    "        #organize the results\n",
    "        predictions = [generated_output]\n",
    "        references = [[reference_output]]\n",
    "\n",
    "        # Compute BLEU score for the current example\n",
    "        bleu_score = round(bleu.compute(predictions=predictions, references=references)[\"bleu\"], 3)\n",
    "        bleu_scores.append(bleu_score)\n",
    "\n",
    "    # save\n",
    "    average_bleu = sum(bleu_scores) / len(bleu_scores) if bleu_scores else 0\n",
    "\n",
    "    return {\n",
    "        \"bleu_scores\": bleu_scores,\n",
    "        \"average_bleu\": average_bleu\n",
    "    }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "60fdd6e0-25d0-4715-997e-b6a98c5239e4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating BLEU: 100%|██████████| 10/10 [00:31<00:00,  3.19s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average BLEU Score: 0.1694\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Fine tuned model\n",
    "num_test_points = 10\n",
    "\n",
    "# Compute BLEU scores\n",
    "results = compute_bleu_scores(\n",
    "    model=model,\n",
    "    tokenizer=tokenizer,\n",
    "    test_data=dataset[\"test\"],\n",
    "    alpaca_prompt=alpaca_prompt,\n",
    "    num_test_points=num_test_points,\n",
    "    max_new_tokens=128\n",
    ")\n",
    "\n",
    "# Print results\n",
    "#print(\"Individual BLEU Scores:\", results[\"bleu_scores\"])\n",
    "print(\"Average BLEU Score:\", results[\"average_bleu\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "fbea905c-f61b-4aa4-be14-883e79372542",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating BLEU: 100%|██████████| 10/10 [00:23<00:00,  2.33s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average BLEU Score: 0.1167\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Base model\n",
    "num_test_points = 10\n",
    "\n",
    "# Compute BLEU scores\n",
    "results = compute_bleu_scores(\n",
    "    model=ref_model,\n",
    "    tokenizer=tokenizer,\n",
    "    test_data=dataset[\"test\"],\n",
    "    alpaca_prompt=alpaca_prompt,\n",
    "    num_test_points=num_test_points,\n",
    "    max_new_tokens=128\n",
    ")\n",
    "\n",
    "# Print results\n",
    "#print(\"Individual BLEU Scores:\", results[\"bleu_scores\"])\n",
    "print(\"Average BLEU Score:\", results[\"average_bleu\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e88122b4-6678-419b-9141-7fbe7c00f7be",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "311b8ee3-7a32-443b-81fd-a442d6fbd333",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4f85b8f-1068-4ef0-93b5-377a7f015ecf",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}