independent_models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "120740cc-1ba9-44ae-a673-1047f23de335",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import transformers\n",
    "transformers.logging.set_verbosity_error()\n",
    "from transformers import Trainer, TrainingArguments\n",
    "from transformers import RobertaTokenizer, RobertaForSequenceClassification\n",
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification, DataCollatorWithPadding\n",
    "from datasets import load_dataset, DatasetDict, Dataset, load_metric\n",
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "17b7a002",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = RobertaTokenizer.from_pretrained(\"roberta-base\")\n",
    "\n",
    "def tokenize_function(examples):\n",
    "    return tokenizer(examples[\"text\"], truncation=True)\n",
    "\n",
    "def read_and_tokenize_data(train,test,evaluate):\n",
    "    d = load_dataset('pandas', data_files={\"train\":train, \"test\":test,\"evaluate\":evaluate})\n",
    "    for i in d:\n",
    "        d[i]= d[i].remove_columns(column_names = ['__index_level_0__'])\n",
    "    d = d.map(tokenize_function, batched=True)\n",
    "    return d\n",
    "\n",
    "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a796187-c8ee-4a74-ace5-199fcaa15b84",
   "metadata": {},
   "source": [
    "#### Reading data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "154ee52e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using custom data configuration default-5d8442b82fe9acdc\n",
      "Reusing dataset pandas (C:\\Users\\dovyd\\.cache\\huggingface\\datasets\\pandas\\default-5d8442b82fe9acdc\\0.0.0\\6197c1e855b639d75a767140856841a562b7a71d129104973fe1962594877ade)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4d721db8038743d2b2ff833cd05dd638",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached processed dataset at C:\\Users\\dovyd\\.cache\\huggingface\\datasets\\pandas\\default-5d8442b82fe9acdc\\0.0.0\\6197c1e855b639d75a767140856841a562b7a71d129104973fe1962594877ade\\cache-c986a07d4e839639.arrow\n",
      "Loading cached processed dataset at C:\\Users\\dovyd\\.cache\\huggingface\\datasets\\pandas\\default-5d8442b82fe9acdc\\0.0.0\\6197c1e855b639d75a767140856841a562b7a71d129104973fe1962594877ade\\cache-1aef789b7c6c8461.arrow\n",
      "Loading cached processed dataset at C:\\Users\\dovyd\\.cache\\huggingface\\datasets\\pandas\\default-5d8442b82fe9acdc\\0.0.0\\6197c1e855b639d75a767140856841a562b7a71d129104973fe1962594877ade\\cache-4ecbd960b02fdce5.arrow\n",
      "Using custom data configuration default-d0f7ab1575e159e4\n",
      "Reusing dataset pandas (C:\\Users\\dovyd\\.cache\\huggingface\\datasets\\pandas\\default-d0f7ab1575e159e4\\0.0.0\\6197c1e855b639d75a767140856841a562b7a71d129104973fe1962594877ade)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "64de16beca074cb9990acf6155b8f81f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dee28ef2053149788269f10c74247369",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bb554c5005ff44308fe99ca0120df476",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d5c851bf2443423d821acae7e91419e5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data_dict = read_and_tokenize_data(\"data/authors_train.pkl\",\\\n",
    "                               \"data/authors_test.pkl\",\\\n",
    "                                   \"data/authors_validation.pkl\")\n",
    "\n",
    "data_dict_2 = read_and_tokenize_data(\"data/imdb_train.pkl\",\\\n",
    "                               \"data/imdb_test.pkl\",\\\n",
    "                                   \"data/imdb_validation.pkl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9932f74d-ce1f-4020-b142-f9ed7e820763",
   "metadata": {},
   "source": [
    "#### Training model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e50a545e-10c3-4b82-9a1e-edec0242abf7",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = RobertaForSequenceClassification.from_pretrained(\"roberta-base\", num_labels = 4)\n",
    "\n",
    "model_2 = RobertaForSequenceClassification.from_pretrained(\"roberta-base\", num_labels = 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d31fa26e-b3e2-482f-a66e-9f870ca2620f",
   "metadata": {},
   "outputs": [],
   "source": [
    "training_args = TrainingArguments(\n",
    "    output_dir='results',\n",
    "    learning_rate=2e-5,\n",
    "    num_train_epochs=20,\n",
    "    weight_decay=0.01,\n",
    "    evaluation_strategy=\"steps\",\n",
    "    eval_steps = 250,\n",
    "    save_steps = 250,\n",
    "    load_best_model_at_end = True\n",
    ")\n",
    "\n",
    "\n",
    "metric = load_metric(\"accuracy\")\n",
    "\n",
    "def compute_metrics(eval_pred):\n",
    "    logits, labels = eval_pred\n",
    "    predictions = np.argmax(logits, axis=-1)\n",
    "    return metric.compute(predictions=predictions, references=labels)\n",
    "\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args = training_args,\n",
    "    train_dataset=data_dict[\"train\"],\n",
    "    eval_dataset=data_dict[\"evaluate\"],\n",
    "    tokenizer=tokenizer,\n",
    "    data_collator=data_collator,\n",
    "    compute_metrics=compute_metrics\n",
    ")\n",
    "\n",
    "\n",
    "trainer_2 = Trainer(\n",
    "    model=model_2,\n",
    "    args = training_args,\n",
    "    train_dataset=data_dict_2[\"train\"],\n",
    "    eval_dataset=data_dict_2[\"evaluate\"],\n",
    "    tokenizer=tokenizer,\n",
    "    data_collator=data_collator,\n",
    "    compute_metrics=compute_metrics\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c361e8a-bb8b-4a45-a270-4bd8eede77dd",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e151a0be",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the training set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: text.\n",
      "***** Running training *****\n",
      "  Num examples = 2700\n",
      "  Num Epochs = 20\n",
      "  Instantaneous batch size per device = 8\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 8\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 6760\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: text.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 150\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to results/checkpoint-250\n",
      "Configuration saved in results/checkpoint-250/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.42121168971061707, 'eval_accuracy': 0.88, 'eval_runtime': 1.4986, 'eval_samples_per_second': 100.092, 'eval_steps_per_second': 12.678, 'epoch': 0.74}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in results/checkpoint-250/pytorch_model.bin\n",
      "tokenizer config file saved in results/checkpoint-250/tokenizer_config.json\n",
      "Special tokens file saved in results/checkpoint-250/special_tokens_map.json\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: text.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 150\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3439, 'learning_rate': 1.85207100591716e-05, 'epoch': 1.48}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to results/checkpoint-500\n",
      "Configuration saved in results/checkpoint-500/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.47201740741729736, 'eval_accuracy': 0.9, 'eval_runtime': 1.4311, 'eval_samples_per_second': 104.816, 'eval_steps_per_second': 13.277, 'epoch': 1.48}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in results/checkpoint-500/pytorch_model.bin\n",
      "tokenizer config file saved in results/checkpoint-500/tokenizer_config.json\n",
      "Special tokens file saved in results/checkpoint-500/special_tokens_map.json\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: text.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 150\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to results/checkpoint-750\n",
      "Configuration saved in results/checkpoint-750/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.445279598236084, 'eval_accuracy': 0.9266666666666666, 'eval_runtime': 1.4328, 'eval_samples_per_second': 104.693, 'eval_steps_per_second': 13.261, 'epoch': 2.22}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in results/checkpoint-750/pytorch_model.bin\n",
      "tokenizer config file saved in results/checkpoint-750/tokenizer_config.json\n",
      "Special tokens file saved in results/checkpoint-750/special_tokens_map.json\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: text.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 150\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.1618, 'learning_rate': 1.70414201183432e-05, 'epoch': 2.96}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to results/checkpoint-1000\n",
      "Configuration saved in results/checkpoint-1000/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.44325175881385803, 'eval_accuracy': 0.9133333333333333, 'eval_runtime': 1.4341, 'eval_samples_per_second': 104.598, 'eval_steps_per_second': 13.249, 'epoch': 2.96}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in results/checkpoint-1000/pytorch_model.bin\n",
      "tokenizer config file saved in results/checkpoint-1000/tokenizer_config.json\n",
      "Special tokens file saved in results/checkpoint-1000/special_tokens_map.json\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: text.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 150\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to results/checkpoint-1250\n",
      "Configuration saved in results/checkpoint-1250/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.4769594967365265, 'eval_accuracy': 0.92, 'eval_runtime': 1.4325, 'eval_samples_per_second': 104.711, 'eval_steps_per_second': 13.263, 'epoch': 3.7}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in results/checkpoint-1250/pytorch_model.bin\n",
      "tokenizer config file saved in results/checkpoint-1250/tokenizer_config.json\n",
      "Special tokens file saved in results/checkpoint-1250/special_tokens_map.json\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: text.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 150\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.06, 'learning_rate': 1.5562130177514792e-05, 'epoch': 4.44}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to results/checkpoint-1500\n",
      "Configuration saved in results/checkpoint-1500/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.394641250371933, 'eval_accuracy': 0.9266666666666666, 'eval_runtime': 1.4295, 'eval_samples_per_second': 104.932, 'eval_steps_per_second': 13.291, 'epoch': 4.44}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in results/checkpoint-1500/pytorch_model.bin\n",
      "tokenizer config file saved in results/checkpoint-1500/tokenizer_config.json\n",
      "Special tokens file saved in results/checkpoint-1500/special_tokens_map.json\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: text.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 150\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to results/checkpoint-1750\n",
      "Configuration saved in results/checkpoint-1750/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.7077649831771851, 'eval_accuracy': 0.9066666666666666, 'eval_runtime': 1.4294, 'eval_samples_per_second': 104.938, 'eval_steps_per_second': 13.292, 'epoch': 5.18}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in results/checkpoint-1750/pytorch_model.bin\n",
      "tokenizer config file saved in results/checkpoint-1750/tokenizer_config.json\n",
      "Special tokens file saved in results/checkpoint-1750/special_tokens_map.json\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: text.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 150\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0274, 'learning_rate': 1.4082840236686392e-05, 'epoch': 5.92}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to results/checkpoint-2000\n",
      "Configuration saved in results/checkpoint-2000/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.6947169899940491, 'eval_accuracy': 0.9066666666666666, 'eval_runtime': 1.4331, 'eval_samples_per_second': 104.665, 'eval_steps_per_second': 13.258, 'epoch': 5.92}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in results/checkpoint-2000/pytorch_model.bin\n",
      "tokenizer config file saved in results/checkpoint-2000/tokenizer_config.json\n",
      "Special tokens file saved in results/checkpoint-2000/special_tokens_map.json\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: text.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 150\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to results/checkpoint-2250\n",
      "Configuration saved in results/checkpoint-2250/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.8670269846916199, 'eval_accuracy': 0.8866666666666667, 'eval_runtime': 1.4325, 'eval_samples_per_second': 104.715, 'eval_steps_per_second': 13.264, 'epoch': 6.66}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in results/checkpoint-2250/pytorch_model.bin\n",
      "tokenizer config file saved in results/checkpoint-2250/tokenizer_config.json\n",
      "Special tokens file saved in results/checkpoint-2250/special_tokens_map.json\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: text.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 150\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0369, 'learning_rate': 1.2603550295857989e-05, 'epoch': 7.4}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to results/checkpoint-2500\n",
      "Configuration saved in results/checkpoint-2500/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.7064142227172852, 'eval_accuracy': 0.9, 'eval_runtime': 1.4329, 'eval_samples_per_second': 104.684, 'eval_steps_per_second': 13.26, 'epoch': 7.4}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in results/checkpoint-2500/pytorch_model.bin\n",
      "tokenizer config file saved in results/checkpoint-2500/tokenizer_config.json\n",
      "Special tokens file saved in results/checkpoint-2500/special_tokens_map.json\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: text.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 150\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to results/checkpoint-2750\n",
      "Configuration saved in results/checkpoint-2750/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.7833418846130371, 'eval_accuracy': 0.92, 'eval_runtime': 1.4336, 'eval_samples_per_second': 104.635, 'eval_steps_per_second': 13.254, 'epoch': 8.14}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in results/checkpoint-2750/pytorch_model.bin\n",
      "tokenizer config file saved in results/checkpoint-2750/tokenizer_config.json\n",
      "Special tokens file saved in results/checkpoint-2750/special_tokens_map.json\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: text.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 150\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0096, 'learning_rate': 1.1124260355029586e-05, 'epoch': 8.88}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to results/checkpoint-3000\n",
      "Configuration saved in results/checkpoint-3000/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.7665197849273682, 'eval_accuracy': 0.9133333333333333, 'eval_runtime': 1.4314, 'eval_samples_per_second': 104.791, 'eval_steps_per_second': 13.274, 'epoch': 8.88}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in results/checkpoint-3000/pytorch_model.bin\n",
      "tokenizer config file saved in results/checkpoint-3000/tokenizer_config.json\n",
      "Special tokens file saved in results/checkpoint-3000/special_tokens_map.json\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: text.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 150\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to results/checkpoint-3250\n",
      "Configuration saved in results/checkpoint-3250/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.7281720042228699, 'eval_accuracy': 0.9266666666666666, 'eval_runtime': 1.4519, 'eval_samples_per_second': 103.314, 'eval_steps_per_second': 13.086, 'epoch': 9.62}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in results/checkpoint-3250/pytorch_model.bin\n",
      "tokenizer config file saved in results/checkpoint-3250/tokenizer_config.json\n",
      "Special tokens file saved in results/checkpoint-3250/special_tokens_map.json\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: text.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 150\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0172, 'learning_rate': 9.644970414201184e-06, 'epoch': 10.36}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to results/checkpoint-3500\n",
      "Configuration saved in results/checkpoint-3500/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.8325244188308716, 'eval_accuracy': 0.9066666666666666, 'eval_runtime': 1.4317, 'eval_samples_per_second': 104.77, 'eval_steps_per_second': 13.271, 'epoch': 10.36}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in results/checkpoint-3500/pytorch_model.bin\n",
      "tokenizer config file saved in results/checkpoint-3500/tokenizer_config.json\n",
      "Special tokens file saved in results/checkpoint-3500/special_tokens_map.json\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: text.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 150\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to results/checkpoint-3750\n",
      "Configuration saved in results/checkpoint-3750/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.8748603463172913, 'eval_accuracy': 0.8933333333333333, 'eval_runtime': 1.4383, 'eval_samples_per_second': 104.289, 'eval_steps_per_second': 13.21, 'epoch': 11.09}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in results/checkpoint-3750/pytorch_model.bin\n",
      "tokenizer config file saved in results/checkpoint-3750/tokenizer_config.json\n",
      "Special tokens file saved in results/checkpoint-3750/special_tokens_map.json\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: text.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 150\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0125, 'learning_rate': 8.165680473372781e-06, 'epoch': 11.83}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to results/checkpoint-4000\n",
      "Configuration saved in results/checkpoint-4000/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.7383723855018616, 'eval_accuracy': 0.92, 'eval_runtime': 1.4346, 'eval_samples_per_second': 104.56, 'eval_steps_per_second': 13.244, 'epoch': 11.83}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in results/checkpoint-4000/pytorch_model.bin\n",
      "tokenizer config file saved in results/checkpoint-4000/tokenizer_config.json\n",
      "Special tokens file saved in results/checkpoint-4000/special_tokens_map.json\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: text.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 150\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to results/checkpoint-4250\n",
      "Configuration saved in results/checkpoint-4250/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.7885671854019165, 'eval_accuracy': 0.92, 'eval_runtime': 1.4379, 'eval_samples_per_second': 104.318, 'eval_steps_per_second': 13.214, 'epoch': 12.57}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in results/checkpoint-4250/pytorch_model.bin\n",
      "tokenizer config file saved in results/checkpoint-4250/tokenizer_config.json\n",
      "Special tokens file saved in results/checkpoint-4250/special_tokens_map.json\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: text.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 150\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0015, 'learning_rate': 6.686390532544379e-06, 'epoch': 13.31}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to results/checkpoint-4500\n",
      "Configuration saved in results/checkpoint-4500/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.8066335320472717, 'eval_accuracy': 0.92, 'eval_runtime': 1.4326, 'eval_samples_per_second': 104.708, 'eval_steps_per_second': 13.263, 'epoch': 13.31}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in results/checkpoint-4500/pytorch_model.bin\n",
      "tokenizer config file saved in results/checkpoint-4500/tokenizer_config.json\n",
      "Special tokens file saved in results/checkpoint-4500/special_tokens_map.json\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: text.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 150\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to results/checkpoint-4750\n",
      "Configuration saved in results/checkpoint-4750/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.794413685798645, 'eval_accuracy': 0.9266666666666666, 'eval_runtime': 1.4352, 'eval_samples_per_second': 104.517, 'eval_steps_per_second': 13.239, 'epoch': 14.05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in results/checkpoint-4750/pytorch_model.bin\n",
      "tokenizer config file saved in results/checkpoint-4750/tokenizer_config.json\n",
      "Special tokens file saved in results/checkpoint-4750/special_tokens_map.json\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: text.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 150\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0012, 'learning_rate': 5.207100591715976e-06, 'epoch': 14.79}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to results/checkpoint-5000\n",
      "Configuration saved in results/checkpoint-5000/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.8257233500480652, 'eval_accuracy': 0.92, 'eval_runtime': 1.4376, 'eval_samples_per_second': 104.342, 'eval_steps_per_second': 13.217, 'epoch': 14.79}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in results/checkpoint-5000/pytorch_model.bin\n",
      "tokenizer config file saved in results/checkpoint-5000/tokenizer_config.json\n",
      "Special tokens file saved in results/checkpoint-5000/special_tokens_map.json\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: text.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 150\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to results/checkpoint-5250\n",
      "Configuration saved in results/checkpoint-5250/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.9668459892272949, 'eval_accuracy': 0.9066666666666666, 'eval_runtime': 1.4353, 'eval_samples_per_second': 104.506, 'eval_steps_per_second': 13.237, 'epoch': 15.53}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in results/checkpoint-5250/pytorch_model.bin\n",
      "tokenizer config file saved in results/checkpoint-5250/tokenizer_config.json\n",
      "Special tokens file saved in results/checkpoint-5250/special_tokens_map.json\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: text.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 150\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0064, 'learning_rate': 3.7278106508875745e-06, 'epoch': 16.27}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to results/checkpoint-5500\n",
      "Configuration saved in results/checkpoint-5500/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.8655685186386108, 'eval_accuracy': 0.9133333333333333, 'eval_runtime': 1.4376, 'eval_samples_per_second': 104.339, 'eval_steps_per_second': 13.216, 'epoch': 16.27}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in results/checkpoint-5500/pytorch_model.bin\n",
      "tokenizer config file saved in results/checkpoint-5500/tokenizer_config.json\n",
      "Special tokens file saved in results/checkpoint-5500/special_tokens_map.json\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: text.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 150\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to results/checkpoint-5750\n",
      "Configuration saved in results/checkpoint-5750/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.8817095160484314, 'eval_accuracy': 0.9133333333333333, 'eval_runtime': 1.4366, 'eval_samples_per_second': 104.413, 'eval_steps_per_second': 13.226, 'epoch': 17.01}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in results/checkpoint-5750/pytorch_model.bin\n",
      "tokenizer config file saved in results/checkpoint-5750/tokenizer_config.json\n",
      "Special tokens file saved in results/checkpoint-5750/special_tokens_map.json\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: text.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 150\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'learning_rate': 2.2485207100591717e-06, 'epoch': 17.75}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to results/checkpoint-6000\n",
      "Configuration saved in results/checkpoint-6000/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.8837414383888245, 'eval_accuracy': 0.9133333333333333, 'eval_runtime': 1.4363, 'eval_samples_per_second': 104.438, 'eval_steps_per_second': 13.229, 'epoch': 17.75}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in results/checkpoint-6000/pytorch_model.bin\n",
      "tokenizer config file saved in results/checkpoint-6000/tokenizer_config.json\n",
      "Special tokens file saved in results/checkpoint-6000/special_tokens_map.json\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: text.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 150\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to results/checkpoint-6250\n",
      "Configuration saved in results/checkpoint-6250/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.8647990822792053, 'eval_accuracy': 0.92, 'eval_runtime': 1.4368, 'eval_samples_per_second': 104.401, 'eval_steps_per_second': 13.224, 'epoch': 18.49}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in results/checkpoint-6250/pytorch_model.bin\n",
      "tokenizer config file saved in results/checkpoint-6250/tokenizer_config.json\n",
      "Special tokens file saved in results/checkpoint-6250/special_tokens_map.json\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: text.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 150\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0017, 'learning_rate': 7.692307692307694e-07, 'epoch': 19.23}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to results/checkpoint-6500\n",
      "Configuration saved in results/checkpoint-6500/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.8549203276634216, 'eval_accuracy': 0.92, 'eval_runtime': 1.4383, 'eval_samples_per_second': 104.292, 'eval_steps_per_second': 13.21, 'epoch': 19.23}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in results/checkpoint-6500/pytorch_model.bin\n",
      "tokenizer config file saved in results/checkpoint-6500/tokenizer_config.json\n",
      "Special tokens file saved in results/checkpoint-6500/special_tokens_map.json\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: text.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 150\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to results/checkpoint-6750\n",
      "Configuration saved in results/checkpoint-6750/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.8569173216819763, 'eval_accuracy': 0.92, 'eval_runtime': 1.4369, 'eval_samples_per_second': 104.39, 'eval_steps_per_second': 13.223, 'epoch': 19.97}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in results/checkpoint-6750/pytorch_model.bin\n",
      "tokenizer config file saved in results/checkpoint-6750/tokenizer_config.json\n",
      "Special tokens file saved in results/checkpoint-6750/special_tokens_map.json\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from results/checkpoint-1500 (score: 0.394641250371933).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'train_runtime': 1842.6312, 'train_samples_per_second': 29.306, 'train_steps_per_second': 3.669, 'train_loss': 0.05029036337585525, 'epoch': 20.0}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=6760, training_loss=0.05029036337585525, metrics={'train_runtime': 1842.6312, 'train_samples_per_second': 29.306, 'train_steps_per_second': 3.669, 'train_loss': 0.05029036337585525, 'epoch': 20.0})"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer_2.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c1fccf4-232a-4537-89c2-942ac12c74d2",
   "metadata": {},
   "source": [
    "#### Prediction and metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ddfe04f2-d949-4338-8d65-69fc1d6bab64",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def make_predictions(trainer,test):\n",
    "    predictions = trainer.predict(test)\n",
    "    y_true = test[\"label\"]\n",
    "    y_pred = np.argmax(predictions.predictions, axis=-1)\n",
    "    confusion_matrix(y_true, y_pred)\n",
    "    metric = load_metric(\"accuracy\")\n",
    "    metric.compute(predictions=y_pred, references=y_true)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "90a845ec",
   "metadata": {},
   "source": [
    "#### Saving model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "73908b10",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to models/sentiment_classification/model\n",
      "Configuration saved in models/sentiment_classification/model/config.json\n",
      "Model weights saved in models/sentiment_classification/model/pytorch_model.bin\n",
      "tokenizer config file saved in models/sentiment_classification/model/tokenizer_config.json\n",
      "Special tokens file saved in models/sentiment_classification/model/special_tokens_map.json\n",
      "tokenizer config file saved in models/sentiment_classification/tokenizer/tokenizer_config.json\n",
      "Special tokens file saved in models/sentiment_classification/tokenizer/special_tokens_map.json\n"
     ]
    }
   ],
   "source": [
    "def save_model(trainer,tokenizer,trainer_path,tokenizer_path):\n",
    "    trainer.save_model(trainer_path)\n",
    "    tokenizer.save_pretrained(tokenizer_path)\n",
    "    \n",
    "save_model(trainer,tokenizer,\"models/author_classification/model\",\"models/author_classification/tokenizer\")\n",
    "save_model(trainer_2,tokenizer,\"models/sentiment_classification/model\",\"models/sentiment_classification/tokenizer\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f9710a8",
   "metadata": {},
   "source": [
    "#### Loading this model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a7953079",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "loading configuration file models/sentiment_classification/model\\config.json\n",
      "Model config RobertaConfig {\n",
      "  \"_name_or_path\": \"models/sentiment_classification/model\",\n",
      "  \"architectures\": [\n",
      "    \"RobertaForSequenceClassification\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"bos_token_id\": 0,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-05,\n",
      "  \"max_position_embeddings\": 514,\n",
      "  \"model_type\": \"roberta\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 1,\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"problem_type\": \"single_label_classification\",\n",
      "  \"torch_dtype\": \"float32\",\n",
      "  \"transformers_version\": \"4.15.0\",\n",
      "  \"type_vocab_size\": 1,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 50265\n",
      "}\n",
      "\n",
      "loading weights file models/sentiment_classification/model\\pytorch_model.bin\n",
      "All model checkpoint weights were used when initializing RobertaForSequenceClassification.\n",
      "\n",
      "All the weights of RobertaForSequenceClassification were initialized from the model checkpoint at models/sentiment_classification/model.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use RobertaForSequenceClassification for predictions without further training.\n",
      "Didn't find file models/sentiment_classification/tokenizer\\added_tokens.json. We won't load it.\n",
      "Didn't find file models/sentiment_classification/tokenizer\\tokenizer.json. We won't load it.\n",
      "loading file models/sentiment_classification/tokenizer\\vocab.json\n",
      "loading file models/sentiment_classification/tokenizer\\merges.txt\n",
      "loading file None\n",
      "loading file models/sentiment_classification/tokenizer\\special_tokens_map.json\n",
      "loading file models/sentiment_classification/tokenizer\\tokenizer_config.json\n",
      "loading file None\n",
      "No `TrainingArguments` passed, using `output_dir=tmp_trainer`.\n",
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
     ]
    }
   ],
   "source": [
    "def load_model(model_path,tokenizer_path,num_labels):\n",
    "    model = AutoModelForSequenceClassification.from_pretrained(model_path, num_labels = num_labels)\n",
    "    tokenizer = RobertaTokenizer.from_pretrained(tokenizer_path)\n",
    "    trainer = Trainer(model = model)\n",
    "    return (model, trainer, tokenizer)\n",
    "    \n",
    "\n",
    "model, trainer, tokenizer = load_model(\"models/author_classification/model\",\"models/author_classification/tokenizer\",4)\n",
    "\n",
    "model_2, trainer_2, _ = load_model(\"models/sentiment_classification/model\",\"models/sentiment_classification/tokenizer\",2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "fc4ad067",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.config.finetuning_task = \"Classify 19th century authors\"\n",
    "model.config.id2label = {0:\"Author one\", 1:\"Author two\", 2:\"Author three\", 3:\"Author four\"}\n",
    "\n",
    "\n",
    "model_2.config.finetuning_task = \"Classify sentiment from imdb reviews\"\n",
    "model_2.config.id2label = {0:\"Negative\", 1:\"Positive\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c0c120f5",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using custom data configuration default-ca238c53a2a39e26\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset pandas/default to C:\\Users\\dovyd\\.cache\\huggingface\\datasets\\pandas\\default-ca238c53a2a39e26\\0.0.0\\6197c1e855b639d75a767140856841a562b7a71d129104973fe1962594877ade...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bed35fdfb43243ddb16d6af918cc03e5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "081be54c15b6495591f207c76697c0a6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset pandas downloaded and prepared to C:\\Users\\dovyd\\.cache\\huggingface\\datasets\\pandas\\default-ca238c53a2a39e26\\0.0.0\\6197c1e855b639d75a767140856841a562b7a71d129104973fe1962594877ade. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8aa54789528344dfb3c8c0916f994910",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the test set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: __index_level_0__, text.\n",
      "***** Running Prediction *****\n",
      "  Num examples = 149\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[48  0  0  0]\n",
      " [ 0 21  0  0]\n",
      " [ 1  0  5  0]\n",
      " [ 2  1  0 71]]\n",
      "{'accuracy': 0.9731543624161074}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using custom data configuration default-58145b16daa5f1bf\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset pandas/default to C:\\Users\\dovyd\\.cache\\huggingface\\datasets\\pandas\\default-58145b16daa5f1bf\\0.0.0\\6197c1e855b639d75a767140856841a562b7a71d129104973fe1962594877ade...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6caeb0e3ab3b4fbea404f34c0f0e6a85",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "45f9b19319df48a1bfa014b157c4fbd4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset pandas downloaded and prepared to C:\\Users\\dovyd\\.cache\\huggingface\\datasets\\pandas\\default-58145b16daa5f1bf\\0.0.0\\6197c1e855b639d75a767140856841a562b7a71d129104973fe1962594877ade. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "735c765e4c0a45ebbb40b8d5394b33e9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the test set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: __index_level_0__, text.\n",
      "***** Running Prediction *****\n",
      "  Num examples = 150\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='19' max='19' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [19/19 05:01]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[73  4]\n",
      " [ 9 64]]\n",
      "{'accuracy': 0.9133333333333333}\n"
     ]
    }
   ],
   "source": [
    "def tokenize_function(examples):\n",
    "    return tokenizer(examples[\"text\"], padding=\"max_length\", truncation=True)\n",
    "\n",
    "\n",
    "def read_and_tokenize_data(data):\n",
    "    d = load_dataset('pandas', data_files=data,split=\"train\")\n",
    "    d = d.map(tokenize_function, batched=True)\n",
    "    return d\n",
    "\n",
    "\n",
    "def make_predictions(trainer,data_path):\n",
    "    test_data = read_and_tokenize_data(data_path)\n",
    "    predictions = trainer.predict(test_data)\n",
    "    y_true = test_data[\"label\"]\n",
    "    y_pred = np.argmax(predictions.predictions, axis=-1)\n",
    "    print(confusion_matrix(test_data[\"label\"], y_pred))\n",
    "    metric = load_metric(\"accuracy\")\n",
    "    print(metric.compute(predictions=y_pred, references=y_true))\n",
    "\n",
    "\n",
    "\n",
    "make_predictions(trainer,\"data/authors_test.pkl\")\n",
    "\n",
    "make_predictions(trainer_2,\"data/imdb_test.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1f7489cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import TextClassificationPipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "14df73c7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'label': 'Author one', 'score': 0.9999176263809204}]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline = TextClassificationPipeline(model=model,tokenizer=tokenizer)\n",
    "\n",
    "pipeline(\"was a great step to take this step of marriage there was a responsibility about it that was when one approached it so nearly supposing things should go wrong what if the market should take a turn to the bad and everything go all at once to the there he would be with a double expense on his hands for in those days he ceased to believe the stories of the extraordinary with which a married couple could live five days more and it would be too late to his steps thoughts of closing his store and running away came into his head and held sway there for hours at a time it would be but would it not in the end prove the part of wisdom ah yes he had been right when he said a man should have fifty thousand dollars and be forty years of age before he such a folly yo fa tt s i he told and her mother that evening that he had a violent headache as an excuse for going to his room and staying there neither of them believed this statement covered the whole truth and they commented upon the matter when he had left them it is not unusual for men to have queer at such a time as this said the wise elder lady i have heard of them being so ill that the wedding had to be postponed we must deal very gently with mr in the matter for it would be awkward to have anything happen now awkward echoed it would be outrageous don t you think if i went to his room and bathed his head it would be a good thing the mother did not agree with the idea she said it was best to leave him entirely alone for the present it would not do to act as if anything was suspected she did not know that at the very minute she was saying this was seriously\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "7dbaf980",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'label': 'Positive', 'score': 0.998927891254425}]\n",
      "[{'label': 'Negative', 'score': 0.9993883371353149}]\n"
     ]
    }
   ],
   "source": [
    "pipeline_2 = TextClassificationPipeline(model=model_2,tokenizer=tokenizer)\n",
    "\n",
    "\n",
    "print(pipeline_2(\"An all time classic\")) \n",
    "\n",
    "\n",
    "print(pipeline_2(\"\"\"Despite the occasional great moments the movie is\n",
    "                 mostly not up to standards for the director\"\"\"))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}