3. NLP with HuggingFace.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "33aee254",
   "metadata": {},
   "source": [
    "<div style=\"width: 100%; overflow: hidden;\">\n",
    "    <div style=\"width: 150px; float: left;\"> <img src=\"data/D4Sci_logo_ball.png\" alt=\"Data For Science, Inc\" align=\"left\" border=\"0\" width=150px> </div>\n",
    "    <div style=\"float: left; margin-left: 10px;\"> <h1>LLMs for Data Science</h1>\n",
    "<h1>NLP With HuggingFace</h1>\n",
    "        <p>Bruno Gonçalves<br/>\n",
    "        <a href=\"http://www.data4sci.com/\">www.data4sci.com</a><br/>\n",
    "            @bgoncalves, @data4sci</p></div>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6e9dc6fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "from pprint import pprint\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt \n",
    "\n",
    "from ipywidgets import interact\n",
    "\n",
    "import transformers\n",
    "from transformers import pipeline\n",
    "from transformers import set_seed\n",
    "set_seed(42) # Set the seed to get reproducible results\n",
    "\n",
    "import os\n",
    "import gzip\n",
    "\n",
    "import tqdm as tq\n",
    "from tqdm.notebook import tqdm\n",
    "tqdm.pandas()\n",
    "\n",
    "import networkx as nx\n",
    "\n",
    "import watermark\n",
    "\n",
    "%load_ext watermark\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aa901ccb",
   "metadata": {},
   "source": [
    "We start by printing out the versions of the libraries we're using for future reference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f0aeb234",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python implementation: CPython\n",
      "Python version       : 3.11.7\n",
      "IPython version      : 8.12.3\n",
      "\n",
      "Compiler    : Clang 14.0.6 \n",
      "OS          : Darwin\n",
      "Release     : 24.3.0\n",
      "Machine     : arm64\n",
      "Processor   : arm\n",
      "CPU cores   : 16\n",
      "Architecture: 64bit\n",
      "\n",
      "Git hash: 03802c3bf87993c3670c2fd8bf86e59d3d60bdfd\n",
      "\n",
      "watermark   : 2.4.3\n",
      "matplotlib  : 3.8.0\n",
      "transformers: 4.41.1\n",
      "pandas      : 2.2.3\n",
      "numpy       : 1.26.4\n",
      "tqdm        : 4.66.4\n",
      "networkx    : 3.3\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%watermark -n -v -m -g -iv"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "83dc8d3a",
   "metadata": {},
   "source": [
    "Load default figure style"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f0db9136",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.style.use('d4sci.mplstyle')\n",
    "colors = plt.rcParams['axes.prop_cycle'].by_key()['color']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b60deea4",
   "metadata": {},
   "source": [
    "# Named Entity Recognition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8f1dc17e",
   "metadata": {},
   "outputs": [],
   "source": [
    "email = \"\"\"Dear Amazon, \\\n",
    "\n",
    "last week I ordered an Optimus Prime action figure \\\n",
    "from your online store in Germany. Unfortunately, when I opened the package, \\\n",
    "I discovered to my horror that I had been sent an action figure of Megatron \\\n",
    "instead! As a lifelong enemy of the Decepticons, I hope you can understand my \\\n",
    "dilemma. To resolve the issue, I demand an exchange of Megatron for the \\\n",
    "Optimus Prime figure I ordered. Enclosed are copies of my records concerning \\\n",
    "this purchase. I expect to hear from you soon. \n",
    "\n",
    "Sincerely, \n",
    "\n",
    "Bumblebee.\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "46484fd5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No model was supplied, defaulted to dbmdz/bert-large-cased-finetuned-conll03-english and revision f2482bf (https://huggingface.co/dbmdz/bert-large-cased-finetuned-conll03-english).\n",
      "Using a pipeline without specifying a model name and revision in production is not recommended.\n",
      "/opt/anaconda3/lib/python3.11/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n",
      "Some weights of the model checkpoint at dbmdz/bert-large-cased-finetuned-conll03-english were not used when initializing BertForTokenClassification: ['bert.pooler.dense.bias', 'bert.pooler.dense.weight']\n",
      "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "ner_tagger = pipeline(\"ner\", aggregation_strategy=\"simple\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a7038ad2",
   "metadata": {},
   "outputs": [],
   "source": [
    "outputs = ner_tagger(email)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0cbfdc95",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'entity_group': 'ORG',\n",
       "  'score': 0.8790102,\n",
       "  'word': 'Amazon',\n",
       "  'start': 5,\n",
       "  'end': 11},\n",
       " {'entity_group': 'MISC',\n",
       "  'score': 0.9908588,\n",
       "  'word': 'Optimus Prime',\n",
       "  'start': 37,\n",
       "  'end': 50},\n",
       " {'entity_group': 'LOC',\n",
       "  'score': 0.9997547,\n",
       "  'word': 'Germany',\n",
       "  'start': 91,\n",
       "  'end': 98},\n",
       " {'entity_group': 'MISC',\n",
       "  'score': 0.5565716,\n",
       "  'word': 'Mega',\n",
       "  'start': 209,\n",
       "  'end': 213},\n",
       " {'entity_group': 'PER',\n",
       "  'score': 0.59025526,\n",
       "  'word': '##tron',\n",
       "  'start': 213,\n",
       "  'end': 217},\n",
       " {'entity_group': 'ORG',\n",
       "  'score': 0.66969275,\n",
       "  'word': 'Decept',\n",
       "  'start': 254,\n",
       "  'end': 260},\n",
       " {'entity_group': 'MISC',\n",
       "  'score': 0.4983484,\n",
       "  'word': '##icons',\n",
       "  'start': 260,\n",
       "  'end': 265},\n",
       " {'entity_group': 'MISC',\n",
       "  'score': 0.7753625,\n",
       "  'word': 'Megatron',\n",
       "  'start': 351,\n",
       "  'end': 359},\n",
       " {'entity_group': 'MISC',\n",
       "  'score': 0.98785394,\n",
       "  'word': 'Optimus Prime',\n",
       "  'start': 368,\n",
       "  'end': 381},\n",
       " {'entity_group': 'PER',\n",
       "  'score': 0.8120968,\n",
       "  'word': 'Bumblebee',\n",
       "  'start': 507,\n",
       "  'end': 516}]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fd6d062d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>entity_group</th>\n",
       "      <th>score</th>\n",
       "      <th>word</th>\n",
       "      <th>start</th>\n",
       "      <th>end</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ORG</td>\n",
       "      <td>0.879010</td>\n",
       "      <td>Amazon</td>\n",
       "      <td>5</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>MISC</td>\n",
       "      <td>0.990859</td>\n",
       "      <td>Optimus Prime</td>\n",
       "      <td>37</td>\n",
       "      <td>50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LOC</td>\n",
       "      <td>0.999755</td>\n",
       "      <td>Germany</td>\n",
       "      <td>91</td>\n",
       "      <td>98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>MISC</td>\n",
       "      <td>0.556572</td>\n",
       "      <td>Mega</td>\n",
       "      <td>209</td>\n",
       "      <td>213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>PER</td>\n",
       "      <td>0.590255</td>\n",
       "      <td>##tron</td>\n",
       "      <td>213</td>\n",
       "      <td>217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ORG</td>\n",
       "      <td>0.669693</td>\n",
       "      <td>Decept</td>\n",
       "      <td>254</td>\n",
       "      <td>260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>MISC</td>\n",
       "      <td>0.498348</td>\n",
       "      <td>##icons</td>\n",
       "      <td>260</td>\n",
       "      <td>265</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>MISC</td>\n",
       "      <td>0.775362</td>\n",
       "      <td>Megatron</td>\n",
       "      <td>351</td>\n",
       "      <td>359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>MISC</td>\n",
       "      <td>0.987854</td>\n",
       "      <td>Optimus Prime</td>\n",
       "      <td>368</td>\n",
       "      <td>381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>PER</td>\n",
       "      <td>0.812097</td>\n",
       "      <td>Bumblebee</td>\n",
       "      <td>507</td>\n",
       "      <td>516</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  entity_group     score           word  start  end\n",
       "0          ORG  0.879010         Amazon      5   11\n",
       "1         MISC  0.990859  Optimus Prime     37   50\n",
       "2          LOC  0.999755        Germany     91   98\n",
       "3         MISC  0.556572           Mega    209  213\n",
       "4          PER  0.590255         ##tron    213  217\n",
       "5          ORG  0.669693         Decept    254  260\n",
       "6         MISC  0.498348        ##icons    260  265\n",
       "7         MISC  0.775362       Megatron    351  359\n",
       "8         MISC  0.987854  Optimus Prime    368  381\n",
       "9          PER  0.812097      Bumblebee    507  516"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(outputs)    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab2e9bb0",
   "metadata": {},
   "source": [
    "# PoS Tagging"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b1ae233e",
   "metadata": {},
   "source": [
    "Load the pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1fb1be6e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at vblagoje/bert-english-uncased-finetuned-pos were not used when initializing BertForTokenClassification: ['bert.pooler.dense.bias', 'bert.pooler.dense.weight']\n",
      "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "pos_tagger = pipeline(\"token-classification\", model=\"vblagoje/bert-english-uncased-finetuned-pos\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2e2ddb05",
   "metadata": {},
   "outputs": [],
   "source": [
    "text = \"The quick brown fox jumps over the lazy dog.\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d40f9a4a",
   "metadata": {},
   "source": [
    "Extract the part of speech tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "77402b6a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>entity</th>\n",
       "      <th>score</th>\n",
       "      <th>index</th>\n",
       "      <th>word</th>\n",
       "      <th>start</th>\n",
       "      <th>end</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DET</td>\n",
       "      <td>0.999445</td>\n",
       "      <td>1</td>\n",
       "      <td>the</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ADJ</td>\n",
       "      <td>0.997063</td>\n",
       "      <td>2</td>\n",
       "      <td>quick</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ADJ</td>\n",
       "      <td>0.942299</td>\n",
       "      <td>3</td>\n",
       "      <td>brown</td>\n",
       "      <td>10</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NOUN</td>\n",
       "      <td>0.997004</td>\n",
       "      <td>4</td>\n",
       "      <td>fox</td>\n",
       "      <td>16</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>VERB</td>\n",
       "      <td>0.999446</td>\n",
       "      <td>5</td>\n",
       "      <td>jumps</td>\n",
       "      <td>20</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ADP</td>\n",
       "      <td>0.999325</td>\n",
       "      <td>6</td>\n",
       "      <td>over</td>\n",
       "      <td>26</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>DET</td>\n",
       "      <td>0.999527</td>\n",
       "      <td>7</td>\n",
       "      <td>the</td>\n",
       "      <td>31</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ADJ</td>\n",
       "      <td>0.997863</td>\n",
       "      <td>8</td>\n",
       "      <td>lazy</td>\n",
       "      <td>35</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>NOUN</td>\n",
       "      <td>0.998858</td>\n",
       "      <td>9</td>\n",
       "      <td>dog</td>\n",
       "      <td>40</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>PUNCT</td>\n",
       "      <td>0.999650</td>\n",
       "      <td>10</td>\n",
       "      <td>.</td>\n",
       "      <td>43</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  entity     score  index   word  start  end\n",
       "0    DET  0.999445      1    the      0    3\n",
       "1    ADJ  0.997063      2  quick      4    9\n",
       "2    ADJ  0.942299      3  brown     10   15\n",
       "3   NOUN  0.997004      4    fox     16   19\n",
       "4   VERB  0.999446      5  jumps     20   25\n",
       "5    ADP  0.999325      6   over     26   30\n",
       "6    DET  0.999527      7    the     31   34\n",
       "7    ADJ  0.997863      8   lazy     35   39\n",
       "8   NOUN  0.998858      9    dog     40   43\n",
       "9  PUNCT  0.999650     10      .     43   44"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pos_tags = pos_tagger(text)\n",
    "pd.DataFrame(pos_tags)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a9a604a9",
   "metadata": {},
   "source": [
    "# Summarization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "fd0bf7b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "summarizer = pipeline(\"summarization\", model=\"sshleifer/distilbart-cnn-12-6\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1fb2e2fc",
   "metadata": {},
   "source": [
    "The first 4 paragraphs of https://en.wikipedia.org/wiki/Transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "780d8669",
   "metadata": {},
   "outputs": [],
   "source": [
    "wiki_text = \"\"\"\n",
    "Transformers is a media franchise produced by American toy company Hasbro and Japanese toy company Takara Tomy. It primarily follows the heroic Autobots and the villainous Decepticons, two alien robot factions at war that can transform into other forms, such as vehicles and animals. The franchise encompasses toys, animation, comic books, video games and films. As of 2011, it generated more than ¥2 trillion ($25 billion) in revenue,[1] making it one of the highest-grossing media franchises of all time.\n",
    "\n",
    "The franchise began in 1984 with the Transformers toy line, comprising transforming mecha toys from Takara's Diaclone and Micro Change toylines rebranded for Western markets.[2] The term \"Generation 1\" (G1) covers both the animated television series The Transformers and the comic book series of the same name, which are further divided into Japanese, British and Canadian spin-offs. Sequels followed, such as the Generation 2 comic book and Beast Wars TV series, which became its own mini-universe. Generation 1 characters have been rebooted multiple times in the 21st century in comics from Dreamwave Productions (starting 2001), IDW Publishing (starting in 2005 and again in 2019), and Skybound Entertainment (beginning in 2023). There have been other incarnations of the story based on different toy lines during and after the 20th century. The first was the Robots in Disguise series, followed by three shows (Armada, Energon, and Cybertron) that constitute a single universe called the \"Unicron Trilogy\".\n",
    "\n",
    "A live-action film series started in 2007, again distinct from previous incarnations, while the Transformers: Animated series merged concepts from the G1 continuity, the 2007 live-action film and the \"Unicron Trilogy\". For most of the 2010s, in an attempt to mitigate the wave of reboots, the \"Aligned Continuity\" was established. In 2018, Transformers: Cyberverse debuted, once again, distinct from the previous incarnations.\n",
    "\n",
    "Although a separate and competing franchise started in 1983, Tonka's GoBots became the intellectual property of Hasbro after their buyout of Tonka in 1991. Subsequently, the universe depicted in the animated series Challenge of the GoBots and follow-up film GoBots: Battle of the Rock Lords was retroactively established as an alternate universe within the Transformers multiverse.[3] \n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "649ff19d",
   "metadata": {},
   "source": [
    "To generate the summary we just have to call the pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "037015f6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " The Transformers is a media franchise produced by Hasbro and Japanese toy company Takara Tomy . It primarily follows the heroic Autobots and the villainous Decepticons, two alien robot factions at war that can transform into other forms, such as vehicles and animals . As of 2011, it generated more than ¥2 trillion ($25 billion) in revenue .\n"
     ]
    }
   ],
   "source": [
    "summary = summarizer(wiki_text)\n",
    "\n",
    "print(summary[0]['summary_text'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c140c5fd",
   "metadata": {},
   "source": [
    "We can also specify a minimum length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d903416c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Transformers is a media franchise produced by Hasbro and Japanese toy company Takara Tomy . It primarily follows the heroic Autobots and the villainous Decepticons, two alien robot factions at war that can transform into other forms, such as vehicles and animals . As of 2011, it generated more than ¥2 trillion ($25 billion) in revenue, making it one of the highest-grossing media franchises of all time . The term \"Generation 1\" (G1) covers both the animated television series The Transformers and the comic book series of the same name .\n"
     ]
    }
   ],
   "source": [
    "summary = summarizer(wiki_text, min_length=100)\n",
    "\n",
    "print(summary[0]['summary_text'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1fb1854",
   "metadata": {},
   "source": [
    "# Question Answering "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b59a33de",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No model was supplied, defaulted to distilbert/distilbert-base-cased-distilled-squad and revision 626af31 (https://huggingface.co/distilbert/distilbert-base-cased-distilled-squad).\n",
      "Using a pipeline without specifying a model name and revision in production is not recommended.\n",
      "/opt/anaconda3/lib/python3.11/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "reader = pipeline(\"question-answering\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "7b06141f",
   "metadata": {},
   "outputs": [],
   "source": [
    "question = \"What does the customer want?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "561cbaff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>score</th>\n",
       "      <th>start</th>\n",
       "      <th>end</th>\n",
       "      <th>answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.631292</td>\n",
       "      <td>336</td>\n",
       "      <td>359</td>\n",
       "      <td>an exchange of Megatron</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      score  start  end                   answer\n",
       "0  0.631292    336  359  an exchange of Megatron"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputs = reader(question=question, context=email)\n",
    "pd.DataFrame([outputs])    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6868cac6",
   "metadata": {},
   "source": [
    "# Translation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "7cc732f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "translator = pipeline(\"translation_en_to_it\", \n",
    "                      model=\"Helsinki-NLP/opus-mt-en-it\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "ac60ca47",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cara Amazon, la scorsa settimana ho ordinato una figura d'azione Optimus Prime dal tuo negozio online in Germania. Purtroppo, quando ho aperto il pacchetto, ho scoperto al mio orrore che ero stato inviato una figura d'azione di Megatron invece! Come un nemico per tutta la vita dei Decepticon, spero che si può capire il mio dilemma. Per risolvere il problema, chiedo uno scambio di Megatron per la figura di Optimus Prime ho ordinato. In allegato sono copie dei miei record riguardanti questo acquisto. Mi aspetto di sentire da voi presto. Cordialmente, Bumblebee.\n"
     ]
    }
   ],
   "source": [
    "outputs = translator(email, clean_up_tokenization_spaces=True, min_length=100, max_length=1000)\n",
    "print(outputs[0]['translation_text'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "924b10ed",
   "metadata": {},
   "source": [
    "For comparison, let us look at the results of google translate:\n",
    "\n",
    "```\n",
    "Caro Amazon, la settimana scorsa ho ordinato un action figure di Optimus Prime dal tuo negozio online in Germania. Sfortunatamente, quando ho aperto il pacco, ho scoperto con orrore che mi era stata invece inviata una action figure di Megatron! Essendo un nemico da sempre dei Decepticon, spero che tu possa capire il mio dilemma. Per risolvere il problema, chiedo uno scambio di Megatron con la figura di Optimus Prime che ho ordinato. In allegato sono presenti copie dei miei documenti relativi a questo acquisto. Mi aspetto di sentirti presto. Cordiali saluti, Bombo.\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "27659fb2",
   "metadata": {},
   "source": [
    "Google translate is less context aware in the translation going so far as translating the name of the email sender (Bumblebee -> Bombo). On the other hand, the Hugging Face model is more formal (\"sentire da voi\" -> \"sentirti\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f5101ea4",
   "metadata": {},
   "source": [
    "# Sentiment Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "6f234a7f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No model was supplied, defaulted to distilbert/distilbert-base-uncased-finetuned-sst-2-english and revision af0f99b (https://huggingface.co/distilbert/distilbert-base-uncased-finetuned-sst-2-english).\n",
      "Using a pipeline without specifying a model name and revision in production is not recommended.\n",
      "/opt/anaconda3/lib/python3.11/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "sentiment_pipeline = pipeline(\"sentiment-analysis\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9d732ee0",
   "metadata": {},
   "source": [
    "Let us use a comple of faily obvious instances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "9194345f",
   "metadata": {},
   "outputs": [],
   "source": [
    "instances = [\"I love you\", \"I hate you\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "659be922",
   "metadata": {},
   "source": [
    "The model does a pretty good job of figuring out which one is positive and which one is negative"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "0258363e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'label': 'POSITIVE', 'score': 0.9998656511306763},\n",
       " {'label': 'NEGATIVE', 'score': 0.9991129040718079}]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentiment_pipeline(instances)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "813b1afb",
   "metadata": {},
   "source": [
    "# Application"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "35bfe5be",
   "metadata": {},
   "source": [
    "Load a few thousand tweets about Apple"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "e38a139b",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('data/Apple-Twitter-Sentiment-DFE.csv', usecols=['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "105924ae",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>#AAPL:The 10 best Steve Jobs emails ever...htt...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>RT @JPDesloges: Why AAPL Stock Had a Mini-Flas...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>My cat only chews @apple cords. Such an #Apple...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>I agree with @jimcramer that the #IndividualIn...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Nobody expects the Spanish Inquisition #AAPL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3881</th>\n",
       "      <td>(Via FC) Apple Is Warming Up To Social Media -...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3882</th>\n",
       "      <td>RT @MMLXIV: there is no avocado emoji may I as...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3883</th>\n",
       "      <td>@marcbulandr I could not agree more. Between @...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3884</th>\n",
       "      <td>My iPhone 5's photos are no longer downloading...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3885</th>\n",
       "      <td>RT @SwiftKey: We're so excited to be named to ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3886 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text\n",
       "0     #AAPL:The 10 best Steve Jobs emails ever...htt...\n",
       "1     RT @JPDesloges: Why AAPL Stock Had a Mini-Flas...\n",
       "2     My cat only chews @apple cords. Such an #Apple...\n",
       "3     I agree with @jimcramer that the #IndividualIn...\n",
       "4          Nobody expects the Spanish Inquisition #AAPL\n",
       "...                                                 ...\n",
       "3881  (Via FC) Apple Is Warming Up To Social Media -...\n",
       "3882  RT @MMLXIV: there is no avocado emoji may I as...\n",
       "3883  @marcbulandr I could not agree more. Between @...\n",
       "3884  My iPhone 5's photos are no longer downloading...\n",
       "3885  RT @SwiftKey: We're so excited to be named to ...\n",
       "\n",
       "[3886 rows x 1 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "006c9b23",
   "metadata": {},
   "source": [
    "Compute the sentiment score for each tweet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "2c76b4c6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6244640c013442768f3883a7065d998f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3886 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sent = pd.DataFrame(data['text'].progress_apply(lambda x: pd.Series(sentiment_pipeline(x)[0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "928e0ab3",
   "metadata": {},
   "outputs": [],
   "source": [
    "sent.rename(columns={'score': 'sentiment_confidence', 'label':'sentiment'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "6116abee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiment</th>\n",
       "      <th>sentiment_confidence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>POSITIVE</td>\n",
       "      <td>0.999432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.999122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.996177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>POSITIVE</td>\n",
       "      <td>0.995648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.932676</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3881</th>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.992046</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3882</th>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.999158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3883</th>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.935773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3884</th>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.998303</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3885</th>\n",
       "      <td>POSITIVE</td>\n",
       "      <td>0.998714</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3886 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     sentiment  sentiment_confidence\n",
       "0     POSITIVE              0.999432\n",
       "1     NEGATIVE              0.999122\n",
       "2     NEGATIVE              0.996177\n",
       "3     POSITIVE              0.995648\n",
       "4     NEGATIVE              0.932676\n",
       "...        ...                   ...\n",
       "3881  NEGATIVE              0.992046\n",
       "3882  NEGATIVE              0.999158\n",
       "3883  NEGATIVE              0.935773\n",
       "3884  NEGATIVE              0.998303\n",
       "3885  POSITIVE              0.998714\n",
       "\n",
       "[3886 rows x 2 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sent"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "accfe71c",
   "metadata": {},
   "source": [
    "We can also use NER to identify when a person is mentioned in the tweet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "cbbfdb31",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'#AAPL:The 10 best Steve Jobs emails ever...http://t.co/82G1kL94tx'"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['text'].iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "7f254e40",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'entity_group': 'PER',\n",
       "  'score': 0.73089933,\n",
       "  'word': 'Steve Jobs',\n",
       "  'start': 18,\n",
       "  'end': 28}]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ner_tagger(data['text'].iloc[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8601ef1",
   "metadata": {},
   "source": [
    "Identify all people mentioned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "ee96e05a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_people(x):\n",
    "    output = ner_tagger(x)\n",
    "    \n",
    "    for tag in output:\n",
    "        if tag['entity_group'] == 'PER':\n",
    "            out = {'confidence':tag[\"score\"], 'person': tag['word']}\n",
    "            return pd.Series(out)\n",
    "    \n",
    "    return pd.Series({\"confidence\": None, \"person\": None})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "574f044a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fcfe613d380f44c98215932d2bcdb1fa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3886 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "people = pd.DataFrame(data['text'].progress_apply(find_people))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e3e40b9",
   "metadata": {},
   "source": [
    "Combine all the results into a single DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "011601d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.concat([data, sent, people], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "f4d9106a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>sentiment_confidence</th>\n",
       "      <th>confidence</th>\n",
       "      <th>person</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>#AAPL:The 10 best Steve Jobs emails ever...htt...</td>\n",
       "      <td>POSITIVE</td>\n",
       "      <td>0.999432</td>\n",
       "      <td>0.730899</td>\n",
       "      <td>Steve Jobs</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>RT @JPDesloges: Why AAPL Stock Had a Mini-Flas...</td>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.999122</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>My cat only chews @apple cords. Such an #Apple...</td>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.996177</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>I agree with @jimcramer that the #IndividualIn...</td>\n",
       "      <td>POSITIVE</td>\n",
       "      <td>0.995648</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Nobody expects the Spanish Inquisition #AAPL</td>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.932676</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3881</th>\n",
       "      <td>(Via FC) Apple Is Warming Up To Social Media -...</td>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.992046</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3882</th>\n",
       "      <td>RT @MMLXIV: there is no avocado emoji may I as...</td>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.999158</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3883</th>\n",
       "      <td>@marcbulandr I could not agree more. Between @...</td>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.935773</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3884</th>\n",
       "      <td>My iPhone 5's photos are no longer downloading...</td>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.998303</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3885</th>\n",
       "      <td>RT @SwiftKey: We're so excited to be named to ...</td>\n",
       "      <td>POSITIVE</td>\n",
       "      <td>0.998714</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3886 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text sentiment  \\\n",
       "0     #AAPL:The 10 best Steve Jobs emails ever...htt...  POSITIVE   \n",
       "1     RT @JPDesloges: Why AAPL Stock Had a Mini-Flas...  NEGATIVE   \n",
       "2     My cat only chews @apple cords. Such an #Apple...  NEGATIVE   \n",
       "3     I agree with @jimcramer that the #IndividualIn...  POSITIVE   \n",
       "4          Nobody expects the Spanish Inquisition #AAPL  NEGATIVE   \n",
       "...                                                 ...       ...   \n",
       "3881  (Via FC) Apple Is Warming Up To Social Media -...  NEGATIVE   \n",
       "3882  RT @MMLXIV: there is no avocado emoji may I as...  NEGATIVE   \n",
       "3883  @marcbulandr I could not agree more. Between @...  NEGATIVE   \n",
       "3884  My iPhone 5's photos are no longer downloading...  NEGATIVE   \n",
       "3885  RT @SwiftKey: We're so excited to be named to ...  POSITIVE   \n",
       "\n",
       "      sentiment_confidence  confidence      person  \n",
       "0                 0.999432    0.730899  Steve Jobs  \n",
       "1                 0.999122         NaN        None  \n",
       "2                 0.996177         NaN        None  \n",
       "3                 0.995648         NaN        None  \n",
       "4                 0.932676         NaN        None  \n",
       "...                    ...         ...         ...  \n",
       "3881              0.992046         NaN        None  \n",
       "3882              0.999158         NaN        None  \n",
       "3883              0.935773         NaN        None  \n",
       "3884              0.998303         NaN        None  \n",
       "3885              0.998714         NaN        None  \n",
       "\n",
       "[3886 rows x 5 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b2ecc528",
   "metadata": {},
   "source": [
    "Subset the data to only the tweets meantioning people"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "341d70d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "people = data[data.person.isna() == False].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "1647ec48",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>sentiment_confidence</th>\n",
       "      <th>confidence</th>\n",
       "      <th>person</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>#AAPL:The 10 best Steve Jobs emails ever...htt...</td>\n",
       "      <td>POSITIVE</td>\n",
       "      <td>0.999432</td>\n",
       "      <td>0.730899</td>\n",
       "      <td>Steve Jobs</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>@Apple John Cantlie has been a prisoner of ISI...</td>\n",
       "      <td>POSITIVE</td>\n",
       "      <td>0.930674</td>\n",
       "      <td>0.984458</td>\n",
       "      <td>John Cantlie</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>I'm hoping @apple won't automatically make us ...</td>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.997191</td>\n",
       "      <td>0.889901</td>\n",
       "      <td>Bruce Springsteen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>@thehill @Apple What a joke! Justice Dept shou...</td>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.992056</td>\n",
       "      <td>0.941649</td>\n",
       "      <td>Killer Wilson</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>#AAPL:10 Steve Jobs emails you need to read......</td>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.994819</td>\n",
       "      <td>0.495679</td>\n",
       "      <td>Job</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3777</th>\n",
       "      <td>Jeff Daniels in Talks to Play Former Apple CEO...</td>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.593705</td>\n",
       "      <td>0.999621</td>\n",
       "      <td>Jeff Daniels</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3798</th>\n",
       "      <td>Dear Santa, All I want for Christmas is for @A...</td>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.979491</td>\n",
       "      <td>0.839571</td>\n",
       "      <td>Santa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3812</th>\n",
       "      <td>@Apple co-founder Steve Wozniak talks about St...</td>\n",
       "      <td>POSITIVE</td>\n",
       "      <td>0.938843</td>\n",
       "      <td>0.941354</td>\n",
       "      <td>Steve Wozniak</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3818</th>\n",
       "      <td>RT @CNET: @Apple pioneer Bill Fernandez on @Go...</td>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.922325</td>\n",
       "      <td>0.999579</td>\n",
       "      <td>Bill Fernandez</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3879</th>\n",
       "      <td>Tim Cook Met With Jesse Jackson for 'Positive ...</td>\n",
       "      <td>POSITIVE</td>\n",
       "      <td>0.998170</td>\n",
       "      <td>0.999658</td>\n",
       "      <td>Tim Cook</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>291 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text sentiment  \\\n",
       "0     #AAPL:The 10 best Steve Jobs emails ever...htt...  POSITIVE   \n",
       "17    @Apple John Cantlie has been a prisoner of ISI...  POSITIVE   \n",
       "80    I'm hoping @apple won't automatically make us ...  NEGATIVE   \n",
       "81    @thehill @Apple What a joke! Justice Dept shou...  NEGATIVE   \n",
       "85    #AAPL:10 Steve Jobs emails you need to read......  NEGATIVE   \n",
       "...                                                 ...       ...   \n",
       "3777  Jeff Daniels in Talks to Play Former Apple CEO...  NEGATIVE   \n",
       "3798  Dear Santa, All I want for Christmas is for @A...  NEGATIVE   \n",
       "3812  @Apple co-founder Steve Wozniak talks about St...  POSITIVE   \n",
       "3818  RT @CNET: @Apple pioneer Bill Fernandez on @Go...  NEGATIVE   \n",
       "3879  Tim Cook Met With Jesse Jackson for 'Positive ...  POSITIVE   \n",
       "\n",
       "      sentiment_confidence  confidence             person  \n",
       "0                 0.999432    0.730899         Steve Jobs  \n",
       "17                0.930674    0.984458       John Cantlie  \n",
       "80                0.997191    0.889901  Bruce Springsteen  \n",
       "81                0.992056    0.941649      Killer Wilson  \n",
       "85                0.994819    0.495679                Job  \n",
       "...                    ...         ...                ...  \n",
       "3777              0.593705    0.999621       Jeff Daniels  \n",
       "3798              0.979491    0.839571              Santa  \n",
       "3812              0.938843    0.941354      Steve Wozniak  \n",
       "3818              0.922325    0.999579     Bill Fernandez  \n",
       "3879              0.998170    0.999658           Tim Cook  \n",
       "\n",
       "[291 rows x 5 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "people"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c4775cd9",
   "metadata": {},
   "source": [
    "Convert the text labels to a numerical score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "64b30b04",
   "metadata": {},
   "outputs": [],
   "source": [
    "people['sentiment'] = people.apply(lambda x: 1 if x.sentiment == 'POSITIVE' else -1, axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c52fab24",
   "metadata": {},
   "source": [
    "Compute the average score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "960b48c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "stats = people[['person', 'sentiment']].groupby('person').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "50efa095",
   "metadata": {},
   "outputs": [],
   "source": [
    "counts = people[['person', 'sentiment']].groupby('person').count()\n",
    "counts.rename(columns={'sentiment':'count'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "4202add7",
   "metadata": {},
   "outputs": [],
   "source": [
    "stats = stats.join(counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "e0b844fe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiment</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>person</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Steve Wozniak</th>\n",
       "      <td>-0.200000</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Steve</th>\n",
       "      <td>-0.333333</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Eddy Cue</th>\n",
       "      <td>-0.500000</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Steve Job</th>\n",
       "      <td>-0.647059</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mark Zuckerberg</th>\n",
       "      <td>-0.714286</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Steve Jobs</th>\n",
       "      <td>-0.777778</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Tim Cook</th>\n",
       "      <td>-0.793103</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Job</th>\n",
       "      <td>-0.800000</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 sentiment  count\n",
       "person                           \n",
       "Steve Wozniak    -0.200000     10\n",
       "Steve            -0.333333      6\n",
       "Eddy Cue         -0.500000      8\n",
       "Steve Job        -0.647059     17\n",
       "Mark Zuckerberg  -0.714286      7\n",
       "Steve Jobs       -0.777778     27\n",
       "Tim Cook         -0.793103     29\n",
       "Job              -0.800000     10"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stats[stats['count']>=5].sort_values('sentiment', ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4f66d008",
   "metadata": {},
   "source": [
    "<center>\n",
    "     <img src=\"data/D4Sci_logo_full.png\" alt=\"Data For Science, Inc\" align=\"center\" border=\"0\" width=300px> \n",
    "</center>"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}