examples/20newsgroup_binary.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "typical-hearing",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "buried-trading",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "I'll take a wild guess and say Freedom is objectively valuable.  I base\n",
      "this on the assumption that if everyone in the world were deprived utterly\n",
      "of their freedom (so that their every act was contrary to their volition),\n",
      "almost all would want to complain.  Therefore I take it that to assert or\n",
      "believe that \"Freedom is not very valuable\", when almost everyone can see\n",
      "that it is, is every bit as absurd as to assert \"it is not raining\" on\n",
      "a rainy day.  I take this to be a candidate for an objective value, and it\n",
      "it is a necessary condition for objective morality that objective values\n",
      "such as this exist.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import fetch_20newsgroups\n",
    "\n",
    "categories = ['alt.atheism', 'comp.graphics']\n",
    "newsgroups_train = fetch_20newsgroups(subset='train', categories=categories, remove=('headers', 'footers', 'quotes'))\n",
    "print(newsgroups_train.data[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "physical-belgium",
   "metadata": {},
   "source": [
    "# Pre-processing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "662120ad-6552-4dff-b849-5f34bf52a474",
   "metadata": {},
   "source": [
    "**Run below for the dependencies:**\n",
    "- ```pip install spacy```\n",
    "- ```python -m spacy download en_core_web_sm```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "covered-glance",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/haystack/maksim/cp_als_numpy/.eggs/tqdm-4.64.0-py3.8.egg/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "from spacy.lang.en.stop_words import STOP_WORDS\n",
    "import en_core_web_sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "sticky-economics",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['only',\n",
       " 'further',\n",
       " 'where',\n",
       " 'throughout',\n",
       " 'anyone',\n",
       " 'down',\n",
       " 'being',\n",
       " 'name',\n",
       " 'whence',\n",
       " 'first']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import string\n",
    "\n",
    "punctuations = string.punctuation\n",
    "stopwords = list(STOP_WORDS)\n",
    "stopwords[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "first-machine",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>I'll take a wild guess and say Freedom is obje...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Is there a precompiled version of hp2xx for DO...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Does anyone know of any good shareware animati...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>It should have been made fairly clear that the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1059</th>\n",
       "      <td>Hi ... Recently I found XV for MS-DOS in a sub...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1060</th>\n",
       "      <td>(reference line trimmed)[...]Yes.Well, for exa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1061</th>\n",
       "      <td>No need to correct it, it stands as it is sa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1062</th>\n",
       "      <td>For the last time, Bobby. Lack of belief in YO...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1063</th>\n",
       "      <td>Hi there,I've made a VGA mode 13h graphics lib...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1064 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text\n",
       "0     I'll take a wild guess and say Freedom is obje...\n",
       "1     Is there a precompiled version of hp2xx for DO...\n",
       "2     Does anyone know of any good shareware animati...\n",
       "3     It should have been made fairly clear that the...\n",
       "4                                                      \n",
       "...                                                 ...\n",
       "1059  Hi ... Recently I found XV for MS-DOS in a sub...\n",
       "1060  (reference line trimmed)[...]Yes.Well, for exa...\n",
       "1061    No need to correct it, it stands as it is sa...\n",
       "1062  For the last time, Bobby. Lack of belief in YO...\n",
       "1063  Hi there,I've made a VGA mode 13h graphics lib...\n",
       "\n",
       "[1064 rows x 1 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.DataFrame(newsgroups_train.data, columns=[\"text\"])\n",
    "df = df.replace('\\n','', regex=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "complex-rapid",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parser\n",
    "parser = spacy.load(\"en_core_web_sm\")\n",
    "parser.max_length = 7000000\n",
    "\n",
    "def spacy_tokenizer(sentence):\n",
    "    mytokens = parser(sentence)\n",
    "    mytokens = [ word.lemma_.lower().strip() if word.lemma_ != \"-PRON-\" else word.lower_ for word in mytokens ]\n",
    "    mytokens = [ word for word in mytokens if word not in stopwords and word not in punctuations ]\n",
    "    mytokens = \" \".join([i for i in mytokens])\n",
    "    return mytokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "subtle-mixer",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1064/1064 [00:22<00:00, 48.20it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "tqdm.pandas()\n",
    "df[\"processed_text\"] = df[\"text\"].progress_apply(spacy_tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "nearby-crawford",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       wild guess freedom objectively valuable baseth...\n",
       "1       precompile version hp2xx dos prefereablyfor 38...\n",
       "2       know good shareware animation paint software s...\n",
       "3       fairly clear crimson wouldever 150 75 old styl...\n",
       "4                                                        \n",
       "                              ...                        \n",
       "1059    hi ... recently find xv ms dos subdirectory gn...\n",
       "1060    reference line trimmed)[ ... ]yes example goal...\n",
       "1061    need correct stand miss point entirely thing d...\n",
       "1062    time bobby lack belief god implyatheism moslem...\n",
       "1063    hi i've vga mode 13h graphic library available...\n",
       "Name: processed_text, Length: 1064, dtype: object"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"processed_text\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "south-browser",
   "metadata": {},
   "source": [
    "# Vectorize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "rural-pilot",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "text = df['processed_text'].values\n",
    "vectorizer = TfidfVectorizer(max_features=30)\n",
    "X = vectorizer.fit_transform(text).toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "involved-sailing",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1064, 30)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "tamil-improvement",
   "metadata": {},
   "outputs": [],
   "source": [
    "y = newsgroups_train.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "delayed-voltage",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.        , 0.        , 1.        , ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.        , 0.        , 0.        , ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.        , 0.        , 0.        , ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       ...,\n",
       "       [0.        , 0.        , 0.        , ..., 0.52671177, 0.        ,\n",
       "        0.        ],\n",
       "       [0.        , 0.31779536, 0.        , ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.        , 0.        , 0.        , ..., 0.        , 0.        ,\n",
       "        0.39560222]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0876167c-a88d-4a0d-a5ab-bdc855b5db4c",
   "metadata": {},
   "source": [
    "# Organize Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "08788f4e-68f9-4a3e-a3ae-c2e3430a74ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = y.copy()\n",
    "y_experiment = y_true.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "cffcd560-941c-46b4-b6cb-6b2b29972fd5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "rng = np.random.RandomState(42)\n",
    "random_unlabeled_points = rng.rand(y_experiment.shape[0]) < 0.3\n",
    "y_experiment[random_unlabeled_points] = -1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "modified-situation",
   "metadata": {},
   "source": [
    "# RFoT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "vocal-relaxation",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:17<00:00,  5.88it/s]\n",
      "100%|██████████| 100/100 [00:08<00:00, 11.87it/s]\n",
      "100%|██████████| 100/100 [00:10<00:00,  9.36it/s]\n",
      "100%|██████████| 100/100 [00:09<00:00, 10.09it/s]\n"
     ]
    }
   ],
   "source": [
    "from RFoT import RFoT\n",
    "\n",
    "model = RFoT(\n",
    "        bin_scale=1,\n",
    "        max_dimensions=8,\n",
    "        component_purity_tol=1.0,\n",
    "        min_rank=2,\n",
    "        max_rank=10,\n",
    "        n_estimators=100,\n",
    "        bin_entry=True,\n",
    "        clustering=\"ms\",\n",
    "        max_depth=4,\n",
    "        n_jobs=50,\n",
    ")\n",
    "y_pred = model.predict(X, y_experiment)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "raising-pacific",
   "metadata": {},
   "source": [
    "# Look at the results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "sapphire-universe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------\n",
      "Num. of Abstaining 235\n",
      "Percent Abstaining 70.35928143712576 %\n",
      "F1= 0.9698783541333965\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import f1_score\n",
    "\n",
    "unknown_indices = np.argwhere(y_experiment == -1).flatten()\n",
    "did_predict_indices = np.argwhere(y_pred[unknown_indices] != -1).flatten()\n",
    "abstaining_count = len(np.argwhere(y_pred == -1))\n",
    "f1 = f1_score(\n",
    "    y_true[unknown_indices][did_predict_indices],\n",
    "    y_pred[unknown_indices][did_predict_indices],\n",
    "    average=\"weighted\",\n",
    ")\n",
    "\n",
    "print(\"------------------------\")\n",
    "print(\"Num. of Abstaining\", abstaining_count)\n",
    "print(\"Percent Abstaining\", (abstaining_count / len(unknown_indices)) * 100, \"%\")\n",
    "print(\"F1=\", f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "undefined-colorado",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.93      0.96      0.94        26\n",
      "           1       0.99      0.97      0.98        73\n",
      "\n",
      "    accuracy                           0.97        99\n",
      "   macro avg       0.96      0.97      0.96        99\n",
      "weighted avg       0.97      0.97      0.97        99\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "\n",
    "y_true_hat = y_true[unknown_indices][did_predict_indices]\n",
    "y_pred_hat = y_pred[unknown_indices][did_predict_indices]\n",
    "print(classification_report(y_true_hat, y_pred_hat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68c347c1-03cc-426e-b9da-249e2cdfe7a6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "RFoT",
   "language": "python",
   "name": "rfot"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}