categorization-svm-2/cross_validation_bubbles.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Implementation on new data with Bubble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import re\n",
    "import sys\n",
    "import csv\n",
    "import time\n",
    "import json\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from featurize import tokenize_dict\n",
    "from tqdm import tqdm_notebook\n",
    "from unidecode import unidecode\n",
    "\n",
    "from collections import defaultdict\n",
    "from collections import Counter\n",
    "from nltk import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import precision_recall_fscore_support\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "from sklearn.cross_validation import cross_val_predict\n",
    "from sklearn.cross_validation import LeaveOneLabelOut\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "source = 'data/product_nuts_with_product_info.jsonl'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "data_pn_usage = []\n",
    "\n",
    "with open(source) as f:\n",
    "    for line in tqdm_notebook(f):\n",
    "        data_pn_usage.append(json.loads(line))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/lib/python2.7/dist-packages/unidecode/__init__.py:46: RuntimeWarning: Argument <type 'str'> is not an unicode object. Passing an encoded string will likely have unexpected results.\n",
      "  _warn_if_not_unicode(string)\n"
     ]
    }
   ],
   "source": [
    "id_tokens = [tokenize_dict(d) for d in data_pn_usage]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Remove doubles\n",
    "- Include only those nut products if the usage appears in at least 3 products"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# removes doubles\n",
    "known = set()\n",
    "no_doubles = []\n",
    "\n",
    "for d in tqdm_notebook(id_tokens):\n",
    "    tok = ' '.join(d['tokens'])\n",
    "    if tok in known: \n",
    "        continue\n",
    "    no_doubles.append(d)\n",
    "    known.add(tok)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# makes a dict where each tuple has one usage and all the product_ids linked to it\n",
    "k = [{x['usage']: x['product_id']} for x in no_doubles]\n",
    "dd = defaultdict(list)\n",
    "\n",
    "for d in tqdm_notebook(k): \n",
    "    for key, value in d.iteritems():\n",
    "        dd[key].append(value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# checks if a usage is linked to at least 3 products\n",
    "allowed_usage = []\n",
    "for x in tqdm_notebook(dd.items()):\n",
    "    if len(set(x[1])) > 2:\n",
    "        allowed_usage.append(x[0])\n",
    "        \n",
    "set_allowed_usage = set(allowed_usage)\n",
    "\n",
    "complete = []\n",
    "for x in tqdm_notebook(no_doubles):\n",
    "    if x['usage'] in set_allowed_usage:\n",
    "        complete.append(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "LABELS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "labels = [x['product_id'] for x in complete]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "K_fold = int(len(set(labels)) / 6)\n",
    "\n",
    "one = set(list(set(labels))[:K_fold])\n",
    "two = set(list(set(labels))[K_fold:(K_fold * 2)])\n",
    "three = set(list(set(labels))[(2 * K_fold):(K_fold * 3)])\n",
    "four = set(list(set(labels))[(3 * K_fold):(K_fold * 4)])\n",
    "five = set(list(set(labels))[(4 * K_fold):(K_fold * 5)])\n",
    "six = set(list(set(labels))[(5 * K_fold):])\n",
    "\n",
    "labels_6 = []\n",
    "for label in tqdm_notebook(labels):\n",
    "    if label in one:\n",
    "        labels_6.append(1)\n",
    "    elif label in two:\n",
    "        labels_6.append(2)\n",
    "    elif label in three:\n",
    "        labels_6.append(3)\n",
    "    elif label in four:\n",
    "        labels_6.append(4)\n",
    "    elif label in five:\n",
    "        labels_6.append(5)\n",
    "    elif label in six:\n",
    "        labels_6.append(6)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "text_per_item_pn_TRAIN = [' '.join(tokens['tokens']) for tokens in complete if tokens['product_id'] not in six]\n",
    "text_per_item_pn_VALIDATE = [' '.join(tokens['tokens']) for tokens in complete if tokens['product_id'] in six]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "vectorizer = CountVectorizer(min_df=1, binary=True)\n",
    "X_TRAIN = vectorizer.fit_transform(text_per_item_pn_TRAIN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vocab = vectorizer.vocabulary_\n",
    "vectorizer = CountVectorizer(min_df=1, vocabulary=vocab)\n",
    "X_VALIDATE = vectorizer.fit_transform(text_per_item_pn_VALIDATE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Y_TRAIN = [x['usage'] for x in complete if x['product_id'] not in six]\n",
    "Y_VALIDATE = [x['usage'] for x in complete if x['product_id'] in six]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "labels_TRAIN = []\n",
    "for x in enumerate(labels_6):\n",
    "    if x[1] != 6:\n",
    "        labels_TRAIN.append(x[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X:  (21021, 26314) Y:  21021\n",
      "X:  (119050, 26314) Y:  119050 Labels:  119050\n"
     ]
    }
   ],
   "source": [
    "print 'X: ', X_VALIDATE.shape, 'Y: ', len(Y_VALIDATE)\n",
    "print 'X: ', X_TRAIN.shape, 'Y: ',len(Y_TRAIN), 'Labels: ', len(labels_TRAIN)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cross_Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LibLinear]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/lib/python2.7/dist-packages/sklearn/metrics/classification.py:1115: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LibLinear][LibLinear][LibLinear][LibLinear]"
     ]
    }
   ],
   "source": [
    "cv = LeaveOneLabelOut(labels_TRAIN)\n",
    "\n",
    "clf = LinearSVC(random_state = 2, verbose = 1)\n",
    "scores = cross_val_score(clf, X_TRAIN, Y_TRAIN, cv=cv, scoring='recall_macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall Macro: 0.51 (+/- 0.01)\n"
     ]
    }
   ],
   "source": [
    "print \"Recall Macro: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Validate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LibLinear]"
     ]
    }
   ],
   "source": [
    "clf.fit(X_TRAIN, Y_TRAIN)\n",
    "predictions = clf.predict(X_VALIDATE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/lib/python2.7/dist-packages/sklearn/metrics/classification.py:1113: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/usr/lib/python2.7/dist-packages/sklearn/metrics/classification.py:1115: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "def classifaction_report_csv(report):\n",
    "    report_data = []\n",
    "    lines = report.encode('ascii', 'ignore').split('\\n')\n",
    "    for line in (lines[2:-3] + [lines[-2]]):\n",
    "        row = {}\n",
    "        row_data = line.strip().split('  ')\n",
    "        row_data = [x for x in row_data if x != '']\n",
    "        row['class'] = row_data[0]\n",
    "        row['precision'] = row_data[1]\n",
    "        row['recall'] = row_data[2]\n",
    "        row['f1_score'] = row_data[3]\n",
    "        row['support'] = row_data[4]\n",
    "        report_data.append(row)\n",
    "    dataframe = pd.DataFrame.from_dict(report_data)\n",
    "    #dataframe.to_csv('classification_report.csv', index = False)\n",
    "    return dataframe\n",
    "\n",
    "report = classification_report(Y_VALIDATE, predictions)\n",
    "df = classifaction_report_csv(report)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "macro recall score:  0.540332692954\n",
      "micro recall score:  0.728652300081\n"
     ]
    }
   ],
   "source": [
    "print 'macro recall score: ',recall_score(Y_VALIDATE, predictions, average = 'macro')\n",
    "print 'micro recall score: ',recall_score(Y_VALIDATE, predictions, average = 'micro')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Reliable Recall Macro Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "macro recall score:  0.613273809524\n"
     ]
    }
   ],
   "source": [
    "report_data = []\n",
    "lines = report.encode('ascii', 'ignore').split('\\n')\n",
    "for line in (lines[2:-3] + [lines[-2]]):\n",
    "    row = {}\n",
    "    row_data = line.strip().split('  ')\n",
    "    row_data = [x for x in row_data if x != '']\n",
    "    row['class'] = row_data[0]\n",
    "    row['precision'] = row_data[1]\n",
    "    row['recall'] = row_data[2]\n",
    "    row['f1_score'] = row_data[3]\n",
    "    row['support'] = row_data[4]\n",
    "    report_data.append(row)\n",
    "    \n",
    "recall = []\n",
    "for d in report_data:\n",
    "    if d['support'].strip() != '0':\n",
    "        recall.append(d['recall'])\n",
    "        \n",
    "n_recall = [float(x) for x in recall]\n",
    "print 'macro recall score: ',sum(n_recall) / len(recall)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  },
  "widgets": {
   "state": {
    "02e0d4a9b2a44c8ca13c23898e4a3518": {
     "views": [
      {
       "cell_index": 7
      }
     ]
    },
    "1ed678490ef540468dbd9e1d096f0cee": {
     "views": [
      {
       "cell_index": 6
      }
     ]
    },
    "35f1b9498112492cad1067f943a4d74e": {
     "views": [
      {
       "cell_index": 11
      }
     ]
    },
    "696e5e261bc943329c4cb47be7a08c65": {
     "views": [
      {
       "cell_index": 8
      }
     ]
    },
    "e98401776cb144ac946a83c65e8cf457": {
     "views": [
      {
       "cell_index": 3
      }
     ]
    },
    "f2bb09cc6c284359ae204057044342a9": {
     "views": [
      {
       "cell_index": 8
      }
     ]
    }
   },
   "version": "1.2.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}