Crime_Types_Model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "#sklearn\n",
    "from sklearn import metrics\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.decomposition import LatentDirichletAllocation\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.tree import DecisionTreeRegressor, DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier, ExtraTreesClassifier\n",
    "from sklearn.linear_model import LinearRegression, Lasso, LogisticRegression, LogisticRegressionCV\n",
    "from sklearn.model_selection import KFold, StratifiedKFold, train_test_split\n",
    "from sklearn.feature_selection import RFECV, RFE\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.model_selection import GridSearchCV, cross_val_score, train_test_split, StratifiedKFold\n",
    "\n",
    "#matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "#vis\n",
    "import altair as alt\n",
    "alt.renderers.enable('notebook')\n",
    "\n",
    "nlp = spacy.load('es')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Data From File\n",
    "## Run through Load_Data.py script first"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "post_df = pd.read_csv(\"./Data/cleaned_data.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Review the output of the keyword-based crime type classifier first"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Crime Type\n",
       "Disappearance     127\n",
       "Drugs               9\n",
       "Extortion          13\n",
       "Homicide           20\n",
       "Kidnapping          5\n",
       "Noncriminal      3264\n",
       "Rape               30\n",
       "Robbery           295\n",
       "Shared Link       240\n",
       "Violence          124\n",
       "dtype: int64"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "crimeTypeGroups = post_df.groupby(\"Crime Type\") \n",
    "crimeTypeGroups.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Group                Crime Type   \n",
       "AlertaHuixquiluca    Disappearance      4\n",
       "                     Extortion          1\n",
       "                     Homicide           2\n",
       "                     Noncriminal      174\n",
       "                     Robbery            2\n",
       "                     Violence           3\n",
       "Atizapa              Noncriminal       37\n",
       "                     Robbery            4\n",
       "                     Violence           1\n",
       "DenunciaChimalhuaca  Disappearance      1\n",
       "                     Drugs              1\n",
       "                     Extortion          1\n",
       "                     Homicide           1\n",
       "                     Noncriminal       93\n",
       "                     Robbery            9\n",
       "                     Violence           4\n",
       "DenunciaenEdoMe      Disappearance     25\n",
       "                     Drugs              2\n",
       "                     Extortion          4\n",
       "                     Homicide           3\n",
       "                     Kidnapping         1\n",
       "                     Noncriminal      391\n",
       "                     Rape               1\n",
       "                     Robbery           58\n",
       "                     Shared Link        1\n",
       "                     Violence           7\n",
       "NoticiasChimalhuaca  Disappearance      4\n",
       "                     Extortion          1\n",
       "                     Noncriminal      150\n",
       "                     Robbery           13\n",
       "                                     ... \n",
       "SanMateoAtenco       Drugs              3\n",
       "                     Extortion          4\n",
       "                     Homicide           1\n",
       "                     Noncriminal      697\n",
       "                     Rape               2\n",
       "                     Robbery           32\n",
       "                     Violence           6\n",
       "ValleDeChalc         Disappearance     21\n",
       "                     Drugs              3\n",
       "                     Extortion          1\n",
       "                     Homicide           4\n",
       "                     Noncriminal      470\n",
       "                     Rape               2\n",
       "                     Robbery           66\n",
       "                     Violence          27\n",
       "denuncianeza0        Disappearance     13\n",
       "                     Extortion          1\n",
       "                     Kidnapping         1\n",
       "                     Noncriminal      208\n",
       "                     Rape               2\n",
       "                     Robbery           70\n",
       "                     Shared Link      237\n",
       "                     Violence          12\n",
       "denunciateotihuaca   Disappearance     21\n",
       "                     Homicide           3\n",
       "                     Kidnapping         1\n",
       "                     Noncriminal      583\n",
       "                     Rape               2\n",
       "                     Robbery           35\n",
       "                     Violence           9\n",
       "dtype: int64"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "statsTemp = post_df.groupby([\"Group\",\"Crime Type\"]) \n",
    "statsTemp.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "alt.Chart(crime_post_df.loc[:,[\"Group\", \"Vertex\", \"Crime Type\"]]).mark_bar().encode(\n",
    "    x='count(Vertex)',\n",
    "    y='Group',\n",
    "    color=alt.Color('Crime Type'),\n",
    "    order=alt.Order(\"Crime Type\",'count',sort='descending'),\n",
    ").properties(\n",
    "    title=\"Post Types by Group\",\n",
    "    height=400, \n",
    "    width=700\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "crimeVsNonDisplayDF = post_df.loc[:,[\"Month\", \"Vertex\", \"Crime Type\", \"Group\"]]\n",
    "crimeVsNonDisplayDF[\"Crime Type\"] = crimeVsNonDisplayDF[\"Crime Type\"].apply(lambda x: \"Other\" if x in [\"Noncriminal\", \"Shared Link\"]\\\n",
    "                                                                            else \"Crime\")\n",
    "base = alt.Chart(crimeVsNonDisplayDF).mark_bar().encode(\n",
    "    y='count(Vertex)',\n",
    "    x=alt.X('Month', sort=[\"Jan\",\"Feb\", \"Mar\", \"Apr\", \"May\", \"Jun\", \"Jul\", \"Aug\", \"Sep\", \"Oct\", \"Nov\", \"Dec\"]),\n",
    "    color=alt.Color('Crime Type')\n",
    ").properties(\n",
    "    width=150,\n",
    "    height=100\n",
    ")\n",
    "\n",
    "#create a list of subplots\n",
    "subplts = []\n",
    "for grp in crimeVsNonDisplayDF['Group'].unique():\n",
    "    subplts.append(base.transform_filter(alt.datum.Group == grp).properties(title= grp))\n",
    "\n",
    "\n",
    "def facet_wrap(subplts, plots_per_row):\n",
    "    rows = [subplts[i:i+plots_per_row] for i in range(0, len(subplts), plots_per_row)]\n",
    "    compound_chart = alt.hconcat()\n",
    "    for r in rows:\n",
    "        rowplot = alt.vconcat() #start a new row\n",
    "        for item in r:\n",
    "            rowplot |= item #add suplot to current row as a new column\n",
    "        compound_chart &= rowplot # add the entire row of plots as a new row\n",
    "    return compound_chart\n",
    "\n",
    "\n",
    "compound_chart = facet_wrap(subplts, plots_per_row=4)    \n",
    "compound_chart"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Try a classifier to get better accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "goldFeatures = post_df[post_df[\"Label_Id\"].isin([0,1,2,3,4,5,6])]\n",
    "goldFeatures = goldFeatures.drop([\"Vertex\", \"Popularity Measure\", \"Total Likes\", \"Total Shares\", \"Total Comments\",\n",
    "                       \"Month\", \"Type\", \"Date\", \"Post Id\", \"Gold_Crime_Type_Label\", \"Text\", \"Crime Type\"], axis=1)\n",
    "goldX = goldFeatures.drop([\"Label_Id\", \"Group\"],axis=1)\n",
    "goldY = goldFeatures.loc[:,\"Label_Id\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.9791376912378303\n"
     ]
    }
   ],
   "source": [
    "#MNB classifier to get important features per cluster\n",
    "feature_names = np.array(goldX.columns.values)\n",
    "clfGB = GradientBoostingClassifier(n_estimators = 200, max_depth=2, random_state=120)\n",
    "clfGB.fit(goldX, goldY)\n",
    "print \"accuracy: \" + str(clfGB.score(goldX, goldY))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### There are likely too many classes and not enough data. Also the class imbalance causes the model to always predict noncriminal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.7285318559556787\n",
      "[[ 42   5   0   0   0   0  15]\n",
      " [  3  11   0   0   0   0  19]\n",
      " [  0   0   0   0   0   0   1]\n",
      " [  0   0   0   0   0   1   0]\n",
      " [  2   0   0   0   1   1   9]\n",
      " [  2   1   0   0   0   0   6]\n",
      " [  7  18   0   0   5   3 209]]\n",
      "accuracy: 0.7569832402234636\n",
      "[[ 47   3   0   0   2   0   9]\n",
      " [  0   5   0   0   0   0  28]\n",
      " [  0   0   0   0   0   0   1]\n",
      " [  0   1   0   0   0   0   0]\n",
      " [  3   1   0   0   1   0   7]\n",
      " [  0   0   0   0   0   3   5]\n",
      " [ 10  12   0   0   3   2 215]]\n"
     ]
    }
   ],
   "source": [
    "#K-Fold Evaluation\n",
    "for train, test in StratifiedKFold(2).split(goldX, goldY):\n",
    "    clfGB.fit(goldX.iloc[train,:], goldY.iloc[train])\n",
    "    predictedK = clfGB.predict(goldX.iloc[test,:])\n",
    "    print \"accuracy: \" + str(metrics.accuracy_score(goldY.iloc[test], predictedK))\n",
    "    print metrics.confusion_matrix(goldY.iloc[test], predictedK)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Review locations that are predictive overall, and for each class\n",
    "### NGRAM_ indicates that this term is in the post text\n",
    "### COMMENT_ indicates that this term is in the comments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NGRAM_robar, NGRAM_placa, Has Image, COMMENT_defender, COMMENT_amigar, NGRAM_camioneta, COMMENT_policias, NGRAM_denunciar, NGRAM_compartir, COMMENT_abusar, NGRAM_dejar, NGRAM_dinero, COMMENT_mesar, NGRAM_hash, COMMENT_hijo, NGRAM_favor, COMMENT_alos, NGRAM_localizar, NGRAM_coacalco, NGRAM_color, COMMENT_agarrar, NGRAM_edad, NGRAM_pagar, NGRAM_por favor, COMMENT_jaja, NGRAM_violencia, COMMENT_autoridad, COMMENT_esto, COMMENT_onda, NGRAM_las, NGRAM_madre, NGRAM_vestir, NGRAM_asaltar, NGRAM_llamar, NGRAM_ecatepec, NGRAM_personar, COMMENT_asi, COMMENT_andar, NGRAM_llegar, NGRAM_vida\n"
     ]
    }
   ],
   "source": [
    "topTermLocs = np.argsort(clfGB.feature_importances_)[::-1][:40]\n",
    "topTerms = \", \".join(feature_names[topTermLocs])\n",
    "print topTerms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Robbery: NGRAM_asaltar, NGRAM_robar, NGRAM_placa, NGRAM_acabar, NGRAM_armar, NGRAM_color, NGRAM_auto, NGRAM_mensaje, NGRAM_noche, NGRAM_calle, NGRAM_camioneta, NGRAM_una, NGRAM_llevar, NGRAM_zona, NGRAM_denunciar, NGRAM_tipo, NGRAM_con, NGRAM_estado xico, NGRAM_cuidar, COMMENT_jajajaja\n",
      "Disappearance: NGRAM_amigo, NGRAM_vestir, NGRAM_edad, NGRAM_marzo, NGRAM_salir, NGRAM_localizar, COMMENT_dios, NGRAM_ecatepec, Has Image, NGRAM_nombrar, NGRAM_tipo, NGRAM_informaci, NGRAM_regresar, NGRAM_negro, NGRAM_nos, NGRAM_familiar, NGRAM_menor, NGRAM_solicitar, NGRAM_favor, NGRAM_publicar\n",
      "Homicide: NGRAM_hash, NGRAM_vida, NGRAM_personar, NGRAM_mesar, NGRAM_con, NGRAM_madre, NGRAM_coacalco, NGRAM_pagar, COMMENT_defender, NGRAM_por favor, NGRAM_compartir, NGRAM_por, COMMENT_polic, COMMENT_andar, NGRAM_favor, COMMENT_osea, COMMENT_resultar, COMMENT_policias, COMMENT_aparecer, COMMENT_garcia\n",
      "Violence: NGRAM_violencia, NGRAM_personar, NGRAM_unidad, NGRAM_las, NGRAM_hijo, NGRAM_edad, NGRAM_poder, NGRAM_armar, NGRAM_sacar, NGRAM_llegar, NGRAM_caer, NGRAM_foto, NGRAM_andar, NGRAM_informar, NGRAM_localizar, NGRAM_polic, NGRAM_casar, NGRAM_mujer, NGRAM_camioneta, NGRAM_del\n",
      "Extortion: COMMENT_cargar, NGRAM_dinero, NGRAM_julio, NGRAM_coacalco, NGRAM_ser, COMMENT_robar, NGRAM_denunciar, NGRAM_buscar, NGRAM_llamar, NGRAM_dejar, NGRAM_andar, NGRAM_traer, NGRAM_personar, NGRAM_llegar, NGRAM_facebook, COMMENT_caro, COMMENT_sentir, COMMENT_siempre, COMMENT_alcanzar, COMMENT_regalar\n",
      "Noncriminal: NGRAM_perrito, NGRAM_aguar, NGRAM_escuela, NGRAM_necesitar, NGRAM_formar, NGRAM_alguien, NGRAM_gracias, NGRAM_perro, NGRAM_san, COMMENT_pri, NGRAM_juan, NGRAM_estar, NGRAM_perder, COMMENT_escuela, NGRAM_ciudad, NGRAM_valle, NGRAM_com, NGRAM_haber, COMMENT_personar, NGRAM_callar\n"
     ]
    }
   ],
   "source": [
    "for i, label in enumerate([\"Robbery\", \"Disappearance\", \"Homicide\",\n",
    "                   \"Violence\", \"Extortion\", \"Noncriminal\"]):\n",
    "    topTermLocs = np.argsort(clfGB.coef_[i])[::-1][:20]\n",
    "    topTerms = \", \".join(feature_names[topTermLocs])\n",
    "    print(\"%s: %s\" % (label, topTerms))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Trimming features results in a slightly more compact explanation for each term, if not a better model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/lib/python2.7/site-packages/sklearn/model_selection/_split.py:597: Warning: The least populated class in y has only 2 members, which is too few. The minimum number of members in any class cannot be less than n_splits=3.\n",
      "  % (min_groups, self.n_splits)), Warning)\n",
      "/anaconda/lib/python2.7/site-packages/sklearn/utils/__init__.py:93: FutureWarning: Conversion of the second argument of issubdtype from `int` to `np.signedinteger` is deprecated. In future, it will be treated as `np.int64 == np.dtype(int).type`.\n",
      "  if np.issubdtype(mask.dtype, np.int):\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimal number of features : 470\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEbCAYAAAA1T5h7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd8FHX6wPHPk0BCC7330EUEFETFfvbKWRDb2Que7Yqe+ju9Oz09z/PsHc/eUBQ7iqIUFRASeqihpVCSQBohfZ/fHzMblpDsDpBJgef9eu1rd2dndp+dbOaZb5nvV1QVY4wxBiCqrgMwxhhTf1hSMMYYU8GSgjHGmAqWFIwxxlSwpGCMMaaCJQVjjDEVLCkYY4yp0MjLSiLSETgW6AoUAsuABFUN+BibMcaYWibhLl4TkZOBe4G2wEIgA2gCDAD6Ah8DT6hqnv+hGmOM8VukpPA48JyqplTxWiPgXCBaVT/xL0RjjDG1JWxSMMYYc3Dx1NAsIneKSEtxvCYiC0TkdL+DM8YYU7u89j66zm03OB1oA/wO+LdvURljjKkTXpOCuPdnA++oalLIMmOMMQcIr0khUUS+w0kKU0UkDrDuqMYYc4Dx1NAsIlHAcGCdquaISDugm6ou8TtAY4wxtcfTxWuqGhCRrcBgtyuqMcaYA5DXK5ofA8YBy4Fyd7ECs3yKyxhjTB3wWn20ChiqqsX+h2SMMaaueG1oXgc09jMQY4wxdc9r+8BOYJGI/ABUlBZU9Q5fojLGGFMnvCaFL9ybMcaYA5jnsY9EJAZndFSAVapa6ltUxhhj6oTXhuaTgLeADThXMvcArlZV631kjDEHEK9JIRG4XFVXuc8HAB+o6gif4zPGGFOLvPY+ahxMCACquhrrjWSMMQccrw3NCSLyP+Bd9/kVQII/IYXXvn177d27d118tDHGNFiJiYlZqtoh0npek8ItwK1AsAvqT8CL+xjbfunduzcJCXWSj4wxpsESkY1e1vM69lEx8KR7M8YYc4AKmxRE5CNVvUREluKMdbQbVR3qW2TGGGNqXaSSwp3u/bl+B2KMMabuhe19pKqb3Ye/V9WNoTfg95HeXETOFJFVIpIsIvdW8XorEflSRBaLSJKIXLtvX8MYY0xN8Nol9bQqlp0VbgMRiQZecNcbDFwmIoMrrXYrsFxVhwEnAU+4V04bY4ypA5HaFG7BKRH0EZHQWdbigF8ivPcoIFlV17nvNREYgzMnQ5ACcSIiQAtgO1C2V9/AGGNMjYnUpvA+8A3wKBBa/ZOvqtsjbNsNSA15ngYcVWmd53EG2tuEk2jGqarN/WyMMXUkUptCrqpuUNXL3HaEQpyz+xYi0rMGPv8MYBHQFWcO6OdFpGXllUTkJhFJEJGEzMzMGvhYY8JTVT5OTCO7oKSuQzGmVnlqUxCR80RkDbAemIkzMN43ETZLxxk4L6i7uyzUtcBkdSS77z+o8hup6gRVHamqIzt0iHhBnqll2wtKyC2sP4PmbsktorR8/wqci1JzuGvSYv7387oaisqYhsFrQ/PDwNHAalWNB04B5kbYZj7QX0Ti3cbjS9lzToYU970QkU7AQJxZ3kwDoapcOmEOt7ybWNehAJC7s5RTnpjBA58t26/3mbLU6Xj340ormZqDi9ekUKqq24AoEYlS1enAyHAbqGoZcBswFVgBfKSqSSIyXkTGu6v9ExjtXhz3A3CPqmbt0zcxu3nq+9Xc/sHCGn3P1O07+eunSznliRmsy9wBQMLGbFZv3cHstdtYn1VQo583a3UmZz3zE7k7vZdCPluUTkFJORPnp7IoNWefPldVmbJ0C9FRworNeWzOLdyn96lO1o5iNuVEfs/kjB2c+uRMNmQVoKosSs2hpKx2mtzKA8rvXvuVD+aloKpc88Y8PklMq7H3zysq5Zxnf+LHlVurfD2/qJTU7Ttr7PPCeejL5Yx54RcmL0jD6/wye+PW9xfw0oy1Nf6+fvGaFHJEpAUwC3hPRJ4BIh4BVHWKqg5Q1b6q+oi77GVVfdl9vElVT1fVw1R1iKq+G/4djRc7S8p47ef1fLN0M0Wl5dWu98OKrXy+yKnRi/TPUFYe4Hev/cqkxDRStxfyovsj/3B+Ks1iookS+CghNex77K1JiWms2JzHO3M37PGaqvLsD2tYkJK927KJ81MZ2CmODnGx3P7BAm57fwFL0nIqXvdicVou6TmF3HB8PAAzVlVdWigqLefRKStYlp672/KC4jIen7qy2oPaTW8ncMoTM/l22ZawcXycmEZyxg4mL0jjqyWb+e0LvzD63z/y5eJNnr5HUEZ+EY9+s4KsHcWRV3Z9u2wLP63J4v1fU1i5JZ8ZqzL5Nil8vHvjg19TSNqUx+eL9vwuRaXljH15Duc8+xOFJeX8kpzFO3M21MjnFhSX8eCXSWxz90VxWTkfzEth5eY8/vTRYqYmVZ2k9tWO4jKmLN3MK7PWUlxW/f9ifeI1KYzBmaf5j8C3wFrgPL+CMo4tuUX8uHIrpeUBkjPy9zj4VOebpVvYUVxGWUBJ2pRX7XoPf72CBz5bRll5gL99nsTVr8+jpCzA6q35JG7M3u0g+m3SFjZs28kz44Zz+VE9+WxhOsvSc/l6yWbGDO/KyQM78kliGmUhdfmLUnO4+Z0EDn/oO75Zurli+coteWzJLQr7HcrKA8xa7RyM3/hlA+uzCpi5OrPiTHlq0lae/H41r87aVdu4LD2PFZvzuPLonjx+8VBaNmnMjysz+OdXy9m2o5iT/juDF6YnV6yfXVDCvPW7d6IrLCnnqe9X0zha+P2J/ejWuikfJaTy0fxUNucWUlRazoxVGewsKWPCrHW8Mmsdl7wyhxmrMire44XpybwwfS03vp1AYcnuB4JVW/JZkJJDTKMobnkvkTlrt+32uqoyb/12ysoDFVVYXy/dzAfzUujSqgntW8Tw4JdJVbaZzFiVwciHp+2RjCYlpPHKzHVc+OLsaktzxWXlLE7dlTxfmunsp6XpuXw4P7UidoCFKdn71QBfXFbO67+sB2D22m0Vv7OMvCImJaTyp48WsXJLPnlFZXy5ZBP3fLKEf361IuwJzqacQtZsza94Hggo01dmkLNz9zi/X76VN37ZUHHmPm/9dgpLy3nussPp3LIJE+eneP4eqspVr8/jka+X7/Ha6q35bM0rYmlaLqqQs7OUacszqniXXaYmbeG4x37cq5KxH7wmhY5AjKqWqepbwKs4XUiNTwIBZfy7iVz3ZgIjH57GqU/OYuzLc8gvKuW7pC08PW31HttMWbqZez5ewhuz19MhLhag4h+9svVZBazPKiCvqIyfkrP4MCGVmaszuf6t+Zz33M9c9NJsznv+ZzbnFqKqvDxzLX3aN+f0Qztzw/HxKHDucz9TUh7g8lG9GHdkDzLyiyvOtD6an8rFL81m3vrtdIiL5Zb3FvDBvBSKSssZ98pc7vlkSZVxBS1KzSG3sJSrj+nFtoISTv7vDK5+fR7HPvYjz0xbw7+mrABgzrptBALKmq353PJeIi1iG3H+8G6cNLAjX99xPHedPpD5G7IZ/24iG7ft5PGpq5juHsDv/ngJl7wyhye+W4WqsrOkjMtencusNZn87dzBtGrWmHOGdmFhSg5/+WQJxz02nVGPTOOaN+Yz9uU5vDRjLScO6EDPts24c+IiikrL2bitgP/9tJ5h3Vuxams+N7+bSNKmXcn8w/mpNI4WvrnzeDq0iOXFGcm7fe8vFm/iklfmcNM7iaRs38mw7q1Ym1nA7LXbuGxUT+4+YyBZO0r4YcXuB5jisnL+/kUSWTuKmTg/hQ1ZBdz7yRIKS8qZv2E7nVrGkldUyv2fLa1yfz/oVqHMWJXB9FUZLEvP4+pjegHw7lxncM2U7TvJyC9i3IS5/OPLpLB/v8oCAWVrXhEbtxXwr69XsDWvmHOGdiEzv5i1blXkP75M4u6PlzBl6RbuPKU/vds1459fLSctu5CS8gBLw5wU3fHBQq5+fV5FgnlxRjLXvjmfox/9gTfcBAQwe61TO/3BvBRyd5YyfWUmMY2iOL5/By4e0Z1ZqzOZviqDP364iOSM/Co/K2jOum3MWp3Jx4lplAd2nUA5v/E53P3xEha7pdR2zWP40C1Jz16bxf2fLaWotJw1W/O5/7OllJYHmJq0hbTsQj5bVLk/Tu3yOnT2JGB0yPNyd9mRNR6RAeDTheksSs3hhuPiycgvpm3zGN6cvYFPF6bz4vS1bM0v4qpjepNbWMrm3EKOim/HI1+vIN2tq/7LmQN5Z85GFqflsGpLPnlFpRzZuy3JGfnkFpZV1LdHCfzzy+WUlAU4tl87flqTxeE9W3Ph4d14ZMoKHvl6BacN7sSy9Dz+feFhREcJ3ds04y9nDCRl+05+d0wvBnVuyeCuLYlv35yXZ64lv6iUeycv5fj+7XnhiiOIiY7i6tfn8Z9vVwKQW1jKL8lZ5OwsoXUz5wL2krIA01ZsZUCnFvTrGMePKzOIjhL+dPpAGkdHER0tHN6jNRPnp/KUmxAvPLwbkxemk7Axm1veTSQqSnjvhqNo1XTX/E+XjurBsz+uYf6GbC4b1YOFKTn88cNFPD1uONNWbKVXu2Y892MyadmFlAeUJWk5vHTFCM4c0hmA+84axA3HxZNbWMrHiWlk7ijmsG6t+M+3qygPKP8cM4SU7Tu58rVfmZq0ha+WbKZRtDDhqpF8u2wLj327knOe/Zl3rh/FqPi2fLowjdMGd6Jr66Zce2w8j327kmXpuQzp1gqAifOcA0fw+z8+dhhnPO3MenvxiO50jIulY1wsHyWkVsQI8NrP69m4bSc92zbj48Q0FqbkMHvtNo7q05bEjdmcO7QrnVs24alpq0ndvpMebZtVbJu0KZcP5qUg4iSH/KIy+nZozn1nH8K3SVvYmlfMIV1asmJzHu/OTaGkLMA3y7bw0M5SWjXbta+/X76VgZ3i6Nlu13uDcwB+eeZaNm7bVYIZM7wrfz5tIF8v2czstdvo2ropP67MYOyI7vz59IF0btWEmEZRPD51Fd1aNyU9p5CEDdkc2bvtHv8ryRn5JGx0qhHXZxWwJbeIJ79fzemDO5G1o5jnfkzmmtG9ERFmr93GwE5xrNqazwszkpm+KoNj+rSjaUw0l4zswfPTk7n2jfmAU7368u9GMLpv+yr/R1+e6ZRSs3eWsig1BxFo3bQxyzblkb3T+Y2XlJXTo21TLhjejeemJ3Pdm/OZtTqTsoAyKr4d3y/fypeLN3H2YV1IdL/Dh/NTuXp07yo/szZ4TQqNVLWiHKaqJTYchT+Ky8qZvCCdx6euYniP1vzf2YcQFSWoKrPXZvHolJUUusXoWaszeX9eCotScnjg3ENIzynk0QsPo3XTxpw8qCNLUnNJ2JDNdW/OJ7ewlJ/vOZnr30pgU04hPdo0o1/HFrRvEcPcddvpGBfLG9eMYubqTI7v354mjaPJ2lHCMz+sYdqKrYzo1YaLRnSviPPmE/vuFnd0lHDzCX24d/JS/u9TJyG8fs2RNI52CqN3nTGQsS/P4cEvk4iLbUR+cRnfJW3lkiN7sCg1hxvfTiAzv5jmMdH84dQBfLYwnRG92tCqaWPuP3fX6ChnDulCcsYONm4rYHDXlkxemM69nyxhW0EJn/5+NMN6tN4trmYxjRh/Yl/e/GUD95w5iJydpZz33M9c9+Z8mjSOYvIto3n/1xSe+N5JNHedPmC3g62I0LFlEzq2bMJ9Zx9Ssfz4/u3ZtqOEnu2a0b1NU7q3acpj36xkU24Rd58xkE4tm3D16N78dng3Tn1qJm/N3khGXjHZO0sZd6Rzic/lR/XkhenJ3P/ZMu47axCdWzVhzrpt3PGbfvycnEX7FrEM6BTHKYM6Edsoiq6tmwJOcnh55lqWpuVyWPdW5BeV8tKMtZx6SCcuHtGd8e8msjWvGBF47sdk8ovKGNmrDUf3bcfTP6xmUkIqVx7Ti3fnpjBjVQabcgpp0yyGB849hD9+uJimjaN5/8ajaNI4mpMHdmTi/FRuOakvd3ywkPd/3YiIk8Rf+3kdyZk7OOewrrRrEcONbyfQullj/nfVSEa6B+8P56dw3+SlHN6zNdeO7k1Mo2iO7tOWPh1aANC9TVNmJ2+jXfNYikoDXHhEdzq3alLxPV+ZuZZ7zhrE09+vJnHjdsD53akqL85YS3pOIdEiiICqUx31wbwUerVrzlPjhvPpwnTu/2wZadnOyVJadiEPnn8o89ZvZ4Jb9RgsEfVs14wzD+3MlrwiHjj3EO79ZClXvTaPB8ccyshebenWpiktYp1D5rL0XGatzuTmE/rw6k/reG/uRqYs20yL2EZ0a+2st6O4jLnrtnPu0C7cfGJfCkrKmZSQyjF927Eus4DXfl7Pis1O9e7HCWls3LaTvh2as3xzHvd87Pymnxw3jLjYRuwsKad5rNfD9f7x+imZInK+qn4BICJjAOslFME7czbQulkMZw7pXHFwDCcjv4gb30pgcVouh3ZtyeMXDyUqSgDn4HTJyB48/PUKBnWOIzO/mLfmbGBhinPG/7cvkmjdrDEXHtGN2EbRAAzr0Xq3xsEb305g47adNIuJZl1WATed0Id2zZ2kcNaQzsQ0iuK0wZ0q1h9/Yl8mJaRSVBbg+csPj/gdLjiiG09PW0OjaOHZS3df/8jebRnRqw2JG7O57fR+TJyfypRlm7loRHfu/WQJjaKEF684ghdnJPPIlBV0ahnL7b/pV+Xn9OvYgn4dnYNKn/bNWZdVwNF92nJ4zzZVrj/+xL7cfEIfRITWzWJ4fOwwxr+byCUje9CuRSy3n9KfPh1akLQpl9+fVPVn7hlDHP06Oo+jopy/zZPfr6Zn22Zcf1x8xXqtmjXmoiO68+pP61i9NZ+BneI4ob9z5tmqaWP+ft5gHv56BeMmzKVlk0ZECVx+VC/uPHUA4r7H/64euVv7zlXH9OazhemMmzCHFy4/guSMHeQXlXH7b/oxuGtL2reIpV3zGI7q05a352ys2P/dWjfl+P4deO3n9bw0cy2l5cqo+LYM79GGa0b35th+7UjPLmRIt1YM6OTUDl93XDxxTRpx9pDO/F9sI7J2lDAqvi0FxWU8+6NT9TVtRQZdWzWha6smxDaO5vL//cpTlwynSeMoHvg8ieP7t+fNa0cRHSVUdly/9ny6MJ2N23fSvkUso+J3lQQ6tWzC4r+fjojw0+pMvl+xlUBA2ZJXxL+mrOCrJbvaqU4f3Ill6bm8OXsDyRk7eGjMoTSPbcRw9yRhUWoOO0ucEXRG923HlUf34rxhXZixKpMxw7tVvM9LVx6BM+oOfHzLaG59bwF//dTp3twsJpqLR3TnrjMG8tBXy2nTrDG/P7kfC1KymbwwnSaNoygoLmdxWi5/PHUAHy9IJXV7IcN7tKZ5bCMeOHcw9501iOgo4Zkf1vD0tDWAkxg/dauMHjh3MOPfTWRSolNi/PNHi1FV5q7bzjd3Hr9bCc8vXpPCeJxeR88DgjN8xVW+RXUAWLUlnwc+d+pde7Ztxiu/G8HUpC28/2sKAVUePH8IpxzSkYtems0VR/XitMGduOil2WTll/DiFUdw1pDOFT/OoAuP6M5bczbwx9MGMDVpC5MXpBMlcO2x8bz283ouOHxXQgAY1sOpkjjnsC7kFZXy05oserdrxjOXHs4fPlzE+cO60qppY96Zu5GxI3tQWdOYaD6+xak17NKqacTvHNsomsm/H02TxtG0ab5nQfLPpw3gvk+XMnZkD/KKynj95/X84UOnUfH5yw/n7MO6cPLAjixMdaoJvCTSY/q2Y11WwR4ll8pC9+WZQzrz9R3HVSQWgHOGduGcoV0ifl51LhnZg0mJqTw05lCaNI6u9JpzZp+yfSdPjRu2WyxjR/bg3KFd+XxROu/+upFDu7SqOFOuLv7OrZrw6a3Hcv1b87nh7QSaxURzTJ92FaWkD28+muYxjUjP2cnbczbSMS6WHm2dv98Nx8WzfFMeY4d24erRvYlv33y3z7ntN/13ez6gUxx/PccpqQ3qHEfCxmxG923HgE5xPD1tNf939iHcN3kpG7bt5PnLD2d03/bc9HYCt76/AIBe7Zrx9LjhVSYEgD+dNoCl6bkkbXI6CFReL/i9R/Zuw6TENK5+Y15F4/RfzhzIgI5xPPTVcm48oQ8T56XyyYI0YhtFMWaYc6Af2DmO2EZRLE7NYXNuEe1bxNKvYwtEhDOHdOHMIV2q/DxwkvYb1x7JrNWZ7Cwpd0rmv6bw7bItZOQX868LDqOVWyqfvyGb35/Uj74dWvDUtNVccmR3dpaW8crMdbuVXhu5v+mLR3TnmR/WcGjXllxweHf++dVyYhtFMbpvez6/9ThaNGnElCWbeWTKCqIEGkdH8fDXy3nld2GvBKgRsjf9ct1uqajqDt8iimDkyJHaEKbj/OdXy3l7zgb+O3YY/5qygsz8YgIKJw/swPLNefRp34LrjovnxrcTaBwtDOgUR3LGDibedHS1Z7yhvly8ids/WMgpgzry0pUjeHFGMpeN6kmnlrsOKKXlAV6YnszlR/VkbUYBl706l0cvPIzLRtXECCX7Z0tuEXd8sJB5G7ZzVHxbJt509B5J0Iu1mTv4dtkWfn9S333avrZcOmEOadmFTL/rJE/JzouC4jJue38B01dl8tZ1ozhxwO5X+wcCyon/nc6Inm14+tLD9/vz7v9sKe/OTeHDm47mqD7tKpav3prPL8lZFfX2RaXlvDA9md7tmnPusC67nahU9z1e+3k9F4/oXlFFVtmGrAJ+88QM4po05tIje3Dl0b32OGv+JDGNP09azAWHd+OpccMrll/00myyC0qctp+je/GP8w/d533w05pMbnl3Ab3bN+PzW48jOkrILijh3bkbufGEPrudEGTkFfHO3I3ceUr/imQQ6u05GxjUuSXtW8TwmydmMqp3Wz4af0zF66rKK7PWMaRrKxan5fD41FW8c/0oju+/b6M6iEiiqkbMKmGTgohcqarvisifqnpdVWt9es6GkBRKygIc/egPHBXflpeuHMGmnELum7yUEwZ04Lpje/PU96t5fnoyx/fvwMKUbFo2bUxadiFPjB22W719OHlFpVzx6q/cf84hu/2DhpOcsYO+HZrXq4PnuswdtGsRu1vj8IEou6CEkvLAbkm7JpSVB1ifVUD/TlV3BszML6ZJ4yjimuz//p21OpMXZyTz1nWjIh7o/ZCckU+31s1oGlP1Z2/bUcw1b8zn0QsPq2i4B+fitNd/WU90lDDz7pPo3mb/qmAy84uJaRRVY79ZVeWmdxI5ZVBHLq3mhK2otJwzn57FRUd05/ZT+le5TiQ1lRRuUtUJIvL3ql5X1Qf3Kbr9UB+SQml5gPKA7lFNoKr8+9uVJGzIJnFjNm9ceyQnD+y4x/YrNudx1jM/AU4x8raT+7FsUy7nDu1aK/EbczD5fFE6d05cxG+Hd62RElNdKSwprzYheuE1KURqUwhW1C5X1Un7HM0B5PNF6fzhw0WownXHxvO38waTuHE7Pdo2Y87abbwycx2DOsdx1pDOnFBNMW9Q5zji2zdnfVYBZw3pTO/2zeldqW7XGFMzju/fgeP7t9/nM+z6Yn8Swt6IlBTOdqfRvA/nuoSD3uu/bKBHm2YM69Ga139ZT8r2AqatyKBjXCwiMKRbS7649biKXkNVEREuHtGdt2Zv4Lj+VfeBNsbUjLbNY3jn+spTuZjqREoK3wLZOPMnhI6XIICq6h5zHxzIVm3JZ3FqDvefcwjXjO5NVn4x01ZkcN6wrizYmE16TiHPXnp42IQQdMuJfbnh+Pg6qZs1xpjqhE0Kqno3cLeIfK6qY2oppnrrowRniIILDu9Go+goXrlqBItScji+f3u2F5Swamu+50bfqCghNsoSgjGmfvF0nYIlBKcR+YvFmzhlUCfatXDGFWrZpDEnuN0A27WIZbS73BhjGqqwHaZF5Gf3Pl9E8tz74K364TcPQJtzi8jML2Z0P28lAWOMaYgiVR8d594f9COiBoetPrRrqwhrGmNMw+V1jua+IhLrPj5JRO4QkdaRtjuQJG3KI0rgkC4HfX40xhzAvF5v/wlQLiL9gAlAD+B936Kqh5I25dK3QwuaxdTOSIXGGFMXvCaFgDvn8gXAc26vpH0fPawBWpaex6FdD6oeuMaYg5DXpFAqIpcBVwNfucsO7MFqQmTtKGZLXtFu46kYY8yByGtdyLU4w2c/oqrrRSQeeMe/sOqHvKJSPl+YzuI0a2Q2xhwcvF6nsBy4A0BE2gBxqvqYn4HVBVWluCxAk8bRJG7M5qrXfqXAnXi9XfMYhnSz6iNjzIHNU1IQkRnA+e76iUCGiPyiqlUOqd1QTV+Vwa3vLWTqH07gnTkbaNwoii9uOprDurVCFU/DVxhjTEPmtU2hlarmARcCb6vqUcCp/oVVN1Zt2UFhaTnv/rqRH1ZkcPrgTgzt3hoRsYRgjDkoeE0KjUSkC3AJuxqaDziZ+cUAvPbzevKLyzjrsIOqg5UxxnhOCg8BU4FkVZ0vIn2ANf6FVTcydzhJoTygxDVpxLF9bVhrY8zBxWtD8yRC5lNQ1XXARX4FVVey8osZ3qM1W/OKOKF/B2Ia1cxcusYY01B4bWhuAlwPHApUTDKrqtf5FFedyNxRTP+OLXjjmiNrbZYjY4ypT7yeCr8DdAbOAGYC3YF8v4KqK5n5xXSIi6VN85g95l82xpiDgdek0E9VHwAKVPUt4BzggJrfrrisnNzCUjrYnAjGmIOY52Eu3PscERkCtAI6+hNS3di2owSA9nGWFIwxBy+vw1xMcK9kfgD4AmgB/M23qOpAsDuqlRSMMQczr72P/uc+nAn08S+culORFKykYIw5iIVNCiISdhgLVX2yZsOpO1nuNQpWfWSMOZhFKikcNNOMBUsK7VvE1HEkxhhTdyLN0fxgbQVS1zJ3FNOqaWNiG1lXVGPMwcvrHM1vhc7JLCJtROR1/8KqfZn5xVZKMMYc9Lx2SR2qqjnBJ6qaDRzuT0h1I2tHsTUyG2MOel6TQpTbJRUAEWmLh55LInKmiKwSkWQRubeK1+8WkUXubZmIlLvvXes25xbRIa5J5BWNMeYA5jUpPAHMEZF/isg/gdnAf8JtICLRwAsZ8VxJAAAehElEQVTAWcBg4DIRGRy6jqo+rqrDVXU4cB8wU1W37+2X2F+p23eSll3I8B6tI69sjDEHME9JQVXfxplgZ6t7u1BVI83RPApnqO11qloCTATGhFn/MuADL/HsjyVpOWzcVrDbshmrMgA4eWAHvz/eGGPqNa9XNAfnaV6+F+/dDUgNeZ5GNeMliUgz4Ezgtmpevwm4CaBnz557EcKe/vLxEuLbN+elK0dULPtxZQa92jUjvn3z/XpvY4xp6OrLhAHnAb9UV3WkqhNUdaSqjuzQYf/O5otKy0nLLtzt+ey12zh5YEdEbMpNY8zBzc+kkA70CHne3V1WlUuphaojgIA6jcpBc9dto7gswElWdWSMMZ6vU3jMy7JK5gP9RSReRGJwDvxfVPE+rYATgc+9xLK/ygNK1o5iSsoCACxMyUEEjuxdJ52ejDGmXvFaUjitimVnhdtAVctw2gimAiuAj1Q1SUTGi8j4kFUvAL5T1YKq3qemqSoAW/Oc0kLSpjz6tG9O81jPzSvGGHPAijQg3i3A74G+IrIk5KU4nG6pYanqFGBKpWUvV3r+JvCmt3D3X7mbFLbkFdGjbTOSNuUyKt5KCcYYA5F7H70PfAM8CoRefJZfF9cT1ISAkxPYnFvE9oISNucWcWjXlnUblDHG1BNhq49UNVdVNwDPANtVdaOqbgTKRKRBTscZcLPCltxCkjblAjCka6u6DMkYY+oNr20KLwE7Qp7vcJc1OAG3+mhTThHL0vMAGGwlBWOMAbxfvCYabKEFVDUgIg2yZba8oqRQRNaOYrq3aUrrZjY6qjHGgPeSwjoRuUNEGru3O4F1fgbml2BqS8vZSeLGbKs6MsaYEF6TwnhgNM7FZ8HhKm7yKyg/BXsfLUvPY3NuERcc0a2OIzLGmPrDUxWQqmbgXHzW4AV21YLRp31zTjukUx1GY4wx9YvXK5oHiMgPIrLMfT5URO73NzR/BALQulljAG44vg9RUTbekTHGBHmtPnoVZ76DUgBVXUIDLTkEVDnz0M7ceUp/LhphVUfGGBPKaw+iZqo6r9IoomU+xOO7clU6xMXyx9MG1HUoxhhT73gtKWSJSF9AAUTkYmCzb1H5RFVRxYbINsaYangtKdwKTAAGiUg6sB64wreofBIc4iLakoIxxlQpYlIQkShgpKqeKiLNgShVzfc/tJoX7HlkbcvGGFO1iNVHqhoA/uI+LmioCQF2Xc1sPY6MMaZqXtsUponIXSLSQ0TaBm++RuaD4CUKUVZ9ZIwxVfLapjDOvb81ZJkCfWo2HH8Fr2aOri8zUxtjTD3jtU3hSlX9pRbi8dWuNgUrKRhjTFW8tik8Xwux+C44l4IlBWOMqZrXipQfROQiaeAd/AMVbQp1G4cxxtRXXpPCzcAkoERE8kQkX0TyfIzLF8HeR9GWFYwxpkpeR0mN8zuQ2hCcJ6iBF3iMMcY3nmdPE5HzgRPcpzNU9St/QvLPrt5HlhSMMaYqXofO/jdwJ7Dcvd0pIo/6GZgfrE3BGGPC81pSOBsY7vZEQkTeAhbiDKfdYFjvI2OMCW9vLuNqHfK4QU5sbNcpGGNMeF5LCo8CC0VkOiA4bQv3+haVT6z3kTHGhOe199EHIjIDONJddI+qbvEtKp8E2xSsoGCMMVXz2tB8AbBTVb9Q1S+AIhH5rb+h1byA9T4yxpiwvLYp/F1Vc4NPVDUH+Ls/IfnH2hSMMSY8r0mhqvU8X+NQX5Rb7yNjjAnLa1JIEJEnRaSve3sSSPQzMD+oXadgjDFheU0KtwMlwIfARKCI3edWaBCs95ExxoTntfdRAQ2wC2plFW0KlhSMMaZKB9UcZNbQbIwx4R1kScG5j7akYIwxVQqbFETkMfd+bO2E469dvY/qOBBjjKmnIpUUznZnW2tQA99Vx9oUjDEmvEhJ4VsgGxgaOuOa15nXRORMEVklIskiUmVDtYicJCKLRCRJRGbuw3fwLBBw7q1NwRhjqhY2Kajq3araGvhaVVuqalzofbhtRSQaeAE4CxgMXCYigyut0xp4EThfVQ8FfK2m2jXMhZ+fYowxDZfXLqljRKQTuwbE+1VVMyNsNgpIVtV1ACIyERiDM0lP0OXAZFVNcT8nY2+C31vlNh2nMcaE5XVAvLHAPJwz+UuAeSJycYTNugGpIc/T3GWhBgBtRGSGiCSKyFXewt43wTmarfeRMcZUzev4RfcDRwbP5EWkAzAN+LgGPn8EcArQFJgjInNVdXXoSiJyE3ATQM+ePff5w8qtTcEYY8LyPCBepaqdbR62TQd6hDzv7i4LlQZMVdUCVc0CZgHDKr+Rqk5Q1ZGqOrJDhw4eQ97Trt5H+/wWxhhzQPN6ePxWRKaKyDUicg3wNTAlwjbzgf4iEi8iMcClwBeV1vkcOE5EGolIM+AoYIX38PeOzdFsjDHheW1ovltELgSOcxdNUNVPI2xTJiK3AVOBaOB1VU0SkfHu6y+r6goR+RZYAgSA/6nqsn39MpFUXNFs1ykYY0yVPM+JoKqTgcl78+aqOoVKJQpVfbnS88eBx/fmffdVudoVzcYYE85BVbuuNiCeMcaEdVAlBZt5zRhjwvNcfeQ2Fg8CFFilqiW+ReUTa1MwxpjwPCUFETkHeBlYCwgQLyI3q+o3fgZX04K9j6ygYIwxVfNaUngCOFlVkwFEpC9Ot9SGlRTUpuM0xphwvLYp5AcTgmsdkO9DPL4qt4ZmY4wJK2xJwb02ASBBRKYAH+G0KYzFuTitQQm2KVhSMMaYqkWqPjov5PFW4ET3cSbOWEUNSsBmXjPGmLDCJgVVvba2AqkN1qZgjDHhee191AG4Eegduo2qXudPWP4oD9h8CsYYE47X3kefAz/hDJdd7l84/rKSgjHGhOc1KTRT1Xt8jaQW7Gports4jDGmvvLaJfUrETnb10hqgQ1zYYwx4XlNCnfiJIZCEckTkXwRyfMzMD/YgHjGGBOe1/kU4vwOpDYEp+O0NgVjjKla2JKCiPSO8LqISPeaDMhPAZtPwRhjwopUUnhcRKJweh8l4ly01gToB5wMnAL8HWeu5XovoIqIdUk1xpjqRLp4bayIDAauAK4DugA7ceZRngI8oqpFvkdZQwKq1p5gjDFhRGxTUNXlwF9rIRbflQcg2pKCMcZU66CaeU3d6iNjjDFVO6iSQnlAreeRMcaEcVAlhYDaNQrGGBOOp6Tgdj29UkT+5j7vKSKj/A2t5jkNzXUdhTHG1F9eSwovAscAl7nP84EXfInIRwFVoiwrGGNMtbwOiHeUqh4hIgsBVDVbRGJ8jMsX5QG13kfGGBOG15JCqYhE40zFGZxfIeBbVD4JqF24Zowx4XhNCs8CnwIdReQR4GfgX75F5ZNAQIk+qJrWjTFm73gdEO89EUnEGdZCgN+q6gpfI/OBXdFsjDHhRUwKbrVRkqoOAlb6H5J/yi0pGGNMWBErU1S1HFglIj1rIR5fqUKUVR8ZY0y1vPY+agMkicg8oCC4UFXP9yUqn1jvI2OMCc9rUnjA1yhqibUpGGNMeF4bmmeKSCfgSHfRPFXN8C8sf9jFa8YYE57XYS4uAeYBY4FLgF9F5GI/A/NDIGCzrhljTDheq4/+ChwZLB24F69NAz72KzA/WO8jY4wJz2tfnKhK1UXb9mLbekMtKRhjTFheSwrfishU4AP3+TjgG39C8o/Np2CMMeF5OttX1buBV4Ch7m2Cqv4l0nYicqaIrBKRZBG5t4rXTxKRXBFZ5N7+trdfYG848yn4+QnGGNOweSopiEg8MEVVJ7vPm4pIb1XdEGabaJzhtU8D0oD5IvKFO+dzqJ9U9dx9in4vWe8jY4wJz2u7wCR2HxW13F0WziggWVXXqWoJMBEYs/ch1hy7TsEYY8LzmhQauQd2ANzHkeZT6AakhjxPc5dVNlpElojINyJyqMd49old0WyMMeF5TQqZIlIxpIWIjAGyauDzFwA9VXUo8BzwWVUrichNIpIgIgmZmZn7/GHOfAr7vLkxxhzwvCaF8cD/iUiKiKQC9wA3R9gmHegR8ry7u6yCquap6g738RSgsYi0r/xGqjpBVUeq6sgOHTp4DHlPAet9ZIwxYXkd5mItcLSItHCf7/Cw2Xygv9tInQ5cClweuoKIdAa2qqqKyCicJLVtL+LfK9amYIwx4Xkd5uJOEWmJM0Lq0yKyQEROD7eNqpYBtwFTgRXAR6qaJCLjRWS8u9rFwDIRWYwzu9ulqqr7+mUiKVes95ExxoTh9eK161T1GRE5A2gH/A54B/gu3EZuldCUSsteDnn8PPD8XkW8H5wrmmvr04wxpuHx2qYQPJSeDbytqkkhyxoM631kjDHheU0KiSLyHU5SmCoicex+3UKD4PQ+sqRgjDHV8Vp9dD0wHFinqjtFpB1wrX9h+cPpfVTXURhjTP3ltfdRAOeaguDzbfjYS8gv1vvIGGPCO6jOm8tt7CNjjAnroEoKqlhJwRhjwvDaphAc9bRT6DaqmuJHUH5xeh/VdRTGGFN/eR06+3bg78BWdvU6Upy5FRoMa1MwxpjwvJYU7gQGug3MDVYgYG0KxhgTjtc2hVQg189AaoPNvGaMMeF5LSmsA2aIyNdAcXChqj7pS1Q+KVcbJdUYY8LxmhRS3FsMkSfXqbfU2hSMMSYsrxevPQiwl0Nn1zvlAUsKxhgTjtehs4eIyEIgCUgSkUS/p870Q0Cx6iNjjAnDa0PzBOBPqtpLVXsBfwZe9S8sfwQCatNxGmNMGF6TQnNVnR58oqozgOa+ROSjgNrQ2cYYE47n3kci8gDOxDoAV+L0SGpQbOwjY4wJz2tJ4TqgAzDZvXVwlzUoARv7yBhjwvLa+ygbuMPnWHwXCNh0nMYYE07YpCAiT6vqH0TkS5yxjnajquf7FpkPAnbxmjHGhBWppBBsQ/iv34H4TVVtOk5jjIkgbFJQ1UT34XBVfSb0NRG5E5jpV2A1Td1yjvU+MsaY6nltaL66imXX1GAcvit3s4LVHhljTPUitSlcBlwOxIvIFyEvxQHb/QyspgWCScGygjHGVCtSm8JsYDPQHngiZHk+sMSvoPwQcKcGsi6pxhhTvUhtChuBjcAxtROOf4IlheiDalZqY4zZO14HxDtaROaLyA4RKRGRchHJ8zu4mrSrTcFKCsYYUx2v583PA5cBa4CmwA3AC34F5Qe16iNjjInIc2WKqiYD0aparqpvAGf6F1bNs95HxhgTmdcB8XaKSAywSET+g9P43KBq53e1KVhWMMaY6ng9sP8OiAZuAwqAHsBFfgXlh0DASQp2RbMxxlTP64B4G92HhcCD/oXjn0DwimYrKRhjTLUiXby2lCoGwgtS1aE1HpFPrE3BGGMii1RSONe9v9W9D51kp9pkUR8Fq4+s95ExxlTPy8VriMhpqnp4yEv3iMgC4F4/g6tJAbtOwRhjIvLa0CwicmzIk9F7sW29YG0KxhgTmdcuqdcDr4tIK0CAbBrYdJzlFb2P6jgQY4ypxzyd7atqoqoOA4YBQ1V1uKouiLSdiJwpIqtEJFlEqq1qEpEjRaRMRC72HvreUbtOwRhjIorU++hKVX1XRP5UaTkAqvpkmG2jcYbCOA1IA+aLyBequryK9R4Dvtunb+CRjX1kjDGRRSopNHfv46q5hTMKSFbVdapaAkwExlSx3u3AJ0CG16D3hQ2dbYwxkUXqffSKe78vF6x1A1JDnqcBR4WuICLdgAuAk4Ej9+EzPAvYdQrGGBNRpOqjZ8O9rqp37OfnPw3co6qBcMNPiMhNwE0APXv23KcPsrGPjDEmski9jxL3473TccZICuruLgs1EpjoJoT2wNkiUqaqn4WupKoTgAkAI0eO3KeL5srt4jVjjIkoUvXRW/vx3vOB/iISj5MMLsWZ7zn0/eODj0XkTeCrygmhpgSvU7A5mo0xpnqerlMQkQ7APcBgoElwuar+prptVLVMRG4DpuKMsPq6qiaJyHj39Zf3J/C9ZW0KxhgTmdeL194DPgTOAcYDVwOZkTZS1SnAlErLqkwGqnqNx1j2SXDso2irPjLGmGp5Haqinaq+BpSq6kxVvQ6otpRQHwWvU7D5FIwxpnpeSwql7v1mETkH2AS09Sckf6iNfWSMMRF5TQoPu+Me/Rl4DmgJ/NG3qHywq/dRHQdijDH1mNek8Kuq5gK5OBeaNTgVDc2WFYwxplpe2xR+EZHvROR6EWnja0Q+sfkUjDEmMq+jpA4A7gcOBRJF5CsRudLXyGpYcOwj631kjDHV8zxRjqrOU9U/4Qx0tx3Ynwvbat2u3kd1HIgxxtRjnpKCiLQUkatF5BtgNrAZJzk0GDafgjHGROa1oXkx8BnwkKrO8TEe35Tb0NnGGBOR16TQR4On2g1U51axnHNYF1o29fqVjTHm4OPpCNnQEwLAiF5tGdGrQV1vZ4wxtc5zQ7MxxpgDnyUFY4wxFbz2PvqP2wOpsYj8ICKZDe06BWOMMZF5LSmcrqp5wLnABqAfcLdfQRljjKkbXpNCsEH6HGCSOw6SMcaYA4zX/plfichKoBC4xZ2Jrci/sIwxxtQFr2Mf3QuMBkaqailQAIzxMzBjjDG1z2tD81icWdfKReR+4F2gq6+RGWOMqXXi5bo0EVmiqkNF5DjgYeBx4G+qepTfAVYRSyawcR82bQ9k1XA4+6s+xgT1My6LyZv6GBPUz7gOtph6qWqHSCt5bVMod+/PASao6tci8vA+h7YfvHypqohIgqqOrOl49kd9jAnqZ1wWkzf1MSaon3FZTFXz2vsoXUReAcYBU0Qkdi+2NcYY00B4PbBfAkwFzlDVHKAtdp2CMcYccLz2PtoJrAXOEJHbgI6q+p2vkdW8CXUdQBXqY0xQP+OymLypjzFB/YzLYqqC14bmO4Ebgcnuogtw2hae8zE2Y4wxtcxz7yPgGFUtcJ83B+ao6lCf4zPGGFOLvLYpCLt6IOE+bhBTmInImSKySkSSReTeOoyjh4hMF5HlIpLklr4QkX+ISLqILHJvZ9dyXBtEZKn72QnusrYi8r2IrHHv29RiPAND9sUiEckTkT/UxX4SkddFJENEloUsq3bfiMh97u9slYicUYsxPS4iK0VkiYh8KiKt3eW9RaQwZJ+9XIsxVfv3qo39FCauD0Ni2iAii9zltbWvqjsO1OnvajeqGvEG/AlnSs5/uLdFwB+8bFuXNyAapy2kDxDjfofBdRRLF+AI93EcsBoY7O7Pu+pwH20A2lda9h/gXvfxvcBjdfj32wL0qov9BJwAHAEsi7Rv3L/lYiAWiHd/d9G1FNPpQCP38WMhMfUOXa+W91OVf6/a2k/VxVXp9SdwrreqzX1V3XGgTn9XoTevDc1PAtcC293btar6tJdt69goIFlV16lqCTCROhqeQ1U3q+oC93E+sALoVhexeDAGeMt9/Bbw2zqK4xRgraruy8WK+01VZ+H83kNVt2/GABNVtVhV1wPJOL8/32NS1e9Utcx9OhfoXtOfu7cxhVEr+ylSXCIiOL0qP/Djs8PEVN1xoE5/V6EiJgURiRaRlaq6QFWfdW8L/QyqBnUDUkOep1EPDsQi0hs4HPjVXXS7W/R/vTaralwKTBORRBG5yV3WSVU3u4+3AJ1qOaagS9n9n7Yu91NQdfumvvzWrgO+CXke71aHzBSR42s5lqr+XvVlPx0PbFXVNSHLanVfVToO1JvfVcSkoKrlwCoR6elnIAcLEWkBfIJT/ZYHvIRTvTUc2IxTpK1Nx6nqcOAs4FYROSH0RXXKsLU+R7eIxADnA5PcRXW9n/ZQV/umOiLyV6AMeM9dtBno6f59/wS8LyItaymcevf3quQydj/hqNV9VcVxoEJd/668DnPRBkgSkXk4I6QCoKrn+xJVzUkHeoQ87+4uqxMi0hjnh/Ceqk4GUNWtIa+/CnxVmzGparp7nyEin+IUTbeKSBdV3SwiXYCM2ozJdRawILh/6no/hahu39Tpb01ErsGZBOsU96CCqhYDxe7jRBFZCwwAEvyOJ8zfq87/J0WkEXAhMCK4rDb3VVXHAerR78pr76MHcH5wD+Fk/OCtvpsP9BeRePfM81Lgi7oIxK3DfA1Y4bbRBJd3CVntAmBZ5W19jKm5iMQFH+M0WC7D2UdXu6tdDXxeWzGF2O1Mri73UyXV7ZsvgEtFJFZE4oH+wLzaCEhEzgT+ApyvzoWmweUdRCTafdzHjWldLcVU3d+rzvZTiFOBlaqaFlxQW/uquuMA9el3FaGlvB9wbBXLjwP6+t1SXxM34GycFv61wF/rMI7jcIqES3B6by1yY3sHWOou/wLoUosx9cHp2bAYSAruH6Ad8AOwBpgGtK3lfdUc2Aa0CllW6/sJJyltBkpx6nKvD7dvgL+6v7NVwFm1GFMyTr1z8Hf1srvuRe7fdRGwADivFmOq9u9VG/upurjc5W8C4yutW1v7qrrjQJ3+rkJvYS9eE5GvgPtUdWml5YcB/1LV86rd2BhjTIMTqfqoU+WEAOAu6+1LRMYYY+pMpKTQOsxrTWsyEGOMMXUvUlJIEJEbKy8UkRuARH9CMsYYU1citSl0Aj4FStiVBEbiDBlxgapu8T1CY4wxtcbrKKknA0Pcp0mq+qOvURljjKkTXsc+mq6qz7k3SwgHORFREXki5PldIvKPGnrvN0Xk4pp4rwifM1ZEVojI9Cpee9wdwfLxfXjf4VLLI93uLRHZsY/b/VZEBtfW55m6YfMsm31RDFwoIu3rOpBQ7pWqXl0P3KiqJ1fx2k3AUFXdlylnh+P0O/dMHA3hf/G3OKN2mgNYQ/ghmvqnDGfawD9WfqHymX7wLFFETnIHGvtcRNaJyL9F5AoRmSfOfA59Q97mVBFJEJHVInKuu320ewY/3x1k7eaQ9/1JRL4AllcRz2Xu+y8TkcfcZX/DuYjotcqlAfd9WgCJIjLOvdL1E/dz54vIse56o0RkjogsFJHZ4swDEYNz1f84cQZWGyfOvAJ3hbz/MnHG7u8tzvj4b+Nc7dtDRE5333OBiEwSZ3wc3H213P3e/63iO54ou+YBWBhylfrdIfvrwar+kNWtIyJXucsWi8g7IjIaZyyqx93P6evevhVnMMWfRGSQu228+z2WisjDVX2uqcf8vjrObgfeDdgBtMSZi6EVcBfwD/e1N4GLQ9d1708CcnDGk4/FGb/lQfe1O4GnQ7b/FueEpT/OlahNcM7e73fXicUZkybefd8CIL6KOLsCKUAHnHG+fgR+6742AxhZ3fcLefw+zqCBAD1xhifA/f7BOQxOBT5xH18DPB+y/T8ImVcAJwH0dm8B4Gh3eXtgFtDcfX4P8DecK11Xsav9r3UV8X6JO/IATkJrhDNkyQScybCicMYeOqHS36TKdYBDcUYBaO+u17aav+0PQH/38VHAj+7jL4Cr3Me3hu5Pu9X/294Ut42poKp57lnuHUChx83mqzs8sDgDjn3nLl8KhFbjfKSqAWCNiKwDBuEcwIaGlEJa4SSNEmCeOmPNV3YkMENVM93PfA/noPeZx3jBOeAPFqmYaLClewbfCnhLRPrjDFvQeC/eM2ijqs51Hx+NUzXzi/tZMcAcIBcowinVfEXVAwH+Ajzpfr/JqpomIqfj7LPgMPctcPbXrJDtqltnGDBJVbMAVHWPOQncfTAamBSyb2Ld+2Nxho0AZ7iLxyLuCVNvWFIw++NpnHFi3ghZVoZbLenWk8eEvFYc8jgQ8jzA7r/Fyl3iFOds9nZVnRr6goicRMjIvT6IwjmbL6r0uc8D01X1AnHGxZ9RzfYV+8PVJORxaNwCfK+ql1V+AxEZhTPh0MXAbcBvQl9X1X+LyNc4bRm/iDNlowCPquorYb5bleuIyO1htgmKAnLUGWq6KvVmSHGzd6xNwewz9wzyI5xG26AN7BqS+Hz27Qx6rIhEue0MfXCqT6YCt4gz7DAiMkCckV3DmQecKCLtxRkB8zJg5l7G8h1QcZAUkeBBsBW7hjC+JmT9fJxpFoM24EwJiYgcgVPlVZW5wLEi0s9dt7n7HVvgDAw4BacNZ1jlDUWkr6ouVdXHcEYGHoSzv64LaZfoJiIdK21a3To/4vwN2rnL21b+burMAbBeRMa664iIBGP7BWdEYoArqvm+pp6ypGD21xM49eFBr+IciBcDx7BvZ/EpOAf0b3BGsywC/ofTkLxAnInYXyFCSdetqroXmI4zEmyiqu7tMOB3ACPdRtflwHh3+X+AR0VkYaU4puNUNy0SkXE44+a3FZEknLP81dXEmomTXD4QkSU4VUeDcA7CX7nLfsaZAKayP7gN2EtwRgT9RlW/w2kPmSMiS4GP2T1ZUd06qpoEPALMdP+OwSGeJwJ3u43ZfXEO+Ne76ySxa6rbO3EmbFpKPZjp0OwdTxevGWOMOThYScEYY0wFSwrGGGMqWFIwxhhTwZKCMcaYCpYUjDHGVLCkYIwxpoIlBWOMMRUsKRhjjKnw/3pXuQC8xhhJAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1239a0150>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "rfecv = RFECV(estimator=clfGB, step=5, cv=StratifiedKFold(3),\n",
    "              scoring='accuracy')\n",
    "rfecv.fit(goldX, goldY)\n",
    "print(\"Optimal number of features : %d\" % rfecv.n_features_)\n",
    "\n",
    "# Plot number of features VS. cross-validation scores\n",
    "plt.figure()\n",
    "plt.xlabel(\"Number of features selected\")\n",
    "plt.ylabel(\"Cross validation score (nb of correct classifications)\")\n",
    "plt.plot(range(1, len(rfecv.grid_scores_) + 1), rfecv.grid_scores_)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Has Image', 0.016838410310518095),\n",
       " ('Total Likes', 0.0034657724235323335),\n",
       " ('Total Shares', 0.004530721982439977),\n",
       " ('Total Comments', 0.0025972116236845743),\n",
       " ('num_words', 0.0062741749875445925),\n",
       " ('avg_word_length', 0.003194262751981193),\n",
       " ('has_url', 0.0005400113982714201),\n",
       " (u'NGRAM_abril', 0.0007601453442567413),\n",
       " (u'NGRAM_abrir', 0.0011025787502268296),\n",
       " (u'NGRAM_acabar', 0.006518716663803417),\n",
       " (u'NGRAM_agradecer', 0.0028866469748862113),\n",
       " (u'NGRAM_alguien', 0.0021118701555809323),\n",
       " (u'NGRAM_altura', 0.0006745862298298491),\n",
       " (u'NGRAM_amigo', 0.0026603752336617314),\n",
       " (u'NGRAM_ana', 0.0011328974880266509),\n",
       " (u'NGRAM_andar', 0.0017986347975011635),\n",
       " (u'NGRAM_apoyar', 0.0014159918115176676),\n",
       " (u'NGRAM_armar', 0.0035596531314213377),\n",
       " (u'NGRAM_asaltar', 0.00971991308872941),\n",
       " (u'NGRAM_auto', 0.0031469735527564478),\n",
       " (u'NGRAM_autoridad', 0.003226719157659515),\n",
       " (u'NGRAM_avenir', 0.000353862272516824),\n",
       " (u'NGRAM_ayudar', 0.005253362061410063),\n",
       " (u'NGRAM_bajar', 0.0012236299528093286),\n",
       " (u'NGRAM_blanco', 0.001890687192806011),\n",
       " (u'NGRAM_blico', 0.001593168490872212),\n",
       " (u'NGRAM_buenas', 0.0013619216923043806),\n",
       " (u'NGRAM_buenas noche', 0.0006341092481359583),\n",
       " (u'NGRAM_buscar', 0.002887147155200446),\n",
       " (u'NGRAM_caer', 0.0021675852742313985),\n",
       " (u'NGRAM_callar', 0.0025979044565180302),\n",
       " (u'NGRAM_calle', 0.0010932212101674612),\n",
       " (u'NGRAM_camioneta', 0.011680712173077604),\n",
       " (u'NGRAM_carro', 0.004117605723296031),\n",
       " (u'NGRAM_casar', 0.0068149807423986235),\n",
       " (u'NGRAM_celular', 0.0026024200269217923),\n",
       " (u'NGRAM_cerrar', 0.002590195400307432),\n",
       " (u'NGRAM_ciudad', 0.0004574907181750505),\n",
       " (u'NGRAM_coacalco', 0.005068850711956205),\n",
       " (u'NGRAM_cobrar', 0.0010434953919661344),\n",
       " (u'NGRAM_colonia', 0.0013888477215654865),\n",
       " (u'NGRAM_color', 0.008969153818070638),\n",
       " (u'NGRAM_como', 0.0006874371570081109),\n",
       " (u'NGRAM_compartir', 0.009356373443732684),\n",
       " (u'NGRAM_comunicarse', 0.0022717318950034046),\n",
       " (u'NGRAM_comunidad', 0.00053374590040927),\n",
       " (u'NGRAM_con', 0.002328376258781074),\n",
       " (u'NGRAM_conocer', 0.0006987106166369615),\n",
       " (u'NGRAM_ctima', 0.0006517792443444197),\n",
       " (u'NGRAM_cuidar', 0.0019058423364395386),\n",
       " (u'NGRAM_culo', 0.002273545240709277),\n",
       " (u'NGRAM_dar', 0.0016026918704255325),\n",
       " (u'NGRAM_decir', 0.0004195998484603067),\n",
       " (u'NGRAM_dejar', 0.004247080107596887),\n",
       " (u'NGRAM_del', 0.0007242417002620813),\n",
       " (u'NGRAM_denunciar', 0.009996274222528802),\n",
       " (u'NGRAM_derecho', 0.0010693729646470617),\n",
       " (u'NGRAM_desaparecida', 0.0005592857026244445),\n",
       " (u'NGRAM_detener', 0.0013496606653316124),\n",
       " (u'NGRAM_difundir', 0.0016255660093599724),\n",
       " (u'NGRAM_dinero', 0.006028582479041442),\n",
       " (u'NGRAM_ecatepec', 0.005656112130663587),\n",
       " (u'NGRAM_edad', 0.007347094839465215),\n",
       " (u'NGRAM_encontrar', 0.0032051863057584364),\n",
       " (u'NGRAM_entrar', 0.0029626998338735905),\n",
       " (u'NGRAM_escuela', 0.0009258019007392702),\n",
       " (u'NGRAM_esperar', 0.0019523794483626187),\n",
       " (u'NGRAM_esta', 0.0032080093154642007),\n",
       " (u'NGRAM_estado', 0.0008046439346742759),\n",
       " (u'NGRAM_estado xico', 0.0008151297182354179),\n",
       " (u'NGRAM_estar', 0.00023427751582663476),\n",
       " (u'NGRAM_este', 0.001276771910052322),\n",
       " (u'NGRAM_esto', 0.00020829910938981264),\n",
       " (u'NGRAM_facebook', 0.0006250390794644849),\n",
       " (u'NGRAM_familia', 0.0020347770582152765),\n",
       " (u'NGRAM_familiar', 0.003553659056401595),\n",
       " (u'NGRAM_favor', 0.008593120856417268),\n",
       " (u'NGRAM_febrero', 0.000992177289919124),\n",
       " (u'NGRAM_formar', 0.0008594067719962994),\n",
       " (u'NGRAM_foto', 0.0029535130224292343),\n",
       " (u'NGRAM_frente', 0.0005369166447043158),\n",
       " (u'NGRAM_gente', 0.002481646746996488),\n",
       " (u'NGRAM_goo', 0.0008030143734762322),\n",
       " (u'NGRAM_gracia', 0.0029991795280401388),\n",
       " (u'NGRAM_gracias', 0.0011526011223102668),\n",
       " (u'NGRAM_gustar', 0.0006473970577747852),\n",
       " (u'NGRAM_hacer', 0.000929986997855377),\n",
       " (u'NGRAM_hash', 0.007604959571711534),\n",
       " (u'NGRAM_hijo', 0.002006298701279366),\n",
       " (u'NGRAM_hola', 0.003752966535687941),\n",
       " (u'NGRAM_hora', 0.0006165035256969912),\n",
       " (u'NGRAM_hoy', 0.0010517014964917424),\n",
       " (u'NGRAM_https', 0.00027802593741211386),\n",
       " (u'NGRAM_https goo', 0.001097193363547026),\n",
       " (u'NGRAM_imagen', 0.0009891913128462436),\n",
       " (u'NGRAM_informaci', 0.004910099490013427),\n",
       " (u'NGRAM_informar', 0.003023445074573531),\n",
       " (u'NGRAM_julio', 0.001211415900101226),\n",
       " (u'NGRAM_las', 0.004014823082066459),\n",
       " (u'NGRAM_les', 0.0010112137485656259),\n",
       " (u'NGRAM_llamar', 0.00761798252865199),\n",
       " (u'NGRAM_llegar', 0.007877252578692063),\n",
       " (u'NGRAM_llevar', 0.0007323389500872441),\n",
       " (u'NGRAM_localizar', 0.006956866789784788),\n",
       " (u'NGRAM_los', 0.0006751191062384871),\n",
       " (u'NGRAM_madre', 0.009597646674521792),\n",
       " (u'NGRAM_mandar', 0.0015176925334209398),\n",
       " (u'NGRAM_marzo', 0.0029657906547526708),\n",
       " (u'NGRAM_menor', 0.0012013398430648116),\n",
       " (u'NGRAM_mensaje', 0.0031727874079695217),\n",
       " (u'NGRAM_mero', 0.0004957448546485784),\n",
       " (u'NGRAM_mesar', 0.0033783438758954744),\n",
       " (u'NGRAM_metro', 0.0015161057564266637),\n",
       " (u'NGRAM_mil', 0.0009636128447394728),\n",
       " (u'NGRAM_municipal', 0.0029803089004393018),\n",
       " (u'NGRAM_municipio', 0.0016452736162382122),\n",
       " (u'NGRAM_necesitar', 0.0003457716078023073),\n",
       " (u'NGRAM_negro', 0.001759077777385867),\n",
       " (u'NGRAM_nezahualc', 0.0004439363262513331),\n",
       " (u'NGRAM_nezahualc yotl', 0.00041440623199012766),\n",
       " (u'NGRAM_nico', 0.0019040849479128113),\n",
       " (u'NGRAM_nimo', 0.0005052153766849571),\n",
       " (u'NGRAM_noche', 0.0021570515979620897),\n",
       " (u'NGRAM_nombrar', 0.00333950488515185),\n",
       " (u'NGRAM_nos', 0.0019086708456139876),\n",
       " (u'NGRAM_padre', 0.00038276118105946876),\n",
       " (u'NGRAM_pagar', 0.003947815101680442),\n",
       " (u'NGRAM_para', 0.0007505559161766369),\n",
       " (u'NGRAM_pasar', 0.0019266977568793194),\n",
       " (u'NGRAM_pedir', 0.0029125612769760854),\n",
       " (u'NGRAM_perder', 0.0004805733480968344),\n",
       " (u'NGRAM_perrito', 0.0005829515559557416),\n",
       " (u'NGRAM_personar', 0.011003151673432717),\n",
       " (u'NGRAM_peso', 0.0016535896890312793),\n",
       " (u'NGRAM_placa', 0.022015589910311033),\n",
       " (u'NGRAM_poder', 0.004728511780164324),\n",
       " (u'NGRAM_polic', 0.0028343061752367253),\n",
       " (u'NGRAM_poner', 0.0004892176255307749),\n",
       " (u'NGRAM_por', 0.006078882681697459),\n",
       " (u'NGRAM_por favor', 0.006082155820801314),\n",
       " (u'NGRAM_porfavor', 0.001314906502729234),\n",
       " (u'NGRAM_pri', 0.0004627008564975169),\n",
       " (u'NGRAM_publicar', 0.0038358594853898646),\n",
       " (u'NGRAM_puerta', 0.001566380186063996),\n",
       " (u'NGRAM_que', 0.0022075801043667043),\n",
       " (u'NGRAM_quedar', 0.0016396032394537397),\n",
       " (u'NGRAM_querer', 0.004026394184136394),\n",
       " (u'NGRAM_recibir', 0.0016221247701199812),\n",
       " (u'NGRAM_regresar', 0.005359187815131482),\n",
       " (u'NGRAM_reportar', 0.0011611404071693592),\n",
       " (u'NGRAM_rez', 0.0003103140365213281),\n",
       " (u'NGRAM_robar', 0.04219805472828082),\n",
       " (u'NGRAM_saber', 0.0006510091819697736),\n",
       " (u'NGRAM_sacar', 0.002454838754359662),\n",
       " (u'NGRAM_salir', 0.0018276071916824455),\n",
       " (u'NGRAM_san', 0.0005436113281015539),\n",
       " (u'NGRAM_seguir', 0.0009198537444250451),\n",
       " (u'NGRAM_seguridad', 0.0006383648642344583),\n",
       " (u'NGRAM_septiembre', 0.00046128407286958706),\n",
       " (u'NGRAM_ser', 0.0024340980004913014),\n",
       " (u'NGRAM_solicitar', 0.0009649130907275219),\n",
       " (u'NGRAM_subir', 0.0031997705339827077),\n",
       " (u'NGRAM_tarde', 0.0014642184847423446),\n",
       " (u'NGRAM_tel', 0.0011375939057633973),\n",
       " (u'NGRAM_tener', 0.001343143754959836),\n",
       " (u'NGRAM_tipo', 0.00332177950597234),\n",
       " (u'NGRAM_tomar', 0.0006662006914215479),\n",
       " (u'NGRAM_traer', 0.0024522085434517983),\n",
       " (u'NGRAM_transportar', 0.0011803492451844268),\n",
       " (u'NGRAM_ubicar', 0.0004006507201710303),\n",
       " (u'NGRAM_una', 0.0006697294710040922),\n",
       " (u'NGRAM_unidad', 0.0010174797780259223),\n",
       " (u'NGRAM_valle', 0.0003101554253462939),\n",
       " (u'NGRAM_vecino', 0.0007537223100317847),\n",
       " (u'NGRAM_veh', 0.001207603274206577),\n",
       " (u'NGRAM_veh culo', 0.00136496125163774),\n",
       " (u'NGRAM_venir', 0.0008327013003312183),\n",
       " (u'NGRAM_ver', 0.002459008839296978),\n",
       " (u'NGRAM_vestir', 0.004810640129786317),\n",
       " (u'NGRAM_vida', 0.0022499917479754648),\n",
       " (u'NGRAM_violencia', 0.007406129101101895),\n",
       " (u'NGRAM_vivir', 0.0008775139177557255),\n",
       " (u'NGRAM_volver', 0.0009240174204312569),\n",
       " (u'NGRAM_xico', 0.0009745519397881715),\n",
       " (u'NGRAM_yotl', 0.0005848218387502586),\n",
       " (u'NGRAM_zona', 0.0005964531207778273),\n",
       " (u'COMMENT_abrir', 0.00010566698186688562),\n",
       " (u'COMMENT_abusar', 0.007143173230412073),\n",
       " (u'COMMENT_acto', 0.0019430081049141704),\n",
       " (u'COMMENT_actuar', 0.0005404087635462313),\n",
       " (u'COMMENT_agan', 0.0011538698285703131),\n",
       " (u'COMMENT_agarrar', 0.0012838516248967974),\n",
       " (u'COMMENT_ahora', 0.0006349377048610324),\n",
       " (u'COMMENT_ala', 0.0010113973890644972),\n",
       " (u'COMMENT_alcanzar', 0.002154754141064564),\n",
       " (u'COMMENT_alguien', 0.0017494892178261364),\n",
       " (u'COMMENT_alos', 0.006930533383871011),\n",
       " (u'COMMENT_alto', 0.0008337427851130208),\n",
       " (u'COMMENT_ami', 0.00041384189088862516),\n",
       " (u'COMMENT_amigar', 0.006484431628520705),\n",
       " (u'COMMENT_amor', 0.0004784545329546741),\n",
       " (u'COMMENT_ana', 0.000383114811077513),\n",
       " (u'COMMENT_andar', 0.00504438321904323),\n",
       " (u'COMMENT_aparecer', 0.003705627242480189),\n",
       " (u'COMMENT_apoyar', 0.00033148972022304627),\n",
       " (u'COMMENT_armar', 0.001340955274396902),\n",
       " (u'COMMENT_asaltar', 0.000916700785160435),\n",
       " (u'COMMENT_asi', 0.007117512766805526),\n",
       " (u'COMMENT_auto', 0.00027335211892643536),\n",
       " (u'COMMENT_autoridad', 0.006655854701490001),\n",
       " (u'COMMENT_avenir', 0.0006926065302926613),\n",
       " (u'COMMENT_aver', 0.00020402946068541096),\n",
       " (u'COMMENT_ayudar', 0.0007435336642535323),\n",
       " (u'COMMENT_bien', 0.0008080410267951661),\n",
       " (u'COMMENT_blico', 0.000691988904925344),\n",
       " (u'COMMENT_bueno', 0.0003415744240339968),\n",
       " (u'COMMENT_buscar', 0.004315896046255389),\n",
       " (u'COMMENT_cabr', 0.00020553791700795782),\n",
       " (u'COMMENT_caer', 0.0012973370785692496),\n",
       " (u'COMMENT_callar', 0.0007841510086515355),\n",
       " (u'COMMENT_calle', 0.0003604068672179453),\n",
       " (u'COMMENT_cargar', 0.0023630477107089605),\n",
       " (u'COMMENT_caro', 0.003944137985150488),\n",
       " (u'COMMENT_carro', 0.0012851148307142827),\n",
       " (u'COMMENT_casar', 0.0013470491799514766),\n",
       " (u'COMMENT_celular', 0.0009658589478697363),\n",
       " (u'COMMENT_chalco', 0.0007280541442383881),\n",
       " (u'COMMENT_chavo', 0.0005172185704877527),\n",
       " (u'COMMENT_chico', 0.0006812114405583157),\n",
       " (u'COMMENT_chimalhuacan', 0.0007143954783310259),\n",
       " (u'COMMENT_chingar', 0.0012960111208175063),\n",
       " (u'COMMENT_chingar madre', 0.0011048124752519152),\n",
       " (u'COMMENT_chisme', 0.0009687291819028113),\n",
       " (u'COMMENT_chismoso', 0.0008451709001188691),\n",
       " (u'COMMENT_comentar', 0.0008061319890631436),\n",
       " (u'COMMENT_comentario', 0.0006787100107780158),\n",
       " (u'COMMENT_comer', 0.00046068847080634416),\n",
       " (u'COMMENT_como', 0.0010455322104609092),\n",
       " (u'COMMENT_compartir', 0.0037762948491952162),\n",
       " (u'COMMENT_comprar', 0.002851312943768978),\n",
       " (u'COMMENT_conocer', 0.0023557268971839407),\n",
       " (u'COMMENT_contrario', 0.0004254229510410881),\n",
       " (u'COMMENT_corrupto', 0.0009195077272318965),\n",
       " (u'COMMENT_creer', 0.0021727213729513615),\n",
       " (u'COMMENT_cuidar', 0.00202501571516423),\n",
       " (u'COMMENT_culero', 0.0012053956487073489),\n",
       " (u'COMMENT_culo', 0.0025729415949345222),\n",
       " (u'COMMENT_culpar', 0.0015479981412925697),\n",
       " (u'COMMENT_dar', 0.0004193883698276715),\n",
       " (u'COMMENT_darle', 0.0007146574750352494),\n",
       " (u'COMMENT_deber', 0.0016354350052147071),\n",
       " (u'COMMENT_decir', 0.0025419416926919965),\n",
       " (u'COMMENT_defender', 0.02120253859970324),\n",
       " (u'COMMENT_dejar', 0.0030690849847508765),\n",
       " (u'COMMENT_del', 0.0008522337792919815),\n",
       " (u'COMMENT_delincuente', 0.0006231196570481586),\n",
       " (u'COMMENT_demas', 0.00019693114172290337),\n",
       " (u'COMMENT_denunciar', 0.0027128090588564384),\n",
       " (u'COMMENT_derecho', 0.0008611861385371037),\n",
       " (u'COMMENT_desgraciadamente', 0.0009370211763163744),\n",
       " (u'COMMENT_detener', 0.0005167280892210387),\n",
       " (u'COMMENT_diario', 0.000746749948624242),\n",
       " (u'COMMENT_dinero', 0.0008493819887251054),\n",
       " (u'COMMENT_dios', 0.0015360657453184975),\n",
       " (u'COMMENT_donde', 0.0005305031986389721),\n",
       " (u'COMMENT_dudar', 0.000315144868397018),\n",
       " (u'COMMENT_due', 0.0005952580365665738),\n",
       " (u'COMMENT_echar', 0.0002077112208784157),\n",
       " (u'COMMENT_educaci', 0.0012548507467078254),\n",
       " (u'COMMENT_encontrar', 0.0009099154215027835),\n",
       " (u'COMMENT_enga', 0.00041586571354134387),\n",
       " (u'COMMENT_ense', 0.0008725196234175664),\n",
       " (u'COMMENT_esa', 0.005849431838102485),\n",
       " (u'COMMENT_escuchar', 0.00038377511429150675),\n",
       " (u'COMMENT_ese', 0.00032972989919748916),\n",
       " (u'COMMENT_eso', 0.0005161668246582023),\n",
       " (u'COMMENT_esperar', 0.0006013762847801559),\n",
       " (u'COMMENT_est', 0.0005787582753496581),\n",
       " (u'COMMENT_esta', 0.0006168875203825181),\n",
       " (u'COMMENT_estar', 0.0017641416197957526),\n",
       " (u'COMMENT_esto', 0.0044814628117739),\n",
       " (u'COMMENT_estudiar', 0.0008804240879452174),\n",
       " (u'COMMENT_exactamente', 0.00109544281037863),\n",
       " (u'COMMENT_exacto', 0.0012610532922517014),\n",
       " (u'COMMENT_facebook', 0.0005348299062441662),\n",
       " (u'COMMENT_falso', 0.0009978751021006585),\n",
       " (u'COMMENT_faltar', 0.0009306121339701012),\n",
       " (u'COMMENT_falto', 0.005016373867510161),\n",
       " (u'COMMENT_familia', 0.0008669123375904358),\n",
       " (u'COMMENT_favor', 0.0009141952283366361),\n",
       " (u'COMMENT_federal', 0.0015000716514436418),\n",
       " (u'COMMENT_feo', 0.0010878777717916457),\n",
       " (u'COMMENT_foto', 0.0023047713209864234),\n",
       " (u'COMMENT_fue', 0.0005038245627267977),\n",
       " (u'COMMENT_funcionar', 0.00017532214224434797),\n",
       " (u'COMMENT_ganar', 0.0014000671344055777),\n",
       " (u'COMMENT_garcia', 0.0010670472598364477),\n",
       " (u'COMMENT_gente', 0.0027459694213039392),\n",
       " (u'COMMENT_gina', 0.0007363838728080481),\n",
       " (u'COMMENT_gobernar', 0.005159983126063188),\n",
       " (u'COMMENT_gracia', 0.0005299397564619749),\n",
       " (u'COMMENT_gracias', 0.001745718701162894),\n",
       " (u'COMMENT_grupo', 0.000936037287756704),\n",
       " (u'COMMENT_gustar', 0.0006634528632810146),\n",
       " (u'COMMENT_haber', 0.0013845600300303551),\n",
       " (u'COMMENT_hablar', 0.0035847884000108393),\n",
       " (u'COMMENT_hacer', 0.0017268220743367837),\n",
       " (u'COMMENT_hay', 0.004509938019663902),\n",
       " (u'COMMENT_hijo', 0.00808067451948633),\n",
       " (u'COMMENT_hijo puto', 0.0022656032824214327),\n",
       " (u'COMMENT_hijos', 0.0008639537318607522),\n",
       " (u'COMMENT_hombre', 0.00044914573161541747),\n",
       " (u'COMMENT_huevo', 0.001899334869084093),\n",
       " (u'COMMENT_importar', 0.0007588242549190737),\n",
       " (u'COMMENT_informaci', 0.0009475296353284293),\n",
       " (u'COMMENT_inventar', 0.0013648041440170462),\n",
       " (u'COMMENT_jaja', 0.003839044882941656),\n",
       " (u'COMMENT_jajaja', 0.0022117033503473455),\n",
       " (u'COMMENT_jajajaja', 0.001015829481370055),\n",
       " (u'COMMENT_jajajajaja', 0.0007757941269003957),\n",
       " (u'COMMENT_joven', 0.0009804002884572702),\n",
       " (u'COMMENT_juan', 7.427256163952925e-05),\n",
       " (u'COMMENT_justicia', 0.006276989398064198),\n",
       " (u'COMMENT_lacrar', 0.0008960067789794552),\n",
       " (u'COMMENT_lado', 0.0008128558203350868),\n",
       " (u'COMMENT_lamentablemente', 0.0004370389803894677),\n",
       " (u'COMMENT_leer', 0.00019212184971807996),\n",
       " (u'COMMENT_llamar', 0.0009054824777693819),\n",
       " (u'COMMENT_llegar', 0.0005027603579893649),\n",
       " (u'COMMENT_llevar', 0.0006446451591522968),\n",
       " (u'COMMENT_loco', 0.0009365117874270709),\n",
       " (u'COMMENT_lopez', 0.0015420125662696954),\n",
       " (u'COMMENT_los', 0.000635707975153161),\n",
       " (u'COMMENT_madre', 0.0030515260249428773),\n",
       " (u'COMMENT_maldito', 0.0018943289243712063),\n",
       " (u'COMMENT_malo', 0.00412656648815306),\n",
       " (u'COMMENT_mam', 0.0001391072901724223),\n",
       " (u'COMMENT_mamar', 0.0024860409059042888),\n",
       " (u'COMMENT_manchar', 0.0017047640063737714),\n",
       " (u'COMMENT_mandar', 0.0004034046852410093),\n",
       " (u'COMMENT_manejar', 0.0011884587011962672),\n",
       " (u'COMMENT_manir', 0.0003300833145352202),\n",
       " (u'COMMENT_mano', 0.0004555183481468591),\n",
       " (u'COMMENT_matar', 0.0010064026563700499),\n",
       " (u'COMMENT_mejor', 0.0009672486588794659),\n",
       " (u'COMMENT_mentira', 0.00536047680579118),\n",
       " (u'COMMENT_merecer', 0.0007618116617101352),\n",
       " (u'COMMENT_mesar', 0.007542577246464632),\n",
       " (u'COMMENT_meter', 0.00018724435685619386),\n",
       " (u'COMMENT_miedo', 0.0005573958150661172),\n",
       " (u'COMMENT_mierda', 0.00048750939203260566),\n",
       " (u'COMMENT_mira', 0.0008174323879025382),\n",
       " (u'COMMENT_mirar', 0.0010183532672991657),\n",
       " (u'COMMENT_morir', 0.0008082212096636295),\n",
       " (u'COMMENT_moto', 0.0005346514240862308),\n",
       " (u'COMMENT_mujer', 0.0008717116407708828),\n",
       " (u'COMMENT_necesitar', 0.00033822553009444),\n",
       " (u'COMMENT_neto', 0.00045612899456090844),\n",
       " (u'COMMENT_neza', 0.001645917783197106),\n",
       " (u'COMMENT_nomas', 0.0011304898040542578),\n",
       " (u'COMMENT_noticiar', 0.0007633586118856392),\n",
       " (u'COMMENT_ocupar', 0.0006791245037379594),\n",
       " (u'COMMENT_ojal', 0.00017966795688063958),\n",
       " (u'COMMENT_ojala', 0.0004941359776999535),\n",
       " (u'COMMENT_ojalar', 0.0011331795295032093),\n",
       " (u'COMMENT_ojete', 0.001818172789623438),\n",
       " (u'COMMENT_ojo', 0.0011788800566651081),\n",
       " (u'COMMENT_onda', 0.004612649358211599),\n",
       " (u'COMMENT_osea', 0.009721835268060275),\n",
       " (u'COMMENT_pagar', 0.0008320910174430465),\n",
       " (u'COMMENT_para', 0.0016781146078926334),\n",
       " (u'COMMENT_parar', 0.0005498136460869268),\n",
       " (u'COMMENT_partir', 0.0004205693667649251),\n",
       " (u'COMMENT_pasar', 0.002866889173066233),\n",
       " (u'COMMENT_paso', 0.00026156031531555826),\n",
       " (u'COMMENT_patrullar', 0.0006701952539139625),\n",
       " (u'COMMENT_peder', 0.000750933539854525),\n",
       " (u'COMMENT_pedir', 0.00015134712658201595),\n",
       " (u'COMMENT_pegar', 0.003193302901011285),\n",
       " (u'COMMENT_penar', 0.00018905365656756982),\n",
       " (u'COMMENT_pendejadas', 0.0008044327176287053),\n",
       " (u'COMMENT_pendejo', 0.001702830924072783),\n",
       " (u'COMMENT_pensar', 0.0017720605857641574),\n",
       " (u'COMMENT_peque', 0.00015325388772885836),\n",
       " (u'COMMENT_perd', 0.0008850609722412418),\n",
       " (u'COMMENT_perder', 0.0007048162352625409),\n",
       " (u'COMMENT_pero', 0.002950633111563099),\n",
       " (u'COMMENT_perro', 0.0005050359082890268),\n",
       " (u'COMMENT_personar', 0.002211728178609245),\n",
       " (u'COMMENT_pesar', 0.0015580138078351417),\n",
       " (u'COMMENT_peso', 0.0017856841427523005),\n",
       " (u'COMMENT_pinchar', 0.00330781449535634),\n",
       " (u'COMMENT_pinche', 0.0016015425662613924),\n",
       " (u'COMMENT_pinches', 0.0007117725575130437),\n",
       " (u'COMMENT_poblar', 0.0003041959584312311),\n",
       " (u'COMMENT_poder', 0.0017079627001599486),\n",
       " (u'COMMENT_polic', 0.00580286722871221),\n",
       " (u'COMMENT_policias', 0.006319097044909915),\n",
       " (u'COMMENT_poner', 0.0012622937603494729),\n",
       " (u'COMMENT_por', 0.0019072916595042036),\n",
       " (u'COMMENT_por favor', 0.00029193449801933545),\n",
       " (u'COMMENT_porq', 0.0009877886634608788),\n",
       " (u'COMMENT_porque', 0.000452917313896348),\n",
       " (u'COMMENT_preguntar', 0.0005929000885221036),\n",
       " (u'COMMENT_pri', 0.0005158703496859338),\n",
       " (u'COMMENT_primero', 0.0006087556243823242),\n",
       " (u'COMMENT_problema', 0.0011027944049231268),\n",
       " (u'COMMENT_publicaci', 0.000820923919204942),\n",
       " (u'COMMENT_publicar', 0.0007241585722245304),\n",
       " (u'COMMENT_pues', 0.0013729022584361005),\n",
       " (u'COMMENT_punto', 8.954567122115302e-05),\n",
       " (u'COMMENT_puro', 0.0009908391219261492),\n",
       " (u'COMMENT_puto', 0.002965603921645207),\n",
       " (u'COMMENT_que', 0.0023916757940799947),\n",
       " (u'COMMENT_que madre', 0.0009328352588476229),\n",
       " (u'COMMENT_quedo', 0.0005821117418657747),\n",
       " (u'COMMENT_quejar', 0.0004989308779483076),\n",
       " (u'COMMENT_quemar', 0.0011987332206851021),\n",
       " (u'COMMENT_querer', 0.0009111602105946047),\n",
       " (u'COMMENT_quien', 0.0026221021318616144),\n",
       " (u'COMMENT_rata', 0.0017078664900142446),\n",
       " (u'COMMENT_ratero', 0.001994059204536602),\n",
       " (u'COMMENT_regalar', 0.0009374943731503633),\n",
       " (u'COMMENT_regresar', 0.000675967035311296),\n",
       " (u'COMMENT_resultar', 0.009325862558696102),\n",
       " (u'COMMENT_robar', 0.004822463490411231),\n",
       " (u'COMMENT_ruta', 0.0018662715519123338),\n",
       " (u'COMMENT_sacar', 0.0006692377844096631),\n",
       " (u'COMMENT_salir', 0.00045268734410555575),\n",
       " (u'COMMENT_sanchez', 0.0009050079454889205),\n",
       " (u'COMMENT_santo', 0.0018987252066338368),\n",
       " (u'COMMENT_seguir', 0.0006627735388584406),\n",
       " (u'COMMENT_seguridad', 0.0006843863379131357),\n",
       " (u'COMMENT_seguro', 0.000843894744062709),\n",
       " (u'COMMENT_semana', 0.0007146843088660484),\n",
       " (u'COMMENT_sentir', 0.003147832171485034),\n",
       " (u'COMMENT_ser', 0.004445169503354089),\n",
       " (u'COMMENT_servicio', 0.000530410747766839),\n",
       " (u'COMMENT_servir', 0.0045980106842289144),\n",
       " (u'COMMENT_siempre', 0.0023617518989775656),\n",
       " (u'COMMENT_simple', 0.0017127606779013776),\n",
       " (u'COMMENT_siquiera', 0.0004477248208762482),\n",
       " (u'COMMENT_sociedad', 0.0016963527404186473),\n",
       " (u'COMMENT_solo', 0.002181355361258655),\n",
       " (u'COMMENT_subir', 0.0006527254356168712),\n",
       " (u'COMMENT_suerte', 0.0004582162746137417),\n",
       " (u'COMMENT_sufrir', 0.0005547787348485649),\n",
       " (u'COMMENT_suponer', 0.0007954055476951828),\n",
       " (u'COMMENT_tanto', 0.0002629100899443362),\n",
       " (u'COMMENT_tener', 0.0026532234750907464),\n",
       " (u'COMMENT_tipo', 0.003867788020021047),\n",
       " (u'COMMENT_tocar', 0.000775781380501044),\n",
       " (u'COMMENT_todos', 0.0007300997822386627),\n",
       " (u'COMMENT_tomar', 0.0012854674809055799),\n",
       " (u'COMMENT_trabajador', 0.0006553616573644793),\n",
       " (u'COMMENT_trabajar', 0.0018719326010379679),\n",
       " (u'COMMENT_traer', 0.0019245064830165604),\n",
       " (u'COMMENT_triste', 0.0013241693337703025),\n",
       " (u'COMMENT_unidad', 0.0008596996533342107),\n",
       " (u'COMMENT_valer', 0.0011285332962842755),\n",
       " (u'COMMENT_vender', 0.0008605815595146611),\n",
       " (u'COMMENT_venir', 0.002905970154138049),\n",
       " (u'COMMENT_ver', 0.0023665252320392814),\n",
       " (u'COMMENT_vestir', 0.0011787408533265314),\n",
       " (u'COMMENT_vida', 0.000681618530709823),\n",
       " (u'COMMENT_video', 0.0009605905021375187),\n",
       " (u'COMMENT_viejo', 0.0007287449921718262),\n",
       " (u'COMMENT_vivir', 0.003416833893927633),\n",
       " (u'COMMENT_volver', 0.001398529073182062),\n",
       " (u'COMMENT_xico', 0.0011515230001069602)]"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rfe = RFE(estimator=clfGB, step=10, n_features_to_select = 470)\n",
    "rfe.fit(goldX, goldY)\n",
    "#list of retained features\n",
    "kept_features = rfe.get_support(indices=True)\n",
    "zip(goldX.iloc[:,kept_features].columns.values, list(rfe.estimator_.feature_importances_.reshape(-1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.9428172942817294\n"
     ]
    }
   ],
   "source": [
    "#MNB classifier to get important features per cluster\n",
    "feature_names = np.array(goldX.iloc[:,kept_features].columns.values)\n",
    "clfGB = GradientBoostingClassifier(n_estimators = 200, max_depth=2, random_state=120)\n",
    "clfGB.fit(goldX.iloc[:,kept_features], goldY)\n",
    "print \"accuracy: \" + str(clfGB.score(goldX.iloc[:,kept_features], goldY))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Robbery: NGRAM_asaltar, NGRAM_robar, NGRAM_placa, NGRAM_acabar, NGRAM_armar, NGRAM_mensaje, NGRAM_color, NGRAM_auto, NGRAM_una, NGRAM_camioneta, NGRAM_calle, NGRAM_noche, NGRAM_llevar, NGRAM_zona, NGRAM_denunciar, NGRAM_tipo, NGRAM_con, NGRAM_violencia, NGRAM_estado xico, COMMENT_jajajaja\n",
      "Disappearance: NGRAM_amigo, NGRAM_vestir, NGRAM_edad, NGRAM_localizar, NGRAM_marzo, NGRAM_salir, NGRAM_informaci, Has Image, COMMENT_dios, NGRAM_nombrar, NGRAM_ecatepec, NGRAM_familiar, NGRAM_tipo, NGRAM_negro, NGRAM_ayudar, NGRAM_regresar, NGRAM_solicitar, NGRAM_menor, NGRAM_favor, NGRAM_las\n",
      "Homicide: NGRAM_hash, NGRAM_personar, NGRAM_con, NGRAM_mesar, NGRAM_madre, NGRAM_vida, NGRAM_coacalco, COMMENT_defender, NGRAM_pagar, NGRAM_por favor, COMMENT_polic, NGRAM_compartir, COMMENT_andar, NGRAM_por, COMMENT_osea, COMMENT_policias, COMMENT_resultar, COMMENT_aparecer, COMMENT_garcia, COMMENT_alos\n",
      "Violence: NGRAM_violencia, NGRAM_unidad, NGRAM_las, NGRAM_personar, NGRAM_edad, NGRAM_hijo, NGRAM_sacar, NGRAM_poder, NGRAM_caer, NGRAM_armar, NGRAM_llegar, NGRAM_localizar, NGRAM_informar, NGRAM_foto, NGRAM_dar, NGRAM_polic, NGRAM_casar, NGRAM_andar, NGRAM_nombrar, COMMENT_autoridad\n",
      "Extortion: COMMENT_cargar, NGRAM_dinero, NGRAM_julio, NGRAM_coacalco, COMMENT_robar, NGRAM_ser, NGRAM_denunciar, NGRAM_llamar, NGRAM_buscar, NGRAM_dejar, NGRAM_andar, NGRAM_traer, NGRAM_personar, NGRAM_llegar, NGRAM_facebook, COMMENT_caro, COMMENT_siempre, COMMENT_sentir, COMMENT_alcanzar, COMMENT_regalar\n",
      "Noncriminal: NGRAM_perrito, NGRAM_escuela, NGRAM_necesitar, NGRAM_gracias, NGRAM_formar, NGRAM_alguien, NGRAM_san, COMMENT_pri, NGRAM_ciudad, NGRAM_estar, NGRAM_perder, NGRAM_valle, NGRAM_para, COMMENT_personar, NGRAM_les, NGRAM_transportar, NGRAM_cobrar, NGRAM_callar, COMMENT_eso, NGRAM_conocer\n"
     ]
    }
   ],
   "source": [
    "for i, label in enumerate([\"Robbery\", \"Disappearance\", \"Homicide\",\n",
    "                   \"Violence\", \"Extortion\", \"Noncriminal\"]):\n",
    "    topTermLocs = np.argsort(clfGB.coef_[i])[::-1][:20]\n",
    "    topTerms = \", \".join(feature_names[topTermLocs])\n",
    "    print(\"%s: %s\" % (label, topTerms))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Now try predicting Disappearance vs Rest, as this is an easier problem and also the new focus of the project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "goldFeatures = post_df.query(\"Gold_Disappearance_Label.notnull()\")\n",
    "goldFeatures = goldFeatures.drop([\"Vertex\", \"Popularity Measure\", \"Total Likes\", \"Total Shares\", \"Total Comments\",\n",
    "                       \"Month\", \"Type\", \"Date\", \"Post Id\", \"Gold_Crime_Type_Label\", \"Text\", \"Crime Type\"], axis=1)\n",
    "goldX = goldFeatures.drop([\"Gold_Disappearance_Label\", \"Group\"],axis=1)\n",
    "goldY = goldFeatures.loc[:,\"Gold_Disappearance_Label\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Person in Image' 'Outdoors in Image' u'POST_acabar' u'POST_amigo'\n",
      " u'POST_blico' u'POST_callar' u'POST_ciudad' u'POST_dejar' u'POST_edad'\n",
      " u'POST_escuela' u'POST_febrero' u'POST_gente' u'POST_goo' u'POST_gracia'\n",
      " u'POST_hoy' u'POST_julio' u'POST_localizar' u'POST_pagar' u'POST_pasar'\n",
      " u'POST_placa' u'POST_por' u'POST_salir' u'POST_san' u'POST_subir'\n",
      " u'POST_vestir' u'COMMENT_ayudar' u'COMMENT_buscar' u'COMMENT_dios'\n",
      " u'COMMENT_gente' u'COMMENT_jajaja' u'COMMENT_madre' u'COMMENT_mira'\n",
      " u'COMMENT_personar' u'COMMENT_puto' u'IMAGE_1 smiling'\n",
      " u'IMAGE_1 standing' u'IMAGE_meme' u'IMAGE_shoes text' u'IMAGE_sitting'\n",
      " u'IMAGE_smiling text']\n"
     ]
    }
   ],
   "source": [
    "clsfFeatures = LogisticRegression(class_weight=\"balanced\", C=5)\n",
    "clsf = MLPClassifier(solver=\"lbfgs\")\n",
    "\n",
    "rfe = RFE(estimator=clsfFeatures, step=1, n_features_to_select = 40)\n",
    "rfe.fit(goldX, goldY)\n",
    "#list of retained features\n",
    "kept_feature_ids = rfe.get_support(indices=True)\n",
    "kept_features = np.array(goldX.iloc[:,kept_feature_ids].columns.values)\n",
    "print kept_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.847564935064935\n",
      "0.8625\n",
      "0.9666036936994875\n"
     ]
    }
   ],
   "source": [
    "#K-Fold Evaluation\n",
    "precisions = []\n",
    "recalls = []\n",
    "accuracies = []\n",
    "for train, test in StratifiedKFold(10).split(goldX.loc[:,kept_features], goldY):\n",
    "    clsf.fit(goldX.loc[:,kept_features].iloc[train,:], goldY.iloc[train])\n",
    "    predictedK = clsf.predict(goldX.loc[:,kept_features].iloc[test,:])\n",
    "    precisions.append(metrics.precision_score(goldY.iloc[test], predictedK))\n",
    "    recalls.append(metrics.recall_score(goldY.iloc[test], predictedK))\n",
    "    accuracies.append(metrics.accuracy_score(goldY.iloc[test], predictedK))\n",
    "\n",
    "print np.mean(precisions)\n",
    "print np.mean(recalls)\n",
    "print np.mean(accuracies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "goldFeatures[\"Prediction\"] = clsf.predict(goldFeatures.loc[:,kept_features])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Investigate ids that were predicted incorrectly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gold Label  Prediction\n",
      "0.0         1.0           6\n",
      "1.0         0.0           6\n",
      "dtype: int64\n",
      "(12, 490)\n"
     ]
    }
   ],
   "source": [
    "goldFeatures[\"Wrong Prediction\"] = goldFeatures.apply(\\\n",
    "                                       lambda x: x[\"Gold_Disappearance_Label\"] is not None\n",
    "                                       and x[\"Prediction\"] != x[\"Gold_Disappearance_Label\"],\\\n",
    "                                       axis=1)\n",
    "wrong_df = goldFeatures[goldFeatures[\"Wrong Prediction\"]].copy()\n",
    "wrongGroups = wrong_df.groupby([\"Gold Label\", \"Prediction\"]) \n",
    "print wrongGroups.size()\n",
    "print wrong_df.shape"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}