ItemBasedCF.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Item-Based Collaborative Filtering"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "As before, we'll start by importing the MovieLens 100K data set into a pandas DataFrame:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movie_id</th>\n",
       "      <th>title</th>\n",
       "      <th>user_id</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>308</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>287</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>148</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>280</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>66</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   movie_id             title  user_id  rating\n",
       "0         1  Toy Story (1995)      308       4\n",
       "1         1  Toy Story (1995)      287       5\n",
       "2         1  Toy Story (1995)      148       4\n",
       "3         1  Toy Story (1995)      280       4\n",
       "4         1  Toy Story (1995)       66       3"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "r_cols = ['user_id', 'movie_id', 'rating']\n",
    "ratings = pd.read_csv('D:/7.DataScience/DataScience-Python3/ml-100k/u.data', sep='\\t', names=r_cols, usecols=range(3), encoding=\"ISO-8859-1\")\n",
    "\n",
    "m_cols = ['movie_id', 'title']\n",
    "movies = pd.read_csv('D:/7.DataScience/DataScience-Python3/ml-100k/u.item', sep='|', names=m_cols, usecols=range(2), encoding=\"ISO-8859-1\")\n",
    "\n",
    "ratings = pd.merge(movies, ratings)\n",
    "\n",
    "ratings.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now we'll pivot this table to construct a nice matrix of users and the movies they rated. NaN indicates missing data, or movies that a given user did not watch:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>title</th>\n",
       "      <th>'Til There Was You (1997)</th>\n",
       "      <th>1-900 (1994)</th>\n",
       "      <th>101 Dalmatians (1996)</th>\n",
       "      <th>12 Angry Men (1957)</th>\n",
       "      <th>187 (1997)</th>\n",
       "      <th>2 Days in the Valley (1996)</th>\n",
       "      <th>20,000 Leagues Under the Sea (1954)</th>\n",
       "      <th>2001: A Space Odyssey (1968)</th>\n",
       "      <th>3 Ninjas: High Noon At Mega Mountain (1998)</th>\n",
       "      <th>39 Steps, The (1935)</th>\n",
       "      <th>...</th>\n",
       "      <th>Yankee Zulu (1994)</th>\n",
       "      <th>Year of the Horse (1997)</th>\n",
       "      <th>You So Crazy (1994)</th>\n",
       "      <th>Young Frankenstein (1974)</th>\n",
       "      <th>Young Guns (1988)</th>\n",
       "      <th>Young Guns II (1990)</th>\n",
       "      <th>Young Poisoner's Handbook, The (1995)</th>\n",
       "      <th>Zeus and Roxanne (1997)</th>\n",
       "      <th>unknown</th>\n",
       "      <th>Á köldum klaka (Cold Fever) (1994)</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>user_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1664 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "title    'Til There Was You (1997)  1-900 (1994)  101 Dalmatians (1996)  \\\n",
       "user_id                                                                   \n",
       "0                              NaN           NaN                    NaN   \n",
       "1                              NaN           NaN                    2.0   \n",
       "2                              NaN           NaN                    NaN   \n",
       "3                              NaN           NaN                    NaN   \n",
       "4                              NaN           NaN                    NaN   \n",
       "\n",
       "title    12 Angry Men (1957)  187 (1997)  2 Days in the Valley (1996)  \\\n",
       "user_id                                                                 \n",
       "0                        NaN         NaN                          NaN   \n",
       "1                        5.0         NaN                          NaN   \n",
       "2                        NaN         NaN                          NaN   \n",
       "3                        NaN         2.0                          NaN   \n",
       "4                        NaN         NaN                          NaN   \n",
       "\n",
       "title    20,000 Leagues Under the Sea (1954)  2001: A Space Odyssey (1968)  \\\n",
       "user_id                                                                      \n",
       "0                                        NaN                           NaN   \n",
       "1                                        3.0                           4.0   \n",
       "2                                        NaN                           NaN   \n",
       "3                                        NaN                           NaN   \n",
       "4                                        NaN                           NaN   \n",
       "\n",
       "title    3 Ninjas: High Noon At Mega Mountain (1998)  39 Steps, The (1935)  \\\n",
       "user_id                                                                      \n",
       "0                                                NaN                   NaN   \n",
       "1                                                NaN                   NaN   \n",
       "2                                                1.0                   NaN   \n",
       "3                                                NaN                   NaN   \n",
       "4                                                NaN                   NaN   \n",
       "\n",
       "title                   ...                  Yankee Zulu (1994)  \\\n",
       "user_id                 ...                                       \n",
       "0                       ...                                 NaN   \n",
       "1                       ...                                 NaN   \n",
       "2                       ...                                 NaN   \n",
       "3                       ...                                 NaN   \n",
       "4                       ...                                 NaN   \n",
       "\n",
       "title    Year of the Horse (1997)  You So Crazy (1994)  \\\n",
       "user_id                                                  \n",
       "0                             NaN                  NaN   \n",
       "1                             NaN                  NaN   \n",
       "2                             NaN                  NaN   \n",
       "3                             NaN                  NaN   \n",
       "4                             NaN                  NaN   \n",
       "\n",
       "title    Young Frankenstein (1974)  Young Guns (1988)  Young Guns II (1990)  \\\n",
       "user_id                                                                       \n",
       "0                              NaN                NaN                   NaN   \n",
       "1                              5.0                3.0                   NaN   \n",
       "2                              NaN                NaN                   NaN   \n",
       "3                              NaN                NaN                   NaN   \n",
       "4                              NaN                NaN                   NaN   \n",
       "\n",
       "title    Young Poisoner's Handbook, The (1995)  Zeus and Roxanne (1997)  \\\n",
       "user_id                                                                   \n",
       "0                                          NaN                      NaN   \n",
       "1                                          NaN                      NaN   \n",
       "2                                          NaN                      NaN   \n",
       "3                                          NaN                      NaN   \n",
       "4                                          NaN                      NaN   \n",
       "\n",
       "title    unknown  Á köldum klaka (Cold Fever) (1994)  \n",
       "user_id                                               \n",
       "0            NaN                                 NaN  \n",
       "1            4.0                                 NaN  \n",
       "2            NaN                                 NaN  \n",
       "3            NaN                                 NaN  \n",
       "4            NaN                                 NaN  \n",
       "\n",
       "[5 rows x 1664 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "userRatings = ratings.pivot_table(index=['user_id'],columns=['title'],values='rating')\n",
    "userRatings.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now the magic happens - pandas has a built-in corr() method that will compute a correlation score for every column pair in the matrix! This gives us a correlation score between every pair of movies (where at least one user rated both movies - otherwise NaN's will show up.) That's amazing!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>title</th>\n",
       "      <th>'Til There Was You (1997)</th>\n",
       "      <th>1-900 (1994)</th>\n",
       "      <th>101 Dalmatians (1996)</th>\n",
       "      <th>12 Angry Men (1957)</th>\n",
       "      <th>187 (1997)</th>\n",
       "      <th>2 Days in the Valley (1996)</th>\n",
       "      <th>20,000 Leagues Under the Sea (1954)</th>\n",
       "      <th>2001: A Space Odyssey (1968)</th>\n",
       "      <th>3 Ninjas: High Noon At Mega Mountain (1998)</th>\n",
       "      <th>39 Steps, The (1935)</th>\n",
       "      <th>...</th>\n",
       "      <th>Yankee Zulu (1994)</th>\n",
       "      <th>Year of the Horse (1997)</th>\n",
       "      <th>You So Crazy (1994)</th>\n",
       "      <th>Young Frankenstein (1974)</th>\n",
       "      <th>Young Guns (1988)</th>\n",
       "      <th>Young Guns II (1990)</th>\n",
       "      <th>Young Poisoner's Handbook, The (1995)</th>\n",
       "      <th>Zeus and Roxanne (1997)</th>\n",
       "      <th>unknown</th>\n",
       "      <th>Á köldum klaka (Cold Fever) (1994)</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>title</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>'Til There Was You (1997)</th>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-0.500000</td>\n",
       "      <td>-0.500000</td>\n",
       "      <td>0.522233</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.426401</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1-900 (1994)</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.981981</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.944911</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101 Dalmatians (1996)</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.049890</td>\n",
       "      <td>0.269191</td>\n",
       "      <td>0.048973</td>\n",
       "      <td>0.266928</td>\n",
       "      <td>-0.043407</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.158840</td>\n",
       "      <td>0.119234</td>\n",
       "      <td>0.680414</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.707107</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12 Angry Men (1957)</th>\n",
       "      <td>-0.5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.049890</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.256625</td>\n",
       "      <td>0.274772</td>\n",
       "      <td>0.178848</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.457176</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.096546</td>\n",
       "      <td>0.068944</td>\n",
       "      <td>-0.361961</td>\n",
       "      <td>0.144338</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>187 (1997)</th>\n",
       "      <td>-0.5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.269191</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.596644</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.554700</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.866025</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.455233</td>\n",
       "      <td>-0.500000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.475327</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1664 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "title                      'Til There Was You (1997)  1-900 (1994)  \\\n",
       "title                                                                \n",
       "'Til There Was You (1997)                        1.0           NaN   \n",
       "1-900 (1994)                                     NaN           1.0   \n",
       "101 Dalmatians (1996)                           -1.0           NaN   \n",
       "12 Angry Men (1957)                             -0.5           NaN   \n",
       "187 (1997)                                      -0.5           NaN   \n",
       "\n",
       "title                      101 Dalmatians (1996)  12 Angry Men (1957)  \\\n",
       "title                                                                   \n",
       "'Til There Was You (1997)              -1.000000            -0.500000   \n",
       "1-900 (1994)                                 NaN                  NaN   \n",
       "101 Dalmatians (1996)                   1.000000            -0.049890   \n",
       "12 Angry Men (1957)                    -0.049890             1.000000   \n",
       "187 (1997)                              0.269191             0.666667   \n",
       "\n",
       "title                      187 (1997)  2 Days in the Valley (1996)  \\\n",
       "title                                                                \n",
       "'Til There Was You (1997)   -0.500000                     0.522233   \n",
       "1-900 (1994)                      NaN                          NaN   \n",
       "101 Dalmatians (1996)        0.269191                     0.048973   \n",
       "12 Angry Men (1957)          0.666667                     0.256625   \n",
       "187 (1997)                   1.000000                     0.596644   \n",
       "\n",
       "title                      20,000 Leagues Under the Sea (1954)  \\\n",
       "title                                                            \n",
       "'Til There Was You (1997)                                  NaN   \n",
       "1-900 (1994)                                               NaN   \n",
       "101 Dalmatians (1996)                                 0.266928   \n",
       "12 Angry Men (1957)                                   0.274772   \n",
       "187 (1997)                                                 NaN   \n",
       "\n",
       "title                      2001: A Space Odyssey (1968)  \\\n",
       "title                                                     \n",
       "'Til There Was You (1997)                     -0.426401   \n",
       "1-900 (1994)                                  -0.981981   \n",
       "101 Dalmatians (1996)                         -0.043407   \n",
       "12 Angry Men (1957)                            0.178848   \n",
       "187 (1997)                                    -0.554700   \n",
       "\n",
       "title                      3 Ninjas: High Noon At Mega Mountain (1998)  \\\n",
       "title                                                                    \n",
       "'Til There Was You (1997)                                          NaN   \n",
       "1-900 (1994)                                                       NaN   \n",
       "101 Dalmatians (1996)                                              NaN   \n",
       "12 Angry Men (1957)                                                NaN   \n",
       "187 (1997)                                                         NaN   \n",
       "\n",
       "title                      39 Steps, The (1935)  \\\n",
       "title                                             \n",
       "'Til There Was You (1997)                   NaN   \n",
       "1-900 (1994)                                NaN   \n",
       "101 Dalmatians (1996)                  0.111111   \n",
       "12 Angry Men (1957)                    0.457176   \n",
       "187 (1997)                             1.000000   \n",
       "\n",
       "title                                     ...                  \\\n",
       "title                                     ...                   \n",
       "'Til There Was You (1997)                 ...                   \n",
       "1-900 (1994)                              ...                   \n",
       "101 Dalmatians (1996)                     ...                   \n",
       "12 Angry Men (1957)                       ...                   \n",
       "187 (1997)                                ...                   \n",
       "\n",
       "title                      Yankee Zulu (1994)  Year of the Horse (1997)  \\\n",
       "title                                                                     \n",
       "'Til There Was You (1997)                 NaN                       NaN   \n",
       "1-900 (1994)                              NaN                       NaN   \n",
       "101 Dalmatians (1996)                     NaN                 -1.000000   \n",
       "12 Angry Men (1957)                       NaN                       NaN   \n",
       "187 (1997)                                NaN                  0.866025   \n",
       "\n",
       "title                      You So Crazy (1994)  Young Frankenstein (1974)  \\\n",
       "title                                                                       \n",
       "'Til There Was You (1997)                  NaN                        NaN   \n",
       "1-900 (1994)                               NaN                  -0.944911   \n",
       "101 Dalmatians (1996)                      NaN                   0.158840   \n",
       "12 Angry Men (1957)                        NaN                   0.096546   \n",
       "187 (1997)                                 NaN                   0.455233   \n",
       "\n",
       "title                      Young Guns (1988)  Young Guns II (1990)  \\\n",
       "title                                                                \n",
       "'Til There Was You (1997)                NaN                   NaN   \n",
       "1-900 (1994)                             NaN                   NaN   \n",
       "101 Dalmatians (1996)               0.119234              0.680414   \n",
       "12 Angry Men (1957)                 0.068944             -0.361961   \n",
       "187 (1997)                         -0.500000              0.500000   \n",
       "\n",
       "title                      Young Poisoner's Handbook, The (1995)  \\\n",
       "title                                                              \n",
       "'Til There Was You (1997)                                    NaN   \n",
       "1-900 (1994)                                                 NaN   \n",
       "101 Dalmatians (1996)                                   0.000000   \n",
       "12 Angry Men (1957)                                     0.144338   \n",
       "187 (1997)                                              0.475327   \n",
       "\n",
       "title                      Zeus and Roxanne (1997)  unknown  \\\n",
       "title                                                         \n",
       "'Til There Was You (1997)                      NaN      NaN   \n",
       "1-900 (1994)                                   NaN      NaN   \n",
       "101 Dalmatians (1996)                     0.707107      NaN   \n",
       "12 Angry Men (1957)                       1.000000      1.0   \n",
       "187 (1997)                                     NaN      NaN   \n",
       "\n",
       "title                      Á köldum klaka (Cold Fever) (1994)  \n",
       "title                                                          \n",
       "'Til There Was You (1997)                                 NaN  \n",
       "1-900 (1994)                                              NaN  \n",
       "101 Dalmatians (1996)                                     NaN  \n",
       "12 Angry Men (1957)                                       NaN  \n",
       "187 (1997)                                                NaN  \n",
       "\n",
       "[5 rows x 1664 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corrMatrix = userRatings.corr()\n",
    "corrMatrix.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "However, we want to avoid spurious results that happened from just a handful of users that happened to rate the same pair of movies. In order to restrict our results to movies that lots of people rated together - and also give us more popular results that are more easily recongnizable - we'll use the min_periods argument to throw out results where fewer than 100 users rated a given movie pair:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>title</th>\n",
       "      <th>'Til There Was You (1997)</th>\n",
       "      <th>1-900 (1994)</th>\n",
       "      <th>101 Dalmatians (1996)</th>\n",
       "      <th>12 Angry Men (1957)</th>\n",
       "      <th>187 (1997)</th>\n",
       "      <th>2 Days in the Valley (1996)</th>\n",
       "      <th>20,000 Leagues Under the Sea (1954)</th>\n",
       "      <th>2001: A Space Odyssey (1968)</th>\n",
       "      <th>3 Ninjas: High Noon At Mega Mountain (1998)</th>\n",
       "      <th>39 Steps, The (1935)</th>\n",
       "      <th>...</th>\n",
       "      <th>Yankee Zulu (1994)</th>\n",
       "      <th>Year of the Horse (1997)</th>\n",
       "      <th>You So Crazy (1994)</th>\n",
       "      <th>Young Frankenstein (1974)</th>\n",
       "      <th>Young Guns (1988)</th>\n",
       "      <th>Young Guns II (1990)</th>\n",
       "      <th>Young Poisoner's Handbook, The (1995)</th>\n",
       "      <th>Zeus and Roxanne (1997)</th>\n",
       "      <th>unknown</th>\n",
       "      <th>Á köldum klaka (Cold Fever) (1994)</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>title</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>'Til There Was You (1997)</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1-900 (1994)</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101 Dalmatians (1996)</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12 Angry Men (1957)</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>187 (1997)</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1664 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "title                      'Til There Was You (1997)  1-900 (1994)  \\\n",
       "title                                                                \n",
       "'Til There Was You (1997)                        NaN           NaN   \n",
       "1-900 (1994)                                     NaN           NaN   \n",
       "101 Dalmatians (1996)                            NaN           NaN   \n",
       "12 Angry Men (1957)                              NaN           NaN   \n",
       "187 (1997)                                       NaN           NaN   \n",
       "\n",
       "title                      101 Dalmatians (1996)  12 Angry Men (1957)  \\\n",
       "title                                                                   \n",
       "'Til There Was You (1997)                    NaN                  NaN   \n",
       "1-900 (1994)                                 NaN                  NaN   \n",
       "101 Dalmatians (1996)                        1.0                  NaN   \n",
       "12 Angry Men (1957)                          NaN                  1.0   \n",
       "187 (1997)                                   NaN                  NaN   \n",
       "\n",
       "title                      187 (1997)  2 Days in the Valley (1996)  \\\n",
       "title                                                                \n",
       "'Til There Was You (1997)         NaN                          NaN   \n",
       "1-900 (1994)                      NaN                          NaN   \n",
       "101 Dalmatians (1996)             NaN                          NaN   \n",
       "12 Angry Men (1957)               NaN                          NaN   \n",
       "187 (1997)                        NaN                          NaN   \n",
       "\n",
       "title                      20,000 Leagues Under the Sea (1954)  \\\n",
       "title                                                            \n",
       "'Til There Was You (1997)                                  NaN   \n",
       "1-900 (1994)                                               NaN   \n",
       "101 Dalmatians (1996)                                      NaN   \n",
       "12 Angry Men (1957)                                        NaN   \n",
       "187 (1997)                                                 NaN   \n",
       "\n",
       "title                      2001: A Space Odyssey (1968)  \\\n",
       "title                                                     \n",
       "'Til There Was You (1997)                           NaN   \n",
       "1-900 (1994)                                        NaN   \n",
       "101 Dalmatians (1996)                               NaN   \n",
       "12 Angry Men (1957)                                 NaN   \n",
       "187 (1997)                                          NaN   \n",
       "\n",
       "title                      3 Ninjas: High Noon At Mega Mountain (1998)  \\\n",
       "title                                                                    \n",
       "'Til There Was You (1997)                                          NaN   \n",
       "1-900 (1994)                                                       NaN   \n",
       "101 Dalmatians (1996)                                              NaN   \n",
       "12 Angry Men (1957)                                                NaN   \n",
       "187 (1997)                                                         NaN   \n",
       "\n",
       "title                      39 Steps, The (1935)  \\\n",
       "title                                             \n",
       "'Til There Was You (1997)                   NaN   \n",
       "1-900 (1994)                                NaN   \n",
       "101 Dalmatians (1996)                       NaN   \n",
       "12 Angry Men (1957)                         NaN   \n",
       "187 (1997)                                  NaN   \n",
       "\n",
       "title                                     ...                  \\\n",
       "title                                     ...                   \n",
       "'Til There Was You (1997)                 ...                   \n",
       "1-900 (1994)                              ...                   \n",
       "101 Dalmatians (1996)                     ...                   \n",
       "12 Angry Men (1957)                       ...                   \n",
       "187 (1997)                                ...                   \n",
       "\n",
       "title                      Yankee Zulu (1994)  Year of the Horse (1997)  \\\n",
       "title                                                                     \n",
       "'Til There Was You (1997)                 NaN                       NaN   \n",
       "1-900 (1994)                              NaN                       NaN   \n",
       "101 Dalmatians (1996)                     NaN                       NaN   \n",
       "12 Angry Men (1957)                       NaN                       NaN   \n",
       "187 (1997)                                NaN                       NaN   \n",
       "\n",
       "title                      You So Crazy (1994)  Young Frankenstein (1974)  \\\n",
       "title                                                                       \n",
       "'Til There Was You (1997)                  NaN                        NaN   \n",
       "1-900 (1994)                               NaN                        NaN   \n",
       "101 Dalmatians (1996)                      NaN                        NaN   \n",
       "12 Angry Men (1957)                        NaN                        NaN   \n",
       "187 (1997)                                 NaN                        NaN   \n",
       "\n",
       "title                      Young Guns (1988)  Young Guns II (1990)  \\\n",
       "title                                                                \n",
       "'Til There Was You (1997)                NaN                   NaN   \n",
       "1-900 (1994)                             NaN                   NaN   \n",
       "101 Dalmatians (1996)                    NaN                   NaN   \n",
       "12 Angry Men (1957)                      NaN                   NaN   \n",
       "187 (1997)                               NaN                   NaN   \n",
       "\n",
       "title                      Young Poisoner's Handbook, The (1995)  \\\n",
       "title                                                              \n",
       "'Til There Was You (1997)                                    NaN   \n",
       "1-900 (1994)                                                 NaN   \n",
       "101 Dalmatians (1996)                                        NaN   \n",
       "12 Angry Men (1957)                                          NaN   \n",
       "187 (1997)                                                   NaN   \n",
       "\n",
       "title                      Zeus and Roxanne (1997)  unknown  \\\n",
       "title                                                         \n",
       "'Til There Was You (1997)                      NaN      NaN   \n",
       "1-900 (1994)                                   NaN      NaN   \n",
       "101 Dalmatians (1996)                          NaN      NaN   \n",
       "12 Angry Men (1957)                            NaN      NaN   \n",
       "187 (1997)                                     NaN      NaN   \n",
       "\n",
       "title                      Á köldum klaka (Cold Fever) (1994)  \n",
       "title                                                          \n",
       "'Til There Was You (1997)                                 NaN  \n",
       "1-900 (1994)                                              NaN  \n",
       "101 Dalmatians (1996)                                     NaN  \n",
       "12 Angry Men (1957)                                       NaN  \n",
       "187 (1997)                                                NaN  \n",
       "\n",
       "[5 rows x 1664 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corrMatrix = userRatings.corr(method='pearson', min_periods=100)\n",
    "corrMatrix.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now let's produce some movie recommendations for user ID 0, who I manually added to the data set as a test case. This guy really likes Star Wars and The Empire Strikes Back, but hated Gone with the Wind. I'll extract his ratings from the userRatings DataFrame, and use dropna() to get rid of missing data (leaving me only with a Series of the movies I actually rated:)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "title\n",
       "Empire Strikes Back, The (1980)    5.0\n",
       "Gone with the Wind (1939)          1.0\n",
       "Star Wars (1977)                   5.0\n",
       "Name: 0, dtype: float64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "myRatings = userRatings.loc[0].dropna()\n",
    "myRatings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now, let's go through each movie I rated one at a time, and build up a list of possible recommendations based on the movies similar to the ones I rated.\n",
    "\n",
    "So for each movie I rated, I'll retrieve the list of similar movies from our correlation matrix. I'll then scale those correlation scores by how well I rated the movie they are similar to, so movies similar to ones I liked count more than movies similar to ones I hated:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adding sims for Empire Strikes Back, The (1980)...\n",
      "Adding sims for Gone with the Wind (1939)...\n",
      "Adding sims for Star Wars (1977)...\n",
      "sorting...\n",
      "Empire Strikes Back, The (1980)                       5.000000\n",
      "Star Wars (1977)                                      5.000000\n",
      "Empire Strikes Back, The (1980)                       3.741763\n",
      "Star Wars (1977)                                      3.741763\n",
      "Return of the Jedi (1983)                             3.606146\n",
      "Return of the Jedi (1983)                             3.362779\n",
      "Raiders of the Lost Ark (1981)                        2.693297\n",
      "Raiders of the Lost Ark (1981)                        2.680586\n",
      "Austin Powers: International Man of Mystery (1997)    1.887164\n",
      "Sting, The (1973)                                     1.837692\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "simCandidates = pd.Series()\n",
    "for i in range(0, len(myRatings.index)):\n",
    "    print (\"Adding sims for \" + myRatings.index[i] + \"...\")\n",
    "    # Retrieve similar movies to this one that I rated\n",
    "    sims = corrMatrix[myRatings.index[i]].dropna()\n",
    "    # Now scale its similarity by how well I rated this movie\n",
    "    sims = sims.map(lambda x: x * myRatings[i])\n",
    "    # Add the score to the list of similarity candidates\n",
    "    simCandidates = simCandidates.append(sims)\n",
    "    \n",
    "#Glance at our results so far:\n",
    "print (\"sorting...\")\n",
    "simCandidates.sort_values(inplace = True, ascending = False)\n",
    "print (simCandidates.head(10))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "This is starting to look like something useful! Note that some of the same movies came up more than once, because they were similar to more than one movie I rated. We'll use groupby() to add together the scores from movies that show up more than once, so they'll count more:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "simCandidates = simCandidates.groupby(simCandidates.index).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Empire Strikes Back, The (1980)              8.877450\n",
       "Star Wars (1977)                             8.870971\n",
       "Return of the Jedi (1983)                    7.178172\n",
       "Raiders of the Lost Ark (1981)               5.519700\n",
       "Indiana Jones and the Last Crusade (1989)    3.488028\n",
       "Bridge on the River Kwai, The (1957)         3.366616\n",
       "Back to the Future (1985)                    3.357941\n",
       "Sting, The (1973)                            3.329843\n",
       "Cinderella (1950)                            3.245412\n",
       "Field of Dreams (1989)                       3.222311\n",
       "dtype: float64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "simCandidates.sort_values(inplace = True, ascending = False)\n",
    "simCandidates.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "The last thing we have to do is filter out movies I've already rated, as recommending a movie I've already watched isn't helpful:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Return of the Jedi (1983)                    7.178172\n",
       "Raiders of the Lost Ark (1981)               5.519700\n",
       "Indiana Jones and the Last Crusade (1989)    3.488028\n",
       "Bridge on the River Kwai, The (1957)         3.366616\n",
       "Back to the Future (1985)                    3.357941\n",
       "Sting, The (1973)                            3.329843\n",
       "Cinderella (1950)                            3.245412\n",
       "Field of Dreams (1989)                       3.222311\n",
       "Wizard of Oz, The (1939)                     3.200268\n",
       "Dumbo (1941)                                 2.981645\n",
       "dtype: float64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filteredSims = simCandidates.drop(myRatings.index)\n",
    "filteredSims.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "There we have it!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Exercise"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Can you improve on these results? Perhaps a different method or min_periods value on the correlation computation would produce more interesting results.\n",
    "\n",
    "Also, it looks like some movies similar to Gone with the Wind - which I hated - made it through to the final list of recommendations. Perhaps movies similar to ones the user rated poorly should actually be penalized, instead of just scaled down?\n",
    "\n",
    "There are also probably some outliers in the user rating data set - some users may have rated a huge amount of movies and have a disporportionate effect on the results. Go back to earlier lectures to learn how to identify these outliers, and see if removing them improves things.\n",
    "\n",
    "For an even bigger project: we're evaluating the result qualitatively here, but we could actually apply train/test and measure our ability to predict user ratings for movies they've already watched. Whether that's actually a measure of a \"good\" recommendation is debatable, though!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>title</th>\n",
       "      <th>'Til There Was You (1997)</th>\n",
       "      <th>1-900 (1994)</th>\n",
       "      <th>101 Dalmatians (1996)</th>\n",
       "      <th>12 Angry Men (1957)</th>\n",
       "      <th>187 (1997)</th>\n",
       "      <th>2 Days in the Valley (1996)</th>\n",
       "      <th>20,000 Leagues Under the Sea (1954)</th>\n",
       "      <th>2001: A Space Odyssey (1968)</th>\n",
       "      <th>3 Ninjas: High Noon At Mega Mountain (1998)</th>\n",
       "      <th>39 Steps, The (1935)</th>\n",
       "      <th>...</th>\n",
       "      <th>Yankee Zulu (1994)</th>\n",
       "      <th>Year of the Horse (1997)</th>\n",
       "      <th>You So Crazy (1994)</th>\n",
       "      <th>Young Frankenstein (1974)</th>\n",
       "      <th>Young Guns (1988)</th>\n",
       "      <th>Young Guns II (1990)</th>\n",
       "      <th>Young Poisoner's Handbook, The (1995)</th>\n",
       "      <th>Zeus and Roxanne (1997)</th>\n",
       "      <th>unknown</th>\n",
       "      <th>Á köldum klaka (Cold Fever) (1994)</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>title</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>'Til There Was You (1997)</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1-900 (1994)</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101 Dalmatians (1996)</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12 Angry Men (1957)</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>187 (1997)</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1664 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "title                      'Til There Was You (1997)  1-900 (1994)  \\\n",
       "title                                                                \n",
       "'Til There Was You (1997)                        NaN           NaN   \n",
       "1-900 (1994)                                     NaN           NaN   \n",
       "101 Dalmatians (1996)                            NaN           NaN   \n",
       "12 Angry Men (1957)                              NaN           NaN   \n",
       "187 (1997)                                       NaN           NaN   \n",
       "\n",
       "title                      101 Dalmatians (1996)  12 Angry Men (1957)  \\\n",
       "title                                                                   \n",
       "'Til There Was You (1997)                    NaN                  NaN   \n",
       "1-900 (1994)                                 NaN                  NaN   \n",
       "101 Dalmatians (1996)                        1.0                  NaN   \n",
       "12 Angry Men (1957)                          NaN                  1.0   \n",
       "187 (1997)                                   NaN                  NaN   \n",
       "\n",
       "title                      187 (1997)  2 Days in the Valley (1996)  \\\n",
       "title                                                                \n",
       "'Til There Was You (1997)         NaN                          NaN   \n",
       "1-900 (1994)                      NaN                          NaN   \n",
       "101 Dalmatians (1996)             NaN                          NaN   \n",
       "12 Angry Men (1957)               NaN                          NaN   \n",
       "187 (1997)                        NaN                          NaN   \n",
       "\n",
       "title                      20,000 Leagues Under the Sea (1954)  \\\n",
       "title                                                            \n",
       "'Til There Was You (1997)                                  NaN   \n",
       "1-900 (1994)                                               NaN   \n",
       "101 Dalmatians (1996)                                      NaN   \n",
       "12 Angry Men (1957)                                        NaN   \n",
       "187 (1997)                                                 NaN   \n",
       "\n",
       "title                      2001: A Space Odyssey (1968)  \\\n",
       "title                                                     \n",
       "'Til There Was You (1997)                           NaN   \n",
       "1-900 (1994)                                        NaN   \n",
       "101 Dalmatians (1996)                               NaN   \n",
       "12 Angry Men (1957)                                 NaN   \n",
       "187 (1997)                                          NaN   \n",
       "\n",
       "title                      3 Ninjas: High Noon At Mega Mountain (1998)  \\\n",
       "title                                                                    \n",
       "'Til There Was You (1997)                                          NaN   \n",
       "1-900 (1994)                                                       NaN   \n",
       "101 Dalmatians (1996)                                              NaN   \n",
       "12 Angry Men (1957)                                                NaN   \n",
       "187 (1997)                                                         NaN   \n",
       "\n",
       "title                      39 Steps, The (1935)  \\\n",
       "title                                             \n",
       "'Til There Was You (1997)                   NaN   \n",
       "1-900 (1994)                                NaN   \n",
       "101 Dalmatians (1996)                       NaN   \n",
       "12 Angry Men (1957)                         NaN   \n",
       "187 (1997)                                  NaN   \n",
       "\n",
       "title                                     ...                  \\\n",
       "title                                     ...                   \n",
       "'Til There Was You (1997)                 ...                   \n",
       "1-900 (1994)                              ...                   \n",
       "101 Dalmatians (1996)                     ...                   \n",
       "12 Angry Men (1957)                       ...                   \n",
       "187 (1997)                                ...                   \n",
       "\n",
       "title                      Yankee Zulu (1994)  Year of the Horse (1997)  \\\n",
       "title                                                                     \n",
       "'Til There Was You (1997)                 NaN                       NaN   \n",
       "1-900 (1994)                              NaN                       NaN   \n",
       "101 Dalmatians (1996)                     NaN                       NaN   \n",
       "12 Angry Men (1957)                       NaN                       NaN   \n",
       "187 (1997)                                NaN                       NaN   \n",
       "\n",
       "title                      You So Crazy (1994)  Young Frankenstein (1974)  \\\n",
       "title                                                                       \n",
       "'Til There Was You (1997)                  NaN                        NaN   \n",
       "1-900 (1994)                               NaN                        NaN   \n",
       "101 Dalmatians (1996)                      NaN                        NaN   \n",
       "12 Angry Men (1957)                        NaN                        NaN   \n",
       "187 (1997)                                 NaN                        NaN   \n",
       "\n",
       "title                      Young Guns (1988)  Young Guns II (1990)  \\\n",
       "title                                                                \n",
       "'Til There Was You (1997)                NaN                   NaN   \n",
       "1-900 (1994)                             NaN                   NaN   \n",
       "101 Dalmatians (1996)                    NaN                   NaN   \n",
       "12 Angry Men (1957)                      NaN                   NaN   \n",
       "187 (1997)                               NaN                   NaN   \n",
       "\n",
       "title                      Young Poisoner's Handbook, The (1995)  \\\n",
       "title                                                              \n",
       "'Til There Was You (1997)                                    NaN   \n",
       "1-900 (1994)                                                 NaN   \n",
       "101 Dalmatians (1996)                                        NaN   \n",
       "12 Angry Men (1957)                                          NaN   \n",
       "187 (1997)                                                   NaN   \n",
       "\n",
       "title                      Zeus and Roxanne (1997)  unknown  \\\n",
       "title                                                         \n",
       "'Til There Was You (1997)                      NaN      NaN   \n",
       "1-900 (1994)                                   NaN      NaN   \n",
       "101 Dalmatians (1996)                          NaN      NaN   \n",
       "12 Angry Men (1957)                            NaN      NaN   \n",
       "187 (1997)                                     NaN      NaN   \n",
       "\n",
       "title                      Á köldum klaka (Cold Fever) (1994)  \n",
       "title                                                          \n",
       "'Til There Was You (1997)                                 NaN  \n",
       "1-900 (1994)                                              NaN  \n",
       "101 Dalmatians (1996)                                     NaN  \n",
       "12 Angry Men (1957)                                       NaN  \n",
       "187 (1997)                                                NaN  \n",
       "\n",
       "[5 rows x 1664 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corrMatrix2 = userRatings.corr(method='spearman', min_periods=100)\n",
    "corrMatrix2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "title\n",
       "Air Force One (1997)                   4.0\n",
       "Amistad (1997)                         4.0\n",
       "Anna Karenina (1997)                   3.0\n",
       "Apostle, The (1997)                    4.0\n",
       "Apt Pupil (1998)                       5.0\n",
       "As Good As It Gets (1997)              5.0\n",
       "Big Bang Theory, The (1994)            4.0\n",
       "Boogie Nights (1997)                   3.0\n",
       "Career Girls (1997)                    1.0\n",
       "Chairman of the Board (1998)           1.0\n",
       "Chasing Amy (1997)                     3.0\n",
       "Conspiracy Theory (1997)               4.0\n",
       "Contact (1997)                         4.0\n",
       "Dante's Peak (1997)                    3.0\n",
       "Dark City (1998)                       4.0\n",
       "Desperate Measures (1998)              3.0\n",
       "English Patient, The (1996)            3.0\n",
       "Eve's Bayou (1997)                     2.0\n",
       "Evita (1996)                           3.0\n",
       "Flubber (1997)                         2.0\n",
       "Full Monty, The (1997)                 4.0\n",
       "Full Speed (1996)                      2.0\n",
       "G.I. Jane (1997)                       3.0\n",
       "Game, The (1997)                       3.0\n",
       "Gattaca (1997)                         3.0\n",
       "Good Will Hunting (1997)               4.0\n",
       "Great Expectations (1998)              3.0\n",
       "Half Baked (1998)                      1.0\n",
       "Hard Rain (1998)                       3.0\n",
       "Jackal, The (1997)                     3.0\n",
       "Jackie Brown (1997)                    3.0\n",
       "Kull the Conqueror (1997)              2.0\n",
       "Kundun (1997)                          4.0\n",
       "L.A. Confidential (1997)               4.0\n",
       "Liar Liar (1997)                       4.0\n",
       "Life Less Ordinary, A (1997)           3.0\n",
       "Man Who Knew Too Little, The (1997)    3.0\n",
       "Money Talks (1997)                     1.0\n",
       "Mother (1996)                          1.0\n",
       "Nénette et Boni (1996)                 3.0\n",
       "Other Voices, Other Rooms (1997)       3.0\n",
       "Peacemaker, The (1997)                 4.0\n",
       "Phantoms (1998)                        2.0\n",
       "Postman, The (1997)                    4.0\n",
       "Rainmaker, The (1997)                  3.0\n",
       "Replacement Killers, The (1998)        4.0\n",
       "Rosewood (1997)                        2.0\n",
       "Scream (1996)                          2.0\n",
       "Scream 2 (1997)                        2.0\n",
       "Seven Years in Tibet (1997)            4.0\n",
       "Soul Food (1997)                       1.0\n",
       "Sphere (1998)                          4.0\n",
       "Starship Troopers (1997)               3.0\n",
       "Titanic (1997)                         5.0\n",
       "Tomorrow Never Dies (1997)             4.0\n",
       "Twisted (1996)                         3.0\n",
       "Volcano (1997)                         3.0\n",
       "Wag the Dog (1997)                     4.0\n",
       "Wedding Singer, The (1998)             2.0\n",
       "Name: 100, dtype: float64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "myRatings2 = userRatings.loc[100].dropna()\n",
    "myRatings2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adding similar for  Air Force One (1997) ...\n",
      "Adding similar for  Amistad (1997) ...\n",
      "Adding similar for  Anna Karenina (1997) ...\n",
      "Adding similar for  Apostle, The (1997) ...\n",
      "Adding similar for  Apt Pupil (1998) ...\n",
      "Adding similar for  As Good As It Gets (1997) ...\n",
      "Adding similar for  Big Bang Theory, The (1994) ...\n",
      "Adding similar for  Boogie Nights (1997) ...\n",
      "Adding similar for  Career Girls (1997) ...\n",
      "Adding similar for  Chairman of the Board (1998) ...\n",
      "Adding similar for  Chasing Amy (1997) ...\n",
      "Adding similar for  Conspiracy Theory (1997) ...\n",
      "Adding similar for  Contact (1997) ...\n",
      "Adding similar for  Dante's Peak (1997) ...\n",
      "Adding similar for  Dark City (1998) ...\n",
      "Adding similar for  Desperate Measures (1998) ...\n",
      "Adding similar for  English Patient, The (1996) ...\n",
      "Adding similar for  Eve's Bayou (1997) ...\n",
      "Adding similar for  Evita (1996) ...\n",
      "Adding similar for  Flubber (1997) ...\n",
      "Adding similar for  Full Monty, The (1997) ...\n",
      "Adding similar for  Full Speed (1996) ...\n",
      "Adding similar for  G.I. Jane (1997) ...\n",
      "Adding similar for  Game, The (1997) ...\n",
      "Adding similar for  Gattaca (1997) ...\n",
      "Adding similar for  Good Will Hunting (1997) ...\n",
      "Adding similar for  Great Expectations (1998) ...\n",
      "Adding similar for  Half Baked (1998) ...\n",
      "Adding similar for  Hard Rain (1998) ...\n",
      "Adding similar for  Jackal, The (1997) ...\n",
      "Adding similar for  Jackie Brown (1997) ...\n",
      "Adding similar for  Kull the Conqueror (1997) ...\n",
      "Adding similar for  Kundun (1997) ...\n",
      "Adding similar for  L.A. Confidential (1997) ...\n",
      "Adding similar for  Liar Liar (1997) ...\n",
      "Adding similar for  Life Less Ordinary, A (1997) ...\n",
      "Adding similar for  Man Who Knew Too Little, The (1997) ...\n",
      "Adding similar for  Money Talks (1997) ...\n",
      "Adding similar for  Mother (1996) ...\n",
      "Adding similar for  Nénette et Boni (1996) ...\n",
      "Adding similar for  Other Voices, Other Rooms (1997) ...\n",
      "Adding similar for  Peacemaker, The (1997) ...\n",
      "Adding similar for  Phantoms (1998) ...\n",
      "Adding similar for  Postman, The (1997) ...\n",
      "Adding similar for  Rainmaker, The (1997) ...\n",
      "Adding similar for  Replacement Killers, The (1998) ...\n",
      "Adding similar for  Rosewood (1997) ...\n",
      "Adding similar for  Scream (1996) ...\n",
      "Adding similar for  Scream 2 (1997) ...\n",
      "Adding similar for  Seven Years in Tibet (1997) ...\n",
      "Adding similar for  Soul Food (1997) ...\n",
      "Adding similar for  Sphere (1998) ...\n",
      "Adding similar for  Starship Troopers (1997) ...\n",
      "Adding similar for  Titanic (1997) ...\n",
      "Adding similar for  Tomorrow Never Dies (1997) ...\n",
      "Adding similar for  Twisted (1996) ...\n",
      "Adding similar for  Volcano (1997) ...\n",
      "Adding similar for  Wag the Dog (1997) ...\n",
      "Adding similar for  Wedding Singer, The (1998) ...\n",
      "1109 movies added\n",
      "Sorting ...\n",
      "Apt Pupil (1998) | Rating =  5.0\n",
      "As Good As It Gets (1997) | Rating =  5.0\n",
      "Titanic (1997) | Rating =  5.0\n",
      "Wag the Dog (1997) | Rating =  4.0\n",
      "Tomorrow Never Dies (1997) | Rating =  4.0\n",
      "Amistad (1997) | Rating =  4.0\n",
      "Conspiracy Theory (1997) | Rating =  4.0\n",
      "Contact (1997) | Rating =  4.0\n",
      "Full Monty, The (1997) | Rating =  4.0\n",
      "Good Will Hunting (1997) | Rating =  4.0\n",
      "L.A. Confidential (1997) | Rating =  4.0\n",
      "Liar Liar (1997) | Rating =  4.0\n",
      "Peacemaker, The (1997) | Rating =  4.0\n",
      "Seven Years in Tibet (1997) | Rating =  4.0\n",
      "Air Force One (1997) | Rating =  4.0\n"
     ]
    }
   ],
   "source": [
    "simCandidates2 = pd.Series()\n",
    "for i in range(0, len(myRatings2.index)):\n",
    "    print(\"Adding similar for \", myRatings2.index[i], \"...\")\n",
    "    sims2 = corrMatrix2[myRatings2.index[i]].dropna()\n",
    "    sims2 = sims2.map(lambda x:x * myRatings2[i])\n",
    "    simCandidates2 = simCandidates2.append(sims2)\n",
    "print(len(simCandidates2), \"movies added\")\n",
    "print(\"Sorting ...\")\n",
    "simCandidates2.sort_values(inplace=True, ascending=False)\n",
    "for j in range(len(simCandidates2)):\n",
    "    if(simCandidates2[j] >= 4):\n",
    "        print(simCandidates2.index[j], \"| Rating = \",simCandidates2[j])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Air Force One (1997)               19.154904\n",
       "Titanic (1997)                     16.202535\n",
       "Liar Liar (1997)                   15.096780\n",
       "Conspiracy Theory (1997)           14.523379\n",
       "Good Will Hunting (1997)           14.031394\n",
       "Contact (1997)                     12.955731\n",
       "Independence Day (ID4) (1996)      12.913024\n",
       "Game, The (1997)                   12.508720\n",
       "English Patient, The (1996)        12.335691\n",
       "Dante's Peak (1997)                12.201964\n",
       "Volcano (1997)                     12.059653\n",
       "G.I. Jane (1997)                   11.658354\n",
       "Saint, The (1997)                  11.351468\n",
       "L.A. Confidential (1997)           10.964604\n",
       "Rock, The (1996)                   10.882433\n",
       "Return of the Jedi (1983)          10.399094\n",
       "Scream (1996)                      10.017850\n",
       "Toy Story (1995)                    9.625620\n",
       "Murder at 1600 (1997)               9.563984\n",
       "Star Trek: First Contact (1996)     9.305534\n",
       "dtype: float64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "simCandidates2 = simCandidates2.groupby(simCandidates2.index).sum()\n",
    "simCandidates2.sort_values(inplace=True, ascending=False)\n",
    "simCandidates2.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "labels [ 4.  4.  3.  4.  5.  5.  4.  3.  1.  1.  3.  4.  4.  3.  4.  3.  3.  2.\n  3.  2.  4.  2.  3.  3.  3.  4.  3.  1.  3.  3.  3.  2.  4.  4.  4.  3.\n  3.  1.  1.  3.  3.  4.  2.  4.  3.  4.  2.  2.  2.  4.  1.  4.  3.  5.\n  4.  3.  3.  4.  2.] not contained in axis",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-15-9d911e02b4f6>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mfilteredsim2\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msimCandidates2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdrop\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmyRatings2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mC:\\Users\\ACER\\AppData\\Local\\Enthought\\Canopy\\edm\\envs\\User\\lib\\site-packages\\pandas\\core\\generic.py\u001b[0m in \u001b[0;36mdrop\u001b[1;34m(self, labels, axis, level, inplace, errors)\u001b[0m\n\u001b[0;32m   2159\u001b[0m                 \u001b[0mnew_axis\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdrop\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlevel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlevel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0merrors\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2160\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2161\u001b[1;33m                 \u001b[0mnew_axis\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdrop\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0merrors\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2162\u001b[0m             \u001b[0mdropped\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreindex\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[1;33m{\u001b[0m\u001b[0maxis_name\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mnew_axis\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2163\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\ACER\\AppData\\Local\\Enthought\\Canopy\\edm\\envs\\User\\lib\\site-packages\\pandas\\core\\indexes\\base.py\u001b[0m in \u001b[0;36mdrop\u001b[1;34m(self, labels, errors)\u001b[0m\n\u001b[0;32m   3622\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0merrors\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[1;34m'ignore'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3623\u001b[0m                 raise ValueError('labels %s not contained in axis' %\n\u001b[1;32m-> 3624\u001b[1;33m                                  labels[mask])\n\u001b[0m\u001b[0;32m   3625\u001b[0m             \u001b[0mindexer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mindexer\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m~\u001b[0m\u001b[0mmask\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3626\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdelete\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: labels [ 4.  4.  3.  4.  5.  5.  4.  3.  1.  1.  3.  4.  4.  3.  4.  3.  3.  2.\n  3.  2.  4.  2.  3.  3.  3.  4.  3.  1.  3.  3.  3.  2.  4.  4.  4.  3.\n  3.  1.  1.  3.  3.  4.  2.  4.  3.  4.  2.  2.  2.  4.  1.  4.  3.  5.\n  4.  3.  3.  4.  2.] not contained in axis"
     ]
    }
   ],
   "source": [
    "filteredsim2 = simCandidates2.drop(myRatings2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}