Project_code_RyanEllison.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:6b728acc798b0e1e745eb9d797073a2e4d4ffa70f9c308875cc3d976975e309d"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import ConfigParser\n",
      "from TwitterAPI import TwitterAPI\n",
      "\n",
      "# This method is done for you.\n",
      "def get_twitter(config_file):\n",
      "    \"\"\" Read the config_file and construct an instance of TwitterAPI.\n",
      "    Args:\n",
      "      config_file ... A config file in ConfigParser format with Twitter credentials\n",
      "    Returns:\n",
      "      An instance of TwitterAPI.\n",
      "    \"\"\"\n",
      " \n",
      "    config = ConfigParser.ConfigParser()\n",
      "    config.read(config_file)\n",
      "    twitter = TwitterAPI(\n",
      "                   config.get('twitter', 'consumer_key'),\n",
      "                   config.get('twitter', 'consumer_secret'),\n",
      "                   config.get('twitter', 'access_token'),\n",
      "                   config.get('twitter', 'access_token_secret'))\n",
      "    return twitter\n",
      "\n",
      "twitter = get_twitter('twitter.cfg')\n",
      "print 'Established Twitter connection.'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Established Twitter connection.\n"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#save streamed tweets to a file\n",
      "#each (full) file contains 10,000 tweets\n",
      "\n",
      "import pickle\n",
      "from datetime import datetime\n",
      "import sys\n",
      "import time\n",
      "\n",
      "#playing_movies_list = ['Interstellar']\n",
      "#playing_movies_list = ['Mockingjay', 'The Hunger Games']\n",
      "playing_movies_list = ['John Wick']\n",
      "#playing_movies_list = ['Penguins of Madagascar', 'Penguins Madagascar']\n",
      "#playing_movies_list = ['Theory of Everything']\n",
      "#playing_movies_list = ['Dumb and Dumber', 'Dumb and Dumber To', 'Dumb and Dumber 2']\n",
      "#playing_movies_list = ['Big Hero', 'Big Hero 2']\n",
      "\n",
      "#this is a general movie list for training and test data...\n",
      "#playing_movies_list = ['Movie', 'Movies', 'Dumb and Dumber', 'Big Hero', 'Interstellar', 'Mockingjay', 'The Hunger Games', 'John Wick', 'Penguins of Madagascar']\n",
      "\n",
      "#file_name = \"tweets \" + str(datetime.now()).split('.')[0].replace(':','_') + \".bin\"\n",
      "file_name = \"tweets \" + str(datetime.now()).split('.')[0].replace(':','_') + \"_\" + playing_movies_list[0] + \"_.bin\"\n",
      "#file_name = \"tweets \" + str(datetime.now()).split('.')[0].replace(':','_') + \"_AllGeneralMovies\" + \"_.bin\"\n",
      "\n",
      "print \"starting new file: \" + file_name\n",
      "\n",
      "r = twitter.request('statuses/filter', {'track':playing_movies_list, 'filter_level':'low'})\n",
      "\n",
      "tweet_number = 1\n",
      "\n",
      "while True:\n",
      "    try:\n",
      "        for tweet in r:\n",
      "            print \"saved tweet %d from %s\" % (tweet_number, tweet['user']['name'])\n",
      "            pickle.dump(tweet, open(file_name, 'ab'))\n",
      "            tweet_number += 1\n",
      "            if tweet_number > 10000:\n",
      "                tweet_number = 1\n",
      "                file_name = \"tweets \" + str(datetime.now()).split('.')[0].replace(':','_') + \".bin\"\n",
      "                print \"starting new file: \" + file_name\n",
      "    except:\n",
      "        print >> sys.stderr, 'Streaming error...trying again in 30 seconds.'\n",
      "        sys.stderr.flush()\n",
      "        time.sleep(30)  "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "starting new file: tweets 2014-12-01 05_25_02_John Wick_.bin\n",
        "saved tweet 1 from Atilla Akal\u0131n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 2 from Holographics.TV"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 3 from Bj\u00f6rn"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 4 from Meral Yaz\u0131rl\u0131o\u011flu"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 5 from Diogo"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 6 from Sezer Ak\u00e7al\u0131"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 7 from Esra Koclu"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 8 from weozer"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 9 from ecki"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 10 from ksuha_mai"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 11 from \u00f6mer mert"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 12 from Bekir Emre G\u00fcll\u00fc"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 13 from keanu reeves latinos"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 14 from sebsan39"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 15 from Munta Mahdi"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 16 from tom"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 17 from \u00d6m\u00fcrcan"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 18 from Nazeer"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 19 from Andy Nguyen"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 20 from Hilal"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 21 from Musa Ayak"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 22 from \u200d\u200d\u200d"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 23 from mistermurano"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 24 from Heval H\u00fcseyin Azimli"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 25 from Thiemi Okawara"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 26 from Christian Duchovny"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 27 from zuka"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 28 from Tuba E."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 29 from Alan J. Taylor"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 30 from Alan J. Taylor"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 31 from \u2654 HAKAN"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 32 from Jack Burton, Me"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 33 from Illusionsprojektor"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 34 from Nurcan EYG\u0130"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 35 from Emily Stevens"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 36 from Nurcan EYG\u0130"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 37 from Carlo Su\u00f1ga"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "saved tweet 38 from \u042e\u043b\u044f \u0428\u0443\u0431\u043a\u0430"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Streaming error...trying again in 30 seconds.\n"
       ]
      }
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from os import listdir\n",
      "from os import getcwd\n",
      "from os.path import isfile, join\n",
      "import pprint, pickle\n",
      "\n",
      "#get a list of all files with saved tweets\n",
      "#used two pages for creating this code...\n",
      "#http://stackoverflow.com/questions/3430372/how-to-get-full-path-of-current-directory-in-python\n",
      "#http://stackoverflow.com/questions/3207219/how-to-list-all-files-of-a-directory-in-python\n",
      "def get_filenames():\n",
      "    tweet_files = [ f for f in listdir(getcwd()) if isfile(join(getcwd(),f)) and \"tweets\" in f and \".bin\" in f]\n",
      "    return tweet_files\n",
      "\n",
      "def get_specific_filenames(nameEnding):\n",
      "    tweet_files = [ f for f in listdir(getcwd()) if isfile(join(getcwd(),f)) and \"tweets\" in f and nameEnding in f]\n",
      "    return tweet_files\n",
      "\n",
      "#create a mechanism to label/save a supervised set of tweets...\n",
      "def classify_tweet(tweet, loaded_tweet_num):\n",
      "    #ignore retweets\n",
      "    if tweet['text'][:2] == \"RT\":\n",
      "        return 4 #automatically ignored\n",
      "    if \"#sex\" in tweet['text'] or \"#porn\" in tweet['text']:\n",
      "        return 5 #automatically filtered\n",
      "    print str(loaded_tweet_num) + \") Tweet by user: \" + tweet['user']['name'] + \" on \" + tweet['created_at'].split('+')[0]\n",
      "    print tweet['text']\n",
      "    classification = input(\"Enter label (0 = neg, 1 = pos, 2 = ignore tweet, 3 = stop classifying): \")\n",
      "    while classification not in (0, 1, 2, 3):\n",
      "        print \"Incorrect input. Enter 0 for negative, 1 for positive, 2 for ignore tweet, 3 to stop classifying.\"\n",
      "        classification = input(\"Enter label (0 = neg, 1 = pos, 2 = ignore tweet, 3 = stop classifying): \")\n",
      "    return classification\n",
      "\n",
      "#save tweets and their label to a file\n",
      "def save_labeled_tweet(tweet, label):\n",
      "    if label == 1:\n",
      "        labeled_tweets_filename = \"labeled_tweet_s.bin\"\n",
      "        labeled_tweet = {'tweet':tweet, 'sentiment_label':label}\n",
      "        pickle.dump(labeled_tweet, open(labeled_tweets_filename, 'ab'))\n",
      "        print \"positive tweet saved\"\n",
      "    elif label == 0:\n",
      "        print \"negative tweet saved\"\n",
      "        labeled_tweets_filename = \"labeled_tweet_s.bin\"\n",
      "        labeled_tweet = {'tweet':tweet, 'sentiment_label':label}\n",
      "        pickle.dump(labeled_tweet, open(labeled_tweets_filename, 'ab'))\n",
      "\n",
      "#print tweet, formatted nicely\n",
      "def print_tweet(tweet, loaded_tweet_num):\n",
      "    print str(loaded_tweet_num) + \") Tweet by user: \" + tweet['user']['name'] + \" on \" + tweet['created_at'].split('+')[0]\n",
      "    print tweet['text']\n",
      "    \n",
      "#read in saved tweets, manually classify them, and save results to a new file...\n",
      "def manually_classify_tweets_files():\n",
      "    #files = get_filenames()\n",
      "    #print \"Tweet Files Located:\"\n",
      "    #print '\\n'.join(files)\n",
      "    \n",
      "    files = get_specific_filenames(\"_AllGeneralMovies_.bin\")\n",
      "    print files\n",
      "    \n",
      "    #A number of tweets to skip before classification prompts\n",
      "    #This is useful to resume classification if inturrupted\n",
      "    skipct = 1500\n",
      "    \n",
      "    classify = True\n",
      "    #classify = False #just print the tweet\n",
      "\n",
      "    stop_now = False #always start with this false\n",
      "    #stop_now = True #this is only for testing...\n",
      "    \n",
      "    for file_to_open in files:\n",
      "        print \"--------------------------------------------\"\n",
      "        print \"opening file: \" + file_to_open\n",
      "        print \"--------------------------------------------\"\n",
      "        pkl_file = open(file_to_open, 'rb')\n",
      "        loaded_tweet_num = 1\n",
      "        while True:\n",
      "            if skipct > 0:\n",
      "                tweet = pickle.load(pkl_file)\n",
      "                skipct -= 1\n",
      "                continue\n",
      "            try:\n",
      "                tweet = pickle.load(pkl_file)\n",
      "                if tweet['lang'] != 'en':\n",
      "                    continue\n",
      "                if classify:\n",
      "                    classification = classify_tweet(tweet, loaded_tweet_num) \n",
      "                    if classification == 2:\n",
      "                        print \"ignoring tweet\"\n",
      "                    elif classification == 1:\n",
      "                        save_labeled_tweet(tweet, 1)\n",
      "                    elif classification == 0:\n",
      "                        save_labeled_tweet(tweet, 0)\n",
      "                    elif classification == 3:\n",
      "                        stop_now = True\n",
      "                    elif classification == 4:\n",
      "                        print \"auto-ignore\"\n",
      "                    elif classification == 5:\n",
      "                        print \"auto-filter\"\n",
      "                    else:\n",
      "                        print \"classification error!\"\n",
      "                else:\n",
      "                    print_tweet(tweet, loaded_tweet_num)\n",
      "                loaded_tweet_num += 1\n",
      "            except EOFError:\n",
      "                break\n",
      "            if stop_now:\n",
      "                break\n",
      "        if stop_now:\n",
      "            print \"stopping classification\"\n",
      "            break\n",
      "\n",
      "print \"Beginning manual classification...\"\n",
      "manually_classify_tweets_files()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Beginning manual classification...\n",
        "['tweets 2014-11-30 17_12_16_AllGeneralMovies_.bin']\n",
        "--------------------------------------------\n",
        "opening file: tweets 2014-11-30 17_12_16_AllGeneralMovies_.bin\n",
        "--------------------------------------------\n",
        "auto-ignore"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "auto-ignore\n",
        "3) Tweet by user: Susan lawal on Sun Nov 30 23:14:48 \n",
        "I liked a @YouTube video http://t.co/NoVoMjsR5s The Hanging Tree - The Hunger Games Mockingjay Part 1 Score James Newton Howard\n"
       ]
      },
      {
       "name": "stdout",
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Enter label (0 = neg, 1 = pos, 2 = ignore tweet, 3 = stop classifying): 3\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "stopping classification\n"
       ]
      }
     ],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#build a classification model with logistic regression\n",
      "\n",
      "from sklearn.linear_model import LogisticRegression\n",
      "from collections import Counter\n",
      "import numpy as np \n",
      "import pickle\n",
      "from sklearn.feature_extraction.text import CountVectorizer\n",
      "from sklearn.cross_validation import KFold\n",
      "from zipfile import ZipFile\n",
      "\n",
      "#this is used to print tweets and their label (from manually labelled)\n",
      "#this is used as a sanity check, when necessary...\n",
      "def print_tweets_with_label():\n",
      "    pkl_file = open('labeled_tweet_s.bin', 'rb')\n",
      "    loaded_tweet_num = 1\n",
      "    while True:\n",
      "        try:\n",
      "            tweet = pickle.load(pkl_file)\n",
      "            print \"<\",\n",
      "            print tweet['tweet']['user']['name'],\n",
      "            print \"- Rating =\",\n",
      "            print tweet['sentiment_label'],\n",
      "            print \">\"\n",
      "            print tweet['tweet']['text']\n",
      "        except EOFError:\n",
      "            break\n",
      "\n",
      "#Convert tweets from training set to vector\n",
      "def tweets_to_feature_vectors(vectorizer):\n",
      "    pkl_file = open('labeled_tweet_s.bin', 'rb')\n",
      "    loaded_tweet_num = 1\n",
      "    tweets = []\n",
      "    while True:\n",
      "        try:\n",
      "            tweet = pickle.load(pkl_file)\n",
      "            tweets.append(tweet['tweet'])\n",
      "        except EOFError:\n",
      "            break\n",
      "    X = vectorizer.fit_transform(t['text'] for t in tweets)\n",
      "    return X\n",
      "\n",
      "#create labels from training set\n",
      "def create_label_vectors():\n",
      "    pkl_file = open('labeled_tweet_s.bin', 'rb')\n",
      "    loaded_tweet_num = 1\n",
      "    tweets = []\n",
      "    while True:\n",
      "        try:\n",
      "            tweet = pickle.load(pkl_file)\n",
      "            tweets.append(tweet)\n",
      "        except EOFError:\n",
      "            break\n",
      "    y = np.array([t['sentiment_label'] for t in tweets])        \n",
      "    return y\n",
      "\n",
      "vectorizer = CountVectorizer()\n",
      "tweets_vector = tweets_to_feature_vectors(vectorizer)\n",
      "print 'vectorized %d tweets. found %d terms.' % (tweets_vector.shape[0], tweets_vector.shape[1])\n",
      "\n",
      "y = create_label_vectors()\n",
      "print 'label counts=', Counter(y)\n",
      "\n",
      "#build model with logicstic regression from training set\n",
      "model = LogisticRegression()\n",
      "model.fit(tweets_vector, y)\n",
      "print model\n",
      "\n",
      "#this is a copy of the accuracy function provided in lecture. This reports the\n",
      "#model fits the training data very well (no surprise there)\n",
      "def accuracy(truth, predicted):\n",
      "    return (1. * len([1 for tr, pr in zip(truth, predicted) if tr == pr]) / len(truth))\n",
      "\n",
      "predicted = model.predict(tweets_vector)\n",
      "print 'accuracy on training data=%.3f' % accuracy(y, predicted)\n",
      "#print tweets_vector[0]\n",
      "#print predicted\n",
      "\n",
      "# 5 Cross-validation accuracy\n",
      "X = tweets_vector\n",
      "cv = KFold(len(y), 5)\n",
      "accuracies = []\n",
      "for train_ind, test_ind in cv:\n",
      "    model.fit(X[train_ind], y[train_ind])\n",
      "    predictions = model.predict(X[test_ind])\n",
      "    accuracies.append(accuracy(y[test_ind], predictions))\n",
      "    \n",
      "print 'Average 5-fold cross validation accuracy=%.2f (std=%.2f)' % (np.mean(accuracies), np.std(accuracies))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "vectorized 211 tweets. found 922 terms.\n",
        "label counts="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " Counter({1: 118, 0: 93})\n",
        "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
        "          intercept_scaling=1, penalty=l2, random_state=None, tol=0.0001)\n",
        "accuracy on training data=1.000\n",
        "Average 5-fold cross validation accuracy=0.70 (std=0.08)\n"
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def load_unlabeled_tweets(file_specifier):\n",
      "    files = get_specific_filenames(file_specifier)\n",
      "    tweets = []\n",
      "    for file in files:\n",
      "        pkl_file = open(file, 'rb')\n",
      "        more = True\n",
      "        while more:\n",
      "            try:\n",
      "                tweet = pickle.load(pkl_file)\n",
      "                if tweet['text'][:2] != \"RT\":\n",
      "                    tweets.append(tweet['text'])\n",
      "            except EOFError:\n",
      "                more = False\n",
      "    return tweets\n",
      "\n",
      "movie_files = ['_Interstellar_.bin', '_Mockingjay_.bin', '_John Wick_.bin', '_Penguins of Madagascar_.bin', '_Big Hero_.bin']\n",
      "\n",
      "for movie in movie_files:\n",
      "    unlabeled_tweets = load_unlabeled_tweets(movie)\n",
      "    print \"------------------\"\n",
      "    print movie + \" [\" + str(len(unlabeled_tweets)) + \" tweets]\"\n",
      "    print \"------------------\"\n",
      "    \n",
      "    unlabeled_tweets_vector = vectorizer.transform(t for t in unlabeled_tweets)\n",
      "    unsupered_pred = model.predict(unlabeled_tweets_vector)\n",
      "    positive = []\n",
      "    negative = []\n",
      "    \n",
      "    for i in range(len(unlabeled_tweets)):\n",
      "        if unsupered_pred[i] == 1:\n",
      "            positive.append(unlabeled_tweets[i])\n",
      "        else:\n",
      "            negative.append(unlabeled_tweets[i])\n",
      "    print len(positive) / (1. * len(positive) + len(negative))\n",
      "    \n",
      "#Rotten Tomatoes Ratings:\n",
      "#Big Hero = 89%\n",
      "#John Wick = 84%\n",
      "#Interstellar = 73%\n",
      "#Penguins of Madagascar = 68%\n",
      "#Mockingjay = 66%\n",
      "\n",
      "#IMDb\n",
      "#Interstellar = 89%\n",
      "#Big Hero = 83%\n",
      "#John Wick = 78%\n",
      "#Penguins of Madagascar = 74%\n",
      "#Mockingjay = 73%\n",
      "\n",
      "#Metacritic\n",
      "#Big Hero = 75%\n",
      "#Interstellar = 74%\n",
      "#John Wick = 67%\n",
      "#Mockingjay = 64%\n",
      "#Penguins of Madagascar = 53%\n",
      "\n",
      "#Average (Rotten Tomatoes, IMDb, Metacritic)\n",
      "#Big Hero = 82%\n",
      "#Interstellar = 79%\n",
      "#John Wick = 76%\n",
      "#Mockingjay = 68%\n",
      "#Penguins of Madagascar = 65%\n",
      "\n",
      "#Twitter Ratings:\n",
      "#Big Hero = 90%\n",
      "#Penguins of Madagascar = 90%\n",
      "#Interstellar = 86%\n",
      "#Mockingjay = 83%\n",
      "#John Wick = 75%\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "------------------\n",
        "_Interstellar_.bin [961 tweets]\n",
        "------------------\n",
        "0.857440166493\n",
        "------------------"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "_Mockingjay_.bin [711 tweets]\n",
        "------------------\n",
        "0.834036568214\n",
        "------------------"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "_John Wick_.bin [313 tweets]\n",
        "------------------\n",
        "0.750798722045\n",
        "------------------"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "_Penguins of Madagascar_.bin [815 tweets]\n",
        "------------------\n",
        "0.901840490798\n",
        "------------------"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "_Big Hero_.bin [944 tweets]\n",
        "------------------\n",
        "0.895127118644\n"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#This is a modified version of code from the lectures to view\n",
      "#the top-weighted terms for the positive and negative classes (good movie/bad movie)\n",
      "vocab = np.array(vectorizer.get_feature_names())\n",
      "coef = model.coef_[0]\n",
      "\n",
      "# Sort pos/neg\n",
      "top_coef_ind = np.argsort(coef)[::-1]\n",
      "bottom_coef_ind = np.argsort(coef)\n",
      "\n",
      "# Get the names of those features.\n",
      "top_coef_terms = vocab[top_coef_ind]\n",
      "bottom_coef_terms = vocab[bottom_coef_ind]\n",
      "\n",
      "# Get the weights of those features\n",
      "top_coef = coef[top_coef_ind]\n",
      "bottom_coef = coef[bottom_coef_ind]\n",
      "\n",
      "# Print the top 20.\n",
      "print 'top weighted terms for positive class:\\n', \\\n",
      "    '\\n'.join('%s %.2f' % (term, weight) for term, weight in zip(top_coef_terms, top_coef)[:20])\n",
      "print \"\"\n",
      "print 'top weighted terms for negative class:\\n', \\\n",
      "    '\\n'.join('%s %.2f' % (term, weight) for term, weight in zip(bottom_coef_terms, bottom_coef)[:20])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "top weighted terms for positive class:\n",
        "good 1.08\n",
        "love 1.04\n",
        "again 0.91\n",
        "one 0.86\n",
        "favorite 0.60\n",
        "loved 0.56\n",
        "wow 0.53\n",
        "day 0.50\n",
        "hot 0.50\n",
        "wonderful 0.49\n",
        "great 0.48\n",
        "watch 0.47\n",
        "was 0.45\n",
        "rn 0.43\n",
        "cool 0.40\n",
        "funniest 0.40\n",
        "who 0.40\n",
        "both 0.39\n",
        "awesome 0.38\n",
        "actually 0.37\n",
        "\n",
        "top weighted terms for negative class:\n",
        "why -0.90\n",
        "some -0.74\n",
        "but -0.72\n",
        "to -0.71\n",
        "they -0.71\n",
        "this -0.70\n",
        "didn -0.69\n",
        "that -0.68\n",
        "don -0.68\n",
        "movies -0.67\n",
        "from -0.64\n",
        "in -0.64\n",
        "fuck -0.62\n",
        "asshole -0.56\n",
        "on -0.54\n",
        "damn -0.53\n",
        "have -0.51\n",
        "waste -0.51\n",
        "not -0.48\n",
        "most -0.48\n"
       ]
      }
     ],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}