python_tutorial_part_3_rule_based_classifier.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Rule-Based Sentiment Classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The code below was written in class, to teach simple Python concepts.\n",
    "No further polishing is provided and the code is not necessarily linear (so a cell does not necessarily follow, a previous cell)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['couthie', 'confidence man', 'definiteness', 'changelessness', 'morally']\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "#-------------------\n",
    "def clean_lexicon():\n",
    "    positive_words= open(\"/Users/mam/CORE/TEACHING/smm/PROJECT-PROBLEMS/pos.swn.txt\", \"r\").readlines()\n",
    "    new_pos_list=[]\n",
    "    for i in positive_words[:5]:\n",
    "        i=i.strip()\n",
    "        #i= i[:-1] # i is a word in the list\n",
    "        i= re.sub(\"_\", \" \", i)\n",
    "        new_pos_list.append(i)\n",
    "    return new_pos_list\n",
    "\n",
    "my_positive_list= clean_lexicon()\n",
    "print my_positive_list[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['couthie', 'confidence man', 'definiteness', 'changelessness', 'morally', 'ethmoidal vein', 'unquestionableness', 'uselessness', 'top-quality', 'good-humoredness']\n",
      "**************************************************\n",
      "['twilight of the gods', 'rumbustious', 'screaming', 'grueling', 'inanimate', 'stern', 'changelessness', 'sugarless', 'order pseudoscorpiones', 'modest']\n"
     ]
    }
   ],
   "source": [
    "# Let's make this function more general so that we can use it to read lexical files,\n",
    "# whether they are positive or negative. To do that, we simply parameterize the function.\n",
    "# What this means is that we make it work with a parameter, which will be a file name that we pass to\n",
    "# the function when we are calling it. Now, this parameter can be either the name of the positive lexicon file\n",
    "# or the name of the negative lexicon file. So, that is a desirable change.\n",
    "\n",
    "import re\n",
    "\n",
    "def clean_lexicon(lex_input):\n",
    "    lex_file_l=open(lex_input, \"r\").readlines()\n",
    "    \n",
    "    new_lex_file_l=[]\n",
    "    for i in lex_file_l:\n",
    "        i=i.strip()\n",
    "        #i= i[:-1] # i is a word in the list\n",
    "        i= re.sub(\"_\", \" \", i)\n",
    "        new_lex_file_l.append(i)\n",
    "    return new_lex_file_l\n",
    "\n",
    "my_positive_list= clean_lexicon(\"/Users/mam/CORE/TEACHING/smm/PROJECT-PROBLEMS/pos.swn.txt\")\n",
    "print my_positive_list[:10]\n",
    "print \"*\"*50\n",
    "my_neg_list= clean_lexicon(\"/Users/mam/CORE/TEACHING/smm/PROJECT-PROBLEMS/neg.swn.txt\")\n",
    "print my_neg_list[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "% of positive:  0.0864\n",
      "% of negative:  0.1653\n"
     ]
    }
   ],
   "source": [
    "# What if we wanted to know the percentages of positive and negative words to the overall words (tokens) in a file.\n",
    "# Let's write some code to do that based on the positive and negative entries we acquired from SentiWordNet:\n",
    "import re\n",
    "\n",
    "def clean_lexicon(lex_input):\n",
    "    lex_file_l=open(lex_input, \"r\").readlines()\n",
    "    \n",
    "    new_lex_file_l=[]\n",
    "    for i in lex_file_l:\n",
    "        i=i.strip()\n",
    "        #i= i[:-1] # i is a word in the list\n",
    "        i= re.sub(\"_\", \" \", i)\n",
    "        new_lex_file_l.append(i)\n",
    "    return new_lex_file_l\n",
    "\n",
    "# Change the path to your local path:\n",
    "pos_lex= clean_lexicon(\"/Users/mam/CORE/TEACHING/smm/PROJECT-PROBLEMS/pos.swn.txt\")\n",
    "neg_lex= clean_lexicon(\"/Users/mam/CORE/TEACHING/smm/PROJECT-PROBLEMS/neg.swn.txt\")\n",
    "\n",
    "\n",
    "# Determine the percentage of positive words in a file:\n",
    "def get_sentiment_diversity(pos_lex, neg_lex, input_file):\n",
    "    '''\n",
    "    just returns some stats about % of pos and neg sentiment in a file...\n",
    "    '''\n",
    "    input_string=open(input_file, \"r\").read().lower()\n",
    "    len_words= float(len(input_string.split()))\n",
    "    pos_count=0\n",
    "    neg_count=0\n",
    "    for w in pos_lex:\n",
    "        pos_count+= input_string.count(w)\n",
    "    for w in neg_lex:\n",
    "        neg_count += input_string.count(w)\n",
    "    return pos_count, neg_count, len_words\n",
    "   \n",
    "# Call the function...\n",
    "input_file=\"/Users/mam/CORE/TEACHING/ssa/git_hub/python_tutorial/hamlet.txt\"\n",
    "pos_count, neg_count, len_words= get_sentiment_diversity(pos_lex, neg_lex, input_file)\n",
    "#-------------------------\n",
    "print \"% of positive: \", round(pos_count/len_words, 4) \n",
    "print \"% of negative: \", round(neg_count/len_words, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['!', '\"', '#', '$', '%', '&', \"'\", '(', ')', '*', '+', ',', '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\\\', ']', '^', '_', '`', '{', '|', '}', '~']\n"
     ]
    }
   ],
   "source": [
    "import string\n",
    "punc = [char for char in string.punctuation]\n",
    "print punc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# What if we wanted to remove all punctuation marks from a file?\n",
    "# There are many ways to do this.\n",
    "# As an introduction to regular expressions and the \"string\" module, let's do something along the following lines:\n",
    "#----------------\n",
    "# Let's take a look at the \"re\" module first. Here's an example:\n",
    "\n",
    "import re\n",
    "s = \" hello \"\n",
    "print \"##\"+ s + \"##\"\n",
    "s2= re.sub(\" \", \"\", s)\n",
    "print \"##\"+ s2 + \"##\"\n",
    "s3=s.strip()\n",
    "print \"##\"+ s3 + \"##\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hey there 654%$21!!!...? $& + ___ | %\n",
      "Hey there 65421     \n"
     ]
    }
   ],
   "source": [
    "\n",
    "import string\n",
    "import re\n",
    "\n",
    "def clean(to_filter_list, text):\n",
    "    '''\n",
    "    input: \n",
    "        a. list of undesirable items we want to remove from text\n",
    "        b. text we want to clean\n",
    "    output:\n",
    "        cleaned text\n",
    "    '''\n",
    "    for i in to_filter_list:\n",
    "        #print i\n",
    "        i=\"\\\\\"+i\n",
    "        text=re.sub(i, \"\", text)\n",
    "    return text\n",
    "\n",
    "#----------------------\n",
    "# Call the function...\n",
    "punc = [char for char in string.punctuation]\n",
    "text=\"Hey there 654%$21!!!...? $& + ___ | %\"\n",
    "\n",
    "new=clean(punc, text)\n",
    "print text\n",
    "print new\n",
    "#print punc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Detecting Retweets & Removing Duplicates..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['@alex Did you make it to the meeting?']\n",
      "['@alex Did you make it to the meeting?']\n"
     ]
    }
   ],
   "source": [
    "# Some work on filtering out undesirable content, for example retweets from a file.\n",
    "# The first step is to do some analysis and understand the structure of a retweet.\n",
    "# Below we assume simply thar a retweet is just a tweet that starts with either \"RT\" or \"rt\"\n",
    "#--------------------------------------------\n",
    "# How do we get red of retweets, for example?\n",
    "# Let's say we have the following list of lines, returned from a file we opened\n",
    "lines=[\"RT @abhi I like #soccer!!!!\", \"rt @abhi I cooked lentil soup\",\\\n",
    "       \"@alex Did you make it to the meeting?\"]\n",
    "\n",
    "new_list=[]\n",
    "for line in lines:\n",
    "    if not line.startswith(\"RT\") and not line.startswith(\"rt\"):\n",
    "        #print line\n",
    "        new_list.append(line)\n",
    "        \n",
    "print new_list\n",
    "\n",
    "clean_list=[line for line in lines if not line.startswith(\"RT\") and not line.startswith(\"rt\")]\n",
    "print clean_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "set(['RT @abhi I like #soccer!!!!', '@alex Did you maaaaake it to the meeting?', '@alex Did you make it there to the meeting?', '@alex Did you make it to the meeting?', 'rt @abhi I cooked lentil soup', '@alex Did you make it to the meeting...', '@alex did you make it to the meeting?'])\n"
     ]
    }
   ],
   "source": [
    "# Now, let's filter out duplicates:\n",
    "lines=[\"RT @abhi I like #soccer!!!!\", \"rt @abhi I cooked lentil soup\",\\\n",
    "       \"@alex Did you make it to the meeting?\",\\\n",
    "       \"@alex Did you make it to the meeting...\",\\\n",
    "      \"@alex Did you make it there to the meeting?\",\\\n",
    "      \"@alex did you make it to the meeting?\",\\\n",
    "      \"@alex Did you maaaaake it to the meeting?\"]\n",
    "\n",
    "print set(lines)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using a Main function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Why don't we now use a main function to call the code we wrote so far?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Welcome to the sentiment statistician!!!\n",
      "% of positive:  0.0864\n",
      "% of negative:  0.1673\n"
     ]
    }
   ],
   "source": [
    "\n",
    "import re\n",
    "import string\n",
    "punc = [char for char in string.punctuation]\n",
    "\n",
    "def clean(to_filter_list, text):\n",
    "    '''\n",
    "    input: \n",
    "        a. list of undesirable items we want to remove from text\n",
    "        b. text we want to clean\n",
    "    output:\n",
    "        cleaned text\n",
    "    '''\n",
    "    for i in to_filter_list:\n",
    "        #print i\n",
    "        i=\"\\\\\"+i\n",
    "        text=re.sub(i, \"\", text)\n",
    "    return text\n",
    "\n",
    "\n",
    "def clean_lexicon(lex_input):\n",
    "    lex_file_l=open(lex_input, \"r\").readlines()\n",
    "    \n",
    "    new_lex_file_l=[]\n",
    "    for i in lex_file_l:\n",
    "        i=i.strip()\n",
    "        #i= i[:-1] # i is a word in the list\n",
    "        i= re.sub(\"_\", \" \", i)\n",
    "        new_lex_file_l.append(i)\n",
    "    return new_lex_file_l\n",
    "\n",
    "\n",
    "# Determine the percentage of positive words in a file:\n",
    "def get_sentiment_diversity(pos_lex, neg_lex, input_file):\n",
    "    '''\n",
    "    just returns some stats about % of pos and neg sentiment in a file...\n",
    "    '''\n",
    "    input_string=open(input_file, \"r\").read().lower()\n",
    "    input_string= clean(punc, input_string)\n",
    "    len_words= float(len(input_string.split()))\n",
    "    pos_count=0\n",
    "    neg_count=0\n",
    "    for w in pos_lex:\n",
    "        pos_count+= input_string.count(w)\n",
    "    for w in neg_lex:\n",
    "        neg_count += input_string.count(w)\n",
    "    return pos_count, neg_count, len_words\n",
    "   \n",
    "def main():\n",
    "    # Call the code...\n",
    "    #------------------\n",
    "    print(\"Welcome to the sentiment statistician!!!\")\n",
    "    # Get the lexicon:\n",
    "    pos_lex= clean_lexicon(\"/Users/mam/CORE/TEACHING/smm/PROJECT-PROBLEMS/pos.swn.txt\")\n",
    "    neg_lex= clean_lexicon(\"/Users/mam/CORE/TEACHING/smm/PROJECT-PROBLEMS/neg.swn.txt\")\n",
    "    # Read the hamlet file\n",
    "    input_file=\"/Users/mam/CORE/TEACHING/ssa/git_hub/python_tutorial/hamlet.txt\"\n",
    "    # get sentiment stats\n",
    "    pos_count, neg_count, len_words= get_sentiment_diversity(pos_lex, neg_lex, input_file)\n",
    "    #-------------------------\n",
    "    print \"% of positive: \", round(pos_count/len_words, 4) \n",
    "    print \"% of negative: \", round(neg_count/len_words, 4)\n",
    "\n",
    "    \n",
    "if __name__ == \"__main__\":\n",
    "    main()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Miscellaneous code to loop over lines from a file, and do something (e.g., counting positive and negative words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<type 'list'>\n",
      "['couthie\\n', 'confidence_man\\n', 'definiteness\\n', 'changelessness\\n', 'morally\\n']\n",
      "5440\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n",
      "Predicted Label= POSITIVE\n"
     ]
    }
   ],
   "source": [
    "lines=open(\"/Users/mam/CORE/TEACHING/smm/PROJECT-PROBLEMS/posTweets.txt\", \"r\").readlines()\n",
    "print type(lines) \n",
    "print positive_words[0:5]\n",
    "print len(positive_words)\n",
    "positive_words=positive_words#+[\"good\"]\n",
    "#print lines[0:5]\n",
    "pos_counter=0\n",
    "for line in lines:\n",
    "    for entry in positive_words:\n",
    "        #print i[:-1]\n",
    "        #break\n",
    "        if entry in line and \"never\" not in line:\n",
    "            #print i\n",
    "            pos_counter+=1\n",
    "    if pos_counter > 1:\n",
    "        print(\"Predicted Label= POSITIVE\")\n",
    "    #else: #pos_counter ==0:\n",
    "     #   print(\"No posiotive words found\")\n",
    "    pos_counter=0\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mixed tweet\t@chelvanderbaan well Idk if Thts good or bad for you ??? But its kinda nice to hear Haha :)\n",
      "\n",
      "positive tweet\n",
      "positive tweet\n",
      "positive tweet\n",
      "positive tweet\n",
      "positive tweet\n",
      "positive tweet\n",
      "positive tweet\n"
     ]
    }
   ],
   "source": [
    "lines=x[:201]\n",
    "from collections import defaultdict\n",
    "d=defaultdict(int)\n",
    "\n",
    "for l in lines:\n",
    "    if \"good\" in l and \"bad\" in l:\n",
    "        print \"mixed tweet\\t\", l\n",
    "    elif \"bad\" in l:\n",
    "        print \"negative tweet\"\n",
    "    elif \"good\" in l:\n",
    "        print \"positive tweet\"\n",
    "    else:\n",
    "        pass #print \"\\t\\tobjective tweet\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n"
     ]
    }
   ],
   "source": [
    "x=open(\"~/Desktop/posTweets.txt\", \"r\").readlines()\n",
    "lines=x[:201]\n",
    "from collections import defaultdict\n",
    "d=defaultdict(int)\n",
    "\n",
    "pos_lex=[\"good\", \"fantastic\", \"wonderful\", \"great\", \"fascinating\", \"pizza\"]\n",
    "neg_lex=[\"bad\", \"ugly\", \"boring\", \"disguting\", \"lazy\"]\n",
    "\n",
    "count_pos=0\n",
    "\n",
    "for l in lines:\n",
    "    for entry in pos_lex:\n",
    "        if entry in l:\n",
    "            count_pos+=1\n",
    "            print count_pos #entry, lines.index(l)\n",
    "    count_pos=0\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}