task1_ML.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "train_set_file_name = \"task1.train.txt\"\n",
    "#test_set_file_name =\"test.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set_file_name = \"task1.train.txt\"\n",
    "\n",
    "#test_set_file_name =\"test.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import xgboost as xgb\n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "stemmer = SnowballStemmer('english')\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of documents in the traning set: 35993\n",
      "Number of labels in the training set: 35993\n",
      "Number of propaganda articles: 4021\n",
      "{'non-propaganda', 'propaganda'}\n",
      "average length of an article is 594.953769\n"
     ]
    }
   ],
   "source": [
    "\n",
    "article_contents,gold_labels = ([],[])\n",
    "article_len = 0\n",
    "with open(train_set_file_name, \"r\", encoding='utf-8') as f:\n",
    "    for line in f.readlines():\n",
    "        article_content, article_id, gold_label = line.rstrip().split(\"\\t\")\n",
    "        article_len += len(article_content.split(' '))\n",
    "        article_contents.append(article_content)\n",
    "        gold_labels.append(gold_label)\n",
    "print(\"Number of documents in the traning set: %d\"%(len(article_contents)))\n",
    "print(\"Number of labels in the training set: %d\"%(len(gold_labels)))\n",
    "print(\"Number of propaganda articles: %d\"%(gold_labels.count('propaganda')))\n",
    "\n",
    "print(set(gold_labels))\n",
    "print(\"average length of an article is %f\" % (article_len/len(article_contents)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of articles in training dataset: 32393\n",
      "Number of propaganda articles in training dataset: 3625\n",
      "Number of articles in testing dataset: 3600\n",
      "Number of propaganda articles in testing dataset: 396\n"
     ]
    }
   ],
   "source": [
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "x_train, x_test, y_train, y_test = train_test_split(article_contents, gold_labels, test_size=0.1)\n",
    "print(\"Number of articles in training dataset: %d\"%(len(x_train)))\n",
    "print(\"Number of propaganda articles in training dataset: %d\"%(y_train.count('propaganda')))\n",
    "print(\"Number of articles in testing dataset: %d\"%(len(x_test)))\n",
    "print(\"Number of propaganda articles in testing dataset: %d\"%(y_test.count('propaganda')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_samples: 32393, n_features: 800864\n",
      "Checking that the number of features in train and test correspond: 800864 - 800864\n"
     ]
    }
   ],
   "source": [
    "vectorizer = TfidfVectorizer(min_df=3, max_df=0.5, max_features=None, \n",
    "                             strip_accents='unicode', analyzer='word', token_pattern=r'\\w{1,}', \n",
    "                             ngram_range=(1,3), use_idf=1, smooth_idf=1, sublinear_tf=1, stop_words='english')\n",
    "\n",
    "train = vectorizer.fit_transform(x_train)\n",
    "print(\"n_samples: %d, n_features: %d\" % train.shape)\n",
    "test = vectorizer.transform(x_test)\n",
    "\n",
    "print(\"Checking that the number of features in train and test correspond: %s - %s\" % (train.shape[1],test.shape[1]))\n",
    "#print(type(train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24.38215046460655\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "cnt = 0\n",
    "for article in x_train:\n",
    "    pattern = re.compile(r'(\\. )|(\\? )|(\\! )')\n",
    "    res = pattern.findall(article)\n",
    "    cnt += len(res)\n",
    "    \n",
    "print((cnt/len(x_train)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision: [0.98227612 0.91927083]\n",
      "recall: [0.99028213 0.86097561]\n",
      "fscore: [0.98626288 0.88916877]\n",
      "support: [3190  410]\n",
      "accuracy: 0.9755555555555555\n"
     ]
    }
   ],
   "source": [
    "#SVM\n",
    "model = LinearSVC(C=1.0, class_weight='balanced', multi_class='ovr', random_state=40)\n",
    "model.fit(train, y_train)\n",
    "predictions = model.predict(test)\n",
    "from sklearn.metrics import precision_recall_fscore_support\n",
    "precision,recall,fscore,support=precision_recall_fscore_support(y_test, predictions)\n",
    "print('precision: {0}'.format(precision))\n",
    "print('recall: {0}'.format(recall))\n",
    "print('fscore: {0}'.format(fscore))\n",
    "print('support: {0}'.format(support))\n",
    "from sklearn.metrics import accuracy_score\n",
    "accuracy = accuracy_score(y_test, predictions)\n",
    "print('accuracy: {0}'.format(accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision: [0.94838902 0.95564516]\n",
      "recall: [0.99655172 0.57804878]\n",
      "fscore: [0.97187404 0.72036474]\n",
      "support: [3190  410]\n",
      "accuracy: 0.9488888888888889\n"
     ]
    }
   ],
   "source": [
    "#LogisticRegression\n",
    "from sklearn.linear_model import LogisticRegression \n",
    " \n",
    "classifier = LogisticRegression() \n",
    "classifier.fit(train, y_train)  \n",
    "\n",
    "predictions = classifier.predict(test) \n",
    "precision,recall,fscore,support=precision_recall_fscore_support(y_test, predictions)\n",
    "print('precision: {0}'.format(precision))\n",
    "print('recall: {0}'.format(recall))\n",
    "print('fscore: {0}'.format(fscore))\n",
    "print('support: {0}'.format(support))\n",
    "accuracy = accuracy_score(y_test, predictions)\n",
    "print('accuracy: {0}'.format(accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision: [0.92571097 1.        ]\n",
      "recall: [1.         0.37560976]\n",
      "fscore: [0.96142254 0.54609929]\n",
      "support: [3190  410]\n",
      "accuracy: 0.9288888888888889\n"
     ]
    }
   ],
   "source": [
    "#RandomForest\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "rf = RandomForestClassifier(oob_score=True, n_estimatsors = 100)\n",
    "rf.fit(train, y_train)\n",
    "predictions = rf.predict(test) \n",
    "precision,recall,fscore,support=precision_recall_fscore_support(y_test, predictions)\n",
    "print('precision: {0}'.format(precision))\n",
    "print('recall: {0}'.format(recall))\n",
    "print('fscore: {0}'.format(fscore))\n",
    "print('support: {0}'.format(support))\n",
    "accuracy = accuracy_score(y_test, predictions)\n",
    "print('accuracy: {0}'.format(accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best k:6, best score:0.9614, best method: distance\n"
     ]
    }
   ],
   "source": [
    "#KNN\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "best_k=-1\n",
    "best_score=0\n",
    "best_method=''\n",
    "for method in ['uniform','distance']:\n",
    "    for i in range(1,11):\n",
    "        knn_clf=KNeighborsClassifier(n_neighbors=i,weights=method)\n",
    "        knn_clf.fit(train,y_train)\n",
    "        scores=knn_clf.score(test,y_test)\n",
    "        if scores>best_score:\n",
    "            best_score=scores\n",
    "            best_k=i\n",
    "            best_method=method\n",
    "\n",
    "print('best k:%d, best score:%.4f, best method: %s'%(best_k,best_score,best_method))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision: [0.96466646 0.92744479]\n",
      "recall: [0.99278997 0.71707317]\n",
      "fscore: [0.97852619 0.8088033 ]\n",
      "support: [3190  410]\n",
      "accuracy: 0.9613888888888888\n"
     ]
    }
   ],
   "source": [
    "knn_clf=KNeighborsClassifier(n_neighbors=6,weights='distance')\n",
    "knn_clf.fit(train,y_train)\n",
    "predictions = knn_clf.predict(test) \n",
    "precision,recall,fscore,support=precision_recall_fscore_support(y_test, predictions)\n",
    "print('precision: {0}'.format(precision))\n",
    "print('recall: {0}'.format(recall))\n",
    "print('fscore: {0}'.format(fscore))\n",
    "print('support: {0}'.format(support))\n",
    "from sklearn.metrics import accuracy_score\n",
    "accuracy = accuracy_score(y_test, predictions)\n",
    "print('accuracy: {0}'.format(accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Use article summaries to do classification\n",
    "from sumy.parsers.html import HtmlParser\n",
    "from sumy.parsers.plaintext import PlaintextParser\n",
    "from sumy.nlp.tokenizers import Tokenizer\n",
    "#from sumy.summarizers.lsa import LsaSummarizer as Summarizer\n",
    "from sumy.summarizers.luhn import LuhnSummarizer as Summarizer\n",
    "#from sumy.summarizers.lex_rank import LexRankSummarizer as Summarizer\n",
    "from sumy.nlp.stemmers import Stemmer\n",
    "from sumy.utils import get_stop_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of articles in training dataset: 28794\n",
      "Number of propaganda articles in training dataset: 3201\n",
      "Number of articles in testing dataset: 720\n",
      "Number of propaganda articles in testing dataset: 74\n",
      "Number of articles in testing dataset: 6479\n",
      "Number of propaganda articles in testing dataset: 746\n"
     ]
    }
   ],
   "source": [
    "train_output_file = '../../data/task_1/train.txt'\n",
    "dev_output_file = '../../data/task_1/dev.txt'\n",
    "test_output_file = '../../data/task_1/test.txt'\n",
    "x_train, x_test, y_train, y_test = train_test_split(article_contents, gold_labels, test_size=0.2)\n",
    "x_test, x_dev, y_test, y_dev = train_test_split(x_test, y_test, test_size=0.1)\n",
    "print(\"Number of articles in training dataset: %d\"%(len(x_train)))\n",
    "print(\"Number of propaganda articles in training dataset: %d\"%(y_train.count('propaganda')))\n",
    "print(\"Number of articles in testing dataset: %d\"%(len(x_dev)))\n",
    "print(\"Number of propaganda articles in testing dataset: %d\"%(y_dev.count('propaganda')))\n",
    "print(\"Number of articles in testing dataset: %d\"%(len(x_test)))\n",
    "print(\"Number of propaganda articles in testing dataset: %d\"%(y_test.count('propaganda')))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import word_tokenize\n",
    "\n",
    "LANGUAGE = \"english\"\n",
    "SENTENCES_COUNT = 5\n",
    "stemmer = Stemmer(LANGUAGE)\n",
    "#parser = PlaintextParser.from_string(\"Check this out.\", Tokenizer(LANGUAGE))\n",
    "summarizer = Summarizer(stemmer)\n",
    "summarizer.stop_words = get_stop_words(LANGUAGE)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_output_file = '../../data/task_1/test.txt'\n",
    "with open(test_output_file,'w') as w:\n",
    "    for (content, label) in zip(x_test, y_test):\n",
    "        parser = PlaintextParser.from_string(content, Tokenizer(LANGUAGE))\n",
    "        summary = \" \".join([str(sentence) for sentence in summarizer(parser.document, SENTENCES_COUNT)])\n",
    "        tokens = word_tokenize(summary)\n",
    "        w.write(\"\\nlabel {}\\n\".format(label))\n",
    "        for token in word_tokenize(summary):\n",
    "            w.write(\"{}\\n\".format(token))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking that the number of features in train_sum and test_sum correspond: 800864 - 800864\n"
     ]
    }
   ],
   "source": [
    "x_train_sum = []\n",
    "x_test_sum = []\n",
    "\n",
    "for content in x_train:\n",
    "    parser = PlaintextParser.from_string(content, Tokenizer(LANGUAGE))\n",
    "    summary = \" \".join([str(sentence) for sentence in summarizer(parser.document, SENTENCES_COUNT)])\n",
    "    x_train_sum.append(summary)\n",
    "\n",
    "for content in x_test:\n",
    "    parser = PlaintextParser.from_string(content, Tokenizer(LANGUAGE))\n",
    "    summary = \" \".join([str(sentence) for sentence in summarizer(parser.document, SENTENCES_COUNT)])\n",
    "    x_test_sum.append(summary)\n",
    "    \n",
    "train_sum = vectorizer.transform(x_train_sum)\n",
    "test_sum = vectorizer.transform(x_test_sum)\n",
    "#print(test)\n",
    "#feature_name = vectorizer.get_feature_names()\n",
    "print(\"Checking that the number of features in train_sum and test_sum correspond: %s - %s\" % (train.shape[1],test.shape[1]))\n",
    "#print(type(train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision: [0.96375921 0.80813953]\n",
      "recall: [0.97940075 0.7020202 ]\n",
      "fscore: [0.97151703 0.75135135]\n",
      "support: [3204  396]\n",
      "accuracy: 0.9488888888888889\n"
     ]
    }
   ],
   "source": [
    "#SVM_sum\n",
    "model = LinearSVC(C=1.0, class_weight='balanced', multi_class='ovr', random_state=40)\n",
    "model.fit(train_sum, y_train)\n",
    "predictions = model.predict(test_sum)\n",
    "from sklearn.metrics import precision_recall_fscore_support\n",
    "precision,recall,fscore,support=precision_recall_fscore_support(y_test, predictions)\n",
    "print('precision: {0}'.format(precision))\n",
    "print('recall: {0}'.format(recall))\n",
    "print('fscore: {0}'.format(fscore))\n",
    "print('support: {0}'.format(support))\n",
    "from sklearn.metrics import accuracy_score\n",
    "accuracy = accuracy_score(y_test, predictions)\n",
    "print('accuracy: {0}'.format(accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision: [0.94300059 0.89252336]\n",
      "recall: [0.99284826 0.49739583]\n",
      "fscore: [0.96728264 0.63879599]\n",
      "support: [3216  384]\n",
      "accuracy: 0.94\n"
     ]
    }
   ],
   "source": [
    "#KNN_sum\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "best_k=-1\n",
    "best_score=0\n",
    "best_method=''\n",
    "knn_clf=KNeighborsClassifier(n_neighbors=6,weights='distance')\n",
    "knn_clf.fit(train_sum,y_train)\n",
    "predictions = knn_clf.predict(test_sum) \n",
    "precision,recall,fscore,support=precision_recall_fscore_support(y_test, predictions)\n",
    "print('precision: {0}'.format(precision))\n",
    "print('recall: {0}'.format(recall))\n",
    "print('fscore: {0}'.format(fscore))\n",
    "print('support: {0}'.format(support))\n",
    "accuracy = accuracy_score(y_test, predictions)\n",
    "print('accuracy: {0}'.format(accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision: [0.92239149 0.94214876]\n",
      "recall: [0.99782338 0.296875  ]\n",
      "fscore: [0.95862584 0.45148515]\n",
      "support: [3216  384]\n",
      "accuracy: 0.9230555555555555\n"
     ]
    }
   ],
   "source": [
    "#LogisticRegression_sum\n",
    "from sklearn.linear_model import LogisticRegression \n",
    " \n",
    "classifier = LogisticRegression() \n",
    "classifier.fit(train_sum, y_train)  \n",
    "\n",
    "predictions = classifier.predict(test_sum) \n",
    "precision,recall,fscore,support=precision_recall_fscore_support(y_test, predictions)\n",
    "print('precision: {0}'.format(precision))\n",
    "print('recall: {0}'.format(recall))\n",
    "print('fscore: {0}'.format(fscore))\n",
    "print('support: {0}'.format(support))\n",
    "accuracy = accuracy_score(y_test, predictions)\n",
    "print('accuracy: {0}'.format(accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision: [0.90950226 1.        ]\n",
      "recall: [1.         0.16666667]\n",
      "fscore: [0.95260664 0.28571429]\n",
      "support: [3216  384]\n",
      "accuracy: 0.9111111111111111\n"
     ]
    }
   ],
   "source": [
    "#RandomForest_sum\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "rf = RandomForestClassifier(oob_score=True, n_estimators = 100)\n",
    "rf.fit(train_sum, y_train)\n",
    "predictions = rf.predict(test_sum) \n",
    "precision,recall,fscore,support=precision_recall_fscore_support(y_test, predictions)\n",
    "print('precision: {0}'.format(precision))\n",
    "print('recall: {0}'.format(recall))\n",
    "print('fscore: {0}'.format(fscore))\n",
    "print('support: {0}'.format(support))\n",
    "accuracy = accuracy_score(y_test, predictions)\n",
    "print('accuracy: {0}'.format(accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}