TextPreprocessing.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "TextPreprocessing.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMcWkVTUh9zr7oVwHGIU2Mr",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Isurie/Text-Classification-Module/blob/master/TextPreprocessing.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nWIddJtszatL"
      },
      "source": [
        "pip install sinling"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G3gq4hbgzpbC"
      },
      "source": [
        "pip install advertools"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MeXSB5FcZy9f"
      },
      "source": [
        "pip install translate"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f0oYF8Tp-MYk"
      },
      "source": [
        "pip install pyenchant"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "15vfbtvabk4g"
      },
      "source": [
        "from sinling import SinhalaTokenizer as tokenizer,SinhalaStemmer as stemmer, POSTagger,preprocess, word_joiner,word_splitter\n",
        "from nltk.tokenize import sent_tokenize,word_tokenize,TweetTokenizer\n",
        "from nltk.probability import FreqDist\n",
        "import advertools as adv\n",
        "from pathlib import Path\n",
        "import string\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.model_selection import train_test_split, GridSearchCV\n",
        "from sklearn.pipeline import Pipeline\n",
        "from sklearn import metrics\n",
        "from sklearn.metrics import accuracy_score, recall_score, precision_score, confusion_matrix\n",
        "\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer,CountVectorizer\n",
        "from sklearn.naive_bayes import MultinomialNB,GaussianNB,BernoulliNB\n",
        "from sklearn import linear_model\n",
        "\n",
        "import codecs\n",
        "import nltk\n",
        "nltk.download('punkt')\n",
        "import re\n",
        "\n",
        "from nltk.corpus import stopwords\n",
        "from collections import Counter\n",
        "\n",
        "from nltk.tokenize.treebank import TreebankWordDetokenizer as Detok\n",
        "from nltk.tokenize.treebank import TreebankWordDetokenizer"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7CgpmdW5jdGlvbiBfdXBsb2FkRmlsZXMoaW5wdXRJZCwgb3V0cHV0SWQpIHsKICBjb25zdCBzdGVwcyA9IHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCk7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICAvLyBDYWNoZSBzdGVwcyBvbiB0aGUgb3V0cHV0RWxlbWVudCB0byBtYWtlIGl0IGF2YWlsYWJsZSBmb3IgdGhlIG5leHQgY2FsbAogIC8vIHRvIHVwbG9hZEZpbGVzQ29udGludWUgZnJvbSBQeXRob24uCiAgb3V0cHV0RWxlbWVudC5zdGVwcyA9IHN0ZXBzOwoKICByZXR1cm4gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpOwp9CgovLyBUaGlzIGlzIHJvdWdobHkgYW4gYXN5bmMgZ2VuZXJhdG9yIChub3Qgc3VwcG9ydGVkIGluIHRoZSBicm93c2VyIHlldCksCi8vIHdoZXJlIHRoZXJlIGFyZSBtdWx0aXBsZSBhc3luY2hyb25vdXMgc3RlcHMgYW5kIHRoZSBQeXRob24gc2lkZSBpcyBnb2luZwovLyB0byBwb2xsIGZvciBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcC4KLy8gVGhpcyB1c2VzIGEgUHJvbWlzZSB0byBibG9jayB0aGUgcHl0aG9uIHNpZGUgb24gY29tcGxldGlvbiBvZiBlYWNoIHN0ZXAsCi8vIHRoZW4gcGFzc2VzIHRoZSByZXN1bHQgb2YgdGhlIHByZXZpb3VzIHN0ZXAgYXMgdGhlIGlucHV0IHRvIHRoZSBuZXh0IHN0ZXAuCmZ1bmN0aW9uIF91cGxvYWRGaWxlc0NvbnRpbnVlKG91dHB1dElkKSB7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICBjb25zdCBzdGVwcyA9IG91dHB1dEVsZW1lbnQuc3RlcHM7CgogIGNvbnN0IG5leHQgPSBzdGVwcy5uZXh0KG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSk7CiAgcmV0dXJuIFByb21pc2UucmVzb2x2ZShuZXh0LnZhbHVlLnByb21pc2UpLnRoZW4oKHZhbHVlKSA9PiB7CiAgICAvLyBDYWNoZSB0aGUgbGFzdCBwcm9taXNlIHZhbHVlIHRvIG1ha2UgaXQgYXZhaWxhYmxlIHRvIHRoZSBuZXh0CiAgICAvLyBzdGVwIG9mIHRoZSBnZW5lcmF0b3IuCiAgICBvdXRwdXRFbGVtZW50Lmxhc3RQcm9taXNlVmFsdWUgPSB2YWx1ZTsKICAgIHJldHVybiBuZXh0LnZhbHVlLnJlc3BvbnNlOwogIH0pOwp9CgovKioKICogR2VuZXJhdG9yIGZ1bmN0aW9uIHdoaWNoIGlzIGNhbGxlZCBiZXR3ZWVuIGVhY2ggYXN5bmMgc3RlcCBvZiB0aGUgdXBsb2FkCiAqIHByb2Nlc3MuCiAqIEBwYXJhbSB7c3RyaW5nfSBpbnB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIGlucHV0IGZpbGUgcGlja2VyIGVsZW1lbnQuCiAqIEBwYXJhbSB7c3RyaW5nfSBvdXRwdXRJZCBFbGVtZW50IElEIG9mIHRoZSBvdXRwdXQgZGlzcGxheS4KICogQHJldHVybiB7IUl0ZXJhYmxlPCFPYmplY3Q+fSBJdGVyYWJsZSBvZiBuZXh0IHN0ZXBzLgogKi8KZnVuY3Rpb24qIHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IGlucHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKGlucHV0SWQpOwogIGlucHV0RWxlbWVudC5kaXNhYmxlZCA9IGZhbHNlOwoKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIG91dHB1dEVsZW1lbnQuaW5uZXJIVE1MID0gJyc7CgogIGNvbnN0IHBpY2tlZFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgaW5wdXRFbGVtZW50LmFkZEV2ZW50TGlzdGVuZXIoJ2NoYW5nZScsIChlKSA9PiB7CiAgICAgIHJlc29sdmUoZS50YXJnZXQuZmlsZXMpOwogICAgfSk7CiAgfSk7CgogIGNvbnN0IGNhbmNlbCA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2J1dHRvbicpOwogIGlucHV0RWxlbWVudC5wYXJlbnRFbGVtZW50LmFwcGVuZENoaWxkKGNhbmNlbCk7CiAgY2FuY2VsLnRleHRDb250ZW50ID0gJ0NhbmNlbCB1cGxvYWQnOwogIGNvbnN0IGNhbmNlbFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgY2FuY2VsLm9uY2xpY2sgPSAoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9OwogIH0pOwoKICAvLyBXYWl0IGZvciB0aGUgdXNlciB0byBwaWNrIHRoZSBmaWxlcy4KICBjb25zdCBmaWxlcyA9IHlpZWxkIHsKICAgIHByb21pc2U6IFByb21pc2UucmFjZShbcGlja2VkUHJvbWlzZSwgY2FuY2VsUHJvbWlzZV0pLAogICAgcmVzcG9uc2U6IHsKICAgICAgYWN0aW9uOiAnc3RhcnRpbmcnLAogICAgfQogIH07CgogIGNhbmNlbC5yZW1vdmUoKTsKCiAgLy8gRGlzYWJsZSB0aGUgaW5wdXQgZWxlbWVudCBzaW5jZSBmdXJ0aGVyIHBpY2tzIGFyZSBub3QgYWxsb3dlZC4KICBpbnB1dEVsZW1lbnQuZGlzYWJsZWQgPSB0cnVlOwoKICBpZiAoIWZpbGVzKSB7CiAgICByZXR1cm4gewogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgICAgfQogICAgfTsKICB9CgogIGZvciAoY29uc3QgZmlsZSBvZiBmaWxlcykgewogICAgY29uc3QgbGkgPSBkb2N1bWVudC5jcmVhdGVFbGVtZW50KCdsaScpOwogICAgbGkuYXBwZW5kKHNwYW4oZmlsZS5uYW1lLCB7Zm9udFdlaWdodDogJ2JvbGQnfSkpOwogICAgbGkuYXBwZW5kKHNwYW4oCiAgICAgICAgYCgke2ZpbGUudHlwZSB8fCAnbi9hJ30pIC0gJHtmaWxlLnNpemV9IGJ5dGVzLCBgICsKICAgICAgICBgbGFzdCBtb2RpZmllZDogJHsKICAgICAgICAgICAgZmlsZS5sYXN0TW9kaWZpZWREYXRlID8gZmlsZS5sYXN0TW9kaWZpZWREYXRlLnRvTG9jYWxlRGF0ZVN0cmluZygpIDoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ24vYSd9IC0gYCkpOwogICAgY29uc3QgcGVyY2VudCA9IHNwYW4oJzAlIGRvbmUnKTsKICAgIGxpLmFwcGVuZENoaWxkKHBlcmNlbnQpOwoKICAgIG91dHB1dEVsZW1lbnQuYXBwZW5kQ2hpbGQobGkpOwoKICAgIGNvbnN0IGZpbGVEYXRhUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICAgIGNvbnN0IHJlYWRlciA9IG5ldyBGaWxlUmVhZGVyKCk7CiAgICAgIHJlYWRlci5vbmxvYWQgPSAoZSkgPT4gewogICAgICAgIHJlc29sdmUoZS50YXJnZXQucmVzdWx0KTsKICAgICAgfTsKICAgICAgcmVhZGVyLnJlYWRBc0FycmF5QnVmZmVyKGZpbGUpOwogICAgfSk7CiAgICAvLyBXYWl0IGZvciB0aGUgZGF0YSB0byBiZSByZWFkeS4KICAgIGxldCBmaWxlRGF0YSA9IHlpZWxkIHsKICAgICAgcHJvbWlzZTogZmlsZURhdGFQcm9taXNlLAogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbnRpbnVlJywKICAgICAgfQogICAgfTsKCiAgICAvLyBVc2UgYSBjaHVua2VkIHNlbmRpbmcgdG8gYXZvaWQgbWVzc2FnZSBzaXplIGxpbWl0cy4gU2VlIGIvNjIxMTU2NjAuCiAgICBsZXQgcG9zaXRpb24gPSAwOwogICAgZG8gewogICAgICBjb25zdCBsZW5ndGggPSBNYXRoLm1pbihmaWxlRGF0YS5ieXRlTGVuZ3RoIC0gcG9zaXRpb24sIE1BWF9QQVlMT0FEX1NJWkUpOwogICAgICBjb25zdCBjaHVuayA9IG5ldyBVaW50OEFycmF5KGZpbGVEYXRhLCBwb3NpdGlvbiwgbGVuZ3RoKTsKICAgICAgcG9zaXRpb24gKz0gbGVuZ3RoOwoKICAgICAgY29uc3QgYmFzZTY0ID0gYnRvYShTdHJpbmcuZnJvbUNoYXJDb2RlLmFwcGx5KG51bGwsIGNodW5rKSk7CiAgICAgIHlpZWxkIHsKICAgICAgICByZXNwb25zZTogewogICAgICAgICAgYWN0aW9uOiAnYXBwZW5kJywKICAgICAgICAgIGZpbGU6IGZpbGUubmFtZSwKICAgICAgICAgIGRhdGE6IGJhc2U2NCwKICAgICAgICB9LAogICAgICB9OwoKICAgICAgbGV0IHBlcmNlbnREb25lID0gZmlsZURhdGEuYnl0ZUxlbmd0aCA9PT0gMCA/CiAgICAgICAgICAxMDAgOgogICAgICAgICAgTWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCk7CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPSBgJHtwZXJjZW50RG9uZX0lIGRvbmVgOwoKICAgIH0gd2hpbGUgKHBvc2l0aW9uIDwgZmlsZURhdGEuYnl0ZUxlbmd0aCk7CiAgfQoKICAvLyBBbGwgZG9uZS4KICB5aWVsZCB7CiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICB9CiAgfTsKfQoKc2NvcGUuZ29vZ2xlID0gc2NvcGUuZ29vZ2xlIHx8IHt9OwpzY29wZS5nb29nbGUuY29sYWIgPSBzY29wZS5nb29nbGUuY29sYWIgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYi5fZmlsZXMgPSB7CiAgX3VwbG9hZEZpbGVzLAogIF91cGxvYWRGaWxlc0NvbnRpbnVlLAp9Owp9KShzZWxmKTsK",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 73
        },
        "id": "wxTwwxqK1PyD",
        "outputId": "43656576-9483-46cc-b6be-283b78fcccea"
      },
      "source": [
        "from google.colab import files\n",
        "uploaded = files.upload()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-cf3f78bd-280a-45b8-8438-61c79f244cd9\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-cf3f78bd-280a-45b8-8438-61c79f244cd9\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving DataLabeled.csv to DataLabeled (1).csv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 482
        },
        "id": "pHiRQMKHdK6e",
        "outputId": "b900d3ff-3438-48bb-e6ca-13b1b2c1440b"
      },
      "source": [
        "# %% read csv file data\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "\n",
        "CSV_FILE = pd.read_csv('DataLabeled.csv')\n",
        "#pd.read_csv(Path().joinpath('data','traindata2.csv'))\n",
        "print(CSV_FILE['Text'].head(11))\n",
        "plt.rcParams['font.sans-serif']= \"cmr10\"\n",
        "sns.countplot(x='Class',hue='Class',data=CSV_FILE)\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0     \"RT @IndrajithLk: \"ඝරසරප\" චිත්‍රපටය, පහළ තියෙන...\n",
            "1     \"@NightWingzzz @lishwebber @Nimezzz මොකක් හරි ...\n",
            "2     \"@dead_mans_creed @Wind_Desika කිමද මේ තරම් කො...\n",
            "3     \"@MrThila_ @maalupaan @punchi_kella @anuwa01 @...\n",
            "4     \"විමලයා ගොනෙක් වගෙ stage එකෙ මෑරෙන්න හදනවා.......\n",
            "5     \"RT @SaliyaT_R: @fernandoharin ලැජ්ජයි ස⁣හෝදරය...\n",
            "6     \"@fernandoharin ලැජ්ජයි ස⁣හෝදරයා මේ යන විදිහ ද...\n",
            "7     \"@She_is_Nu @nthathsaranii ඇයි මං තම්බි වගේද ප...\n",
            "8     \"@Sagara_9 ඉටිම් බං උතුර ඉල්ලනසෙට් එකයි නැගෙනහ...\n",
            "9     \"RT @SaliyaT_R: පේනවානේ තම්බි කවදත් ඉන්නේ උන්ට...\n",
            "10    \"RT @ApiWenuwen: ඉතිහාසය ඉදිරියේ මෙවැනි හෙළිදර...\n",
            "Name: Text, dtype: object\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEHCAYAAABBW1qbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXwUVbr/8c8TJLIGFAhhCROuyAgiOhDwjiA7CnpFEAQZvWN0hHFjZFCZuQqOG86Io+gFfs6gIgooiyiiQjQysqMQQFACClzZxIEALhAIspzfH1UpswGJdFezfN+vV16pOrX0c7q6++lzquq0OecQEREBiIt1ACIicvJQUhARkYCSgoiIBJQUREQkoKQgIiIBJQUREQlENSmYWZyZjfCn483sMTPraWYDSlMmIiLhOCtaOzazc4E0oK1f9FtgvXNumpm9ZGb1gY4lKXPOfXW0x6levbpLSUmJVjVERE47y5Yt2+mcq1HcsqglBefcbuAZM+vmF7UCRvjTq/CSRUnLjpoUUlJSyMzMjGzwIiKnMTPbdLRlYZ5TSARy/Om9QFIpygows/5mlmlmmdnZ2VENWkTkTBJmUogDDvvTZfzpkpYV4Jwb45xLdc6l1qhRbAtIRER+hjCTwnagoj+dAGSXokxEREIQtXMKxVgANAVWAy2Ah/3HL0mZiEixDh48yNatW8nNzY11KCedcuXKUbduXcqWLVvibaJ59VEloB/QyMwGAS8Bj5iZAz5zzq0xs83AsOOVRStGETn1bd26lcqVK5OSkoKZxTqck4Zzjl27drF161bq169f4u2iefXRXryriEbkKx5YaJ2ckpSJiBxNbm6uEkIxzIxq1apR2otxdEeziJzylBCK93OelzDPKYiInLL27dvH0KFDqVmzJomJidSqVYuPP/6Y8uXL079/f6pWrRrrECPitEwKze9/9ZjLlz3125AiEZHTxa233spvfvMbunXz7sdduHAhGzdu5LvvvqN3796nTVJQ95GIyHEsXbqUzz//PEgIAK1ateLCCy+kSpUqMYws8k7LloKISCTNnTuXCy64oEh5v379uOeeewDYvXs39957L9dccw2rV69m6NChZGVlsXbtWs4++2w2bNhAp06dCsz/4Q9/CLsqx6WkICJyHEc7YZu/lXDuuecyatQoKlasyPjx4/n3v//N7NmzqVq1Kl27dqVOnTpF5k9G6j4SETmOtm3bsnbt2iLls2fPLjD/+eefM2nSJPbu3Utubi5paWksXbqUli1bsnHjxiLzJyMlBRGR40hNTeWiiy5ixowZQdmWLVsKnFyePHkykydP5oYbbiAxMZHNmzczffp0nnvuOZYvX87ChQt5++23C8yfjNR9JCJSAq+88gqPPPIIa9asoU6dOlSvXp169eqxfPlyXn31Vdq3b09OTg4zZ86kXr16vPvuuyQnJzN+/HjKly9Px44dWbduXYH5k5GSgohICcTHxzNs2LAi5atWrQqmL7/8cgCuuuqqYvfRpUuX6AQXQeo+EhGRgJKCiIgElBRERCSgpCAiIgElBRERCSgpiIhIQJekishp5XijJJfW8UZVnj17Ng8//DDnnXce48aN44cffmDw4MF8++23PPvss9SqVSui8YD3a3OTJ0+mZ8+epKSkRHTfaimIiJyAjh070q9fP1avXk1mZiYJCQn8+c9/ZsCAASVOCEeOHGHatGklfsy6deuydevWqAyVoaQgInKC4uLiGDFiBAMHlv6XhJ1zvPzyy+zZs6dU20VryG4lBRGRCGjdujXJycm8/vrrQdncuXMZMWIEw4YN48svv+TNN98MhuD+05/+RFpaGrt27WLhwoXMmzeP9957jy1bttCmTRueeuoprr32Wn744QduvPFGPvzwQ+677z727t0b1XooKYiIRMjw4cN55JFH2L9/P4cPH+a+++5j4MCB9O/fn6effprrrruOpKQkAO644w4AqlevTps2bWjTpg1XX301ycnJdOjQgWrVqvHiiy+SkJDAmDFj6NSpE8nJybz77rtRrYOSgohIhCQnJ9O3b1+GDx9OXFwc+/bt4/3332fZsmU0bNjwuNvn5OQE0ykpKdSoUQOAXbt2MX78eDZt2kRubm7U4gddfSQiElGDBw+mcePGpKWlUa1atWAQvCZNmgBQpkwZwPugz1O2bFn279/PokWL6Ny5M+CdpwDYtGkTaWlppKenBwlh06ZNUYtfSUFETivHu4Q00tLT0xkxYgRnn302119/PeXLl+fJJ5/krLPO4umnn+aZZ56hXr161KtXj7p169KpUyemTp1Kbm4uy5YtY/Xq1bRt25Zhw4ZRu3ZttmzZwkcffcTGjRtp3LgxFSpUICkpifT0dMyM+fPnc9555zFnzhy2bdvG5ZdfHiSaSDDnXMR2FgupqakuMzOzQNnxrlMO+0UjItGzZs0aGjVqFOswTlrFPT9mtsw5l1rc+jqnICIiASUFEREJKCmIiEhASUFERAJKCiIiEtAlqSJyWtn86EUR3V+9hz475vKMjAwGDRpE+/btufDCC4NLTHv27Flk3bFjx1K9enW6devG0KFDueiii+jdu3eR9Tp06MD48eOpU6dOiWLMv98TpZaCiMgJ6Ny5M82bN+e6667j97//PcOHD+e2224jOzu7yLrNmjXjzTffBGDQoEFce+21AGRnZzNnzpxgvddee63ECaHwfk+UkoKISASVK1eOChUq8NVXXxVZVrVq1WB6xYoVbNiwgcOHDzNq1KigfM+ePaxYsYIdO3aU+DHz7/dEqftIRCSCtmzZQqVKlahZsybPPPMMKSkpbNu2jbvvvrvAepmZmezbt4+0tDRWrFjBvn37iIuLo1mzZowbN47y5cuTmJjIqFGjqFy5MmvXruWOO+5g4cKFbN++nT179tCxY0cuu+yyiMYfWlIwszJAX2A/kAS8CAwBPgVqO+dGmlk8MDR/WVjxiYiciHnz5rFz585gmIoHHniAv/3tbyQlJfHEE08we/ZszjvvvGD9li1bMmfOHFJSUmjWrBnt2rWjTZs2AMEdyGvXrmXBggVMmjSJGTNm4JyjR48elC1blr1799KnTx/S09MjWo8wWwpdgM+dc5+aWU/gHmC9c26amb1kZvWBjoXLnHNF22AiIieZli1bBoPfASxZsoTq1asDkJiYyNKlSwskheLk5ORQsWLFYP7TTz+lfv36AMFJ5AMHDvDGG29QqVKlqIyYGuY5hT3AI2ZWCagNNAJW+MtWAW2BVsWUiYiccpo0acKWLVsA7zeVmzZtetR1y5Yty6FDh1i0aFGRfWzYsAHwfrJzzZo1dO/enUaNGnHFFVcAkR8xNbSWgnNunpndAqwGHgESgbzBw/fidSkVV1aEmfUH+gPUq1cvilGLyKnmeJeQRlpGRgaZmZmYGY0bNw4+k0aPHs0LL7xA/fr1qVGjBldddRWPPvooy5cvZ/Xq1UycOJF169axc+dOrrrqKl577TWuuuoqtmzZwr/+9S+ys7MZPXo07du3Z8yYMVSoUIGrr76aiy++mPnz57Nx40aSkpJYs2YNS5YsYfny5WRlZdG4ceMTqk9oo6SaWS3gv4AjeElhHXCLc26j/yFfBegA3JG/zDn31LH2q1FSRc5sGiX12Eo7SmqY5xT6AX91zh00s6+AGUBe51kCkA1sL6ZMRERCEuY5BQPi/elVwJNAXidbC+ATYEExZSIiEpIwWwr/C9xlZt8ADngGGGZmDvjMObfGzDYXLgsxPhGRM16YJ5q/BYYXKh5YaJ2cwmUiIhIeDXMhIiIBJQUREQlo7CMROa20GtkqovtbOGDhMZfnDZ3doUMHmjRpwhdffMGVV15J586dS7T/Dz/8kOnTpxcYFC+WlBRERE5A3tDZPXr0oF27duzevZvatWuzY8cOEhISjrt927Zt+dWvfnXU5VOnTuX666+PZMjHpO4jEZEIys7OJjk5mXLlypVo/aysLLKysopdlpGRcdRl0aKWgohIBCxZsoRvvvmGadOmsXjxYuLj47nxxhu55ZZbSE9P5+GHH6ZSpUpMmjSJ7777Lhje4tChQwwdOpQ5c+YwY8YM4uPjWbp0Kb179+aDDz7giy++YPLkyfTp0yeUeqilICISAS1btqRv3778x3/8BxkZGQCMGTOGTp06kZyczLvvvst3333HP/7xD26//XbatWuHmdGyZctgH1OmTKFOnTrceuutnHvuuVx99dU0a9YstIQASgoiIhHVuHFjZs6cCcCuXbsYP348mzZtIjc3l7Vr11KrVi0AWrduXeRcwt13383QoUPp1asXcXE/fTzn5OQQFiUFEZEIqly5Mhs2bGDTpk3cfPPN9O7dOxiQrlatWmzdujVY9/PPPy+w7Zdffsn06dN58sknmTt3bjCk9scff8yRI0dCiV/nFETktHK8S0gjLf/Q2Q0aNKBr165MmDCBefPmkZSURHp6OmbG/Pnzad26NQMHDuTpp58mOTmZli1b8sorr7B+/Xo+/fRTFi1aREJCAtu2beP666+nQoUKjB07lm+//bZAyyGaQhs6O1o0dLbImU1DZx9baYfOVveRiIgElBRERCSgpCAiIgElBRERCSgpiIhIQJekishpZW6bthHdX9t5c4+5/NChQ4wYMYLExERyc3PJyclh0KBBx93vnj176NChA0uXLo1UqBGhpCAicgImTpxI+/btSU1NJScnh7/85S8l2q5y5cq88847UY6u9NR9JCJyAr799ltWrVoFQMWKFenevXuJttuyZQuffPJJNEP7WdRSEBE5ATfccAPNmzfnhRdeoGfPngwcOJCpU6fy/fffs3nzZgYMGMBLL73EqFGjWLNmDZ06daJv37706tWL+++/n2uvvZbFixeza9cu9u/fz4EDB6hatSpPPPEEDz74IKtWraJmzZrUrFmTjIwM7r//furUqRO1+igpiIicgKSkJLKyspg+fTqvvvoqX375JRs2bGD27NksXbqUF154gQceeIDc3FwmTpzIU089RZs2bQCoXbs2AO+88w7t2rXjiiuuICsri0suuYQHHniAK664gvPPP5977rmHWbNm8fXXXzNv3jz69u0btfqo+0hE5AQsWLCAKlWqcPPNN5Oens6mTZtwzpGens7WrVuDUVH/53/+hxEjRhT7K2sDBgxg8uTJNGvWjNzcXADOPfdcypYtS3x8PDVr1gQgPj6eAwcORLU+SgoiIidg0aJFfP311wCULVuWBg0aULFiRbp06UKPHj2C32qeOXMmo0ePZsiQIUX2kZGRwUsvvcTixYt57733Qo2/MHUfichp5XiXkEZalSpVmD59OnFxcezcuZM//vGPrFu3jueff55q1arRpEkTBg8ezPr165kyZQq33347CQkJdOnShS+//JKpU6eyceNGpkyZwuHDh+natSuzZs0iKyuL999/n6VLl7J8+XI++eQTZsyYAUCPHj2oUqVKVOqjUVJF5JSmUVKPTaOkiojIz6akICIiASUFEREJKCmIiEhASUFERAJKCiIiEtB9CiJyWhl1b2RHHr376Wsiur+TnVoKIiIngalTp8Y6BEBJQUQk5nJycnjxxRdjHQYQcveRmXUBGgJHgPHAfcCnQG3n3EgziweG5i8LMz4RkdLKP+x1VlYWb731FkOGDGH16tW0adOG9u3b8+677/L999+zf/9+GjZsSLVq1ejTpw+DBw9m8uTJDBkyhI0bN/Liiy/Stm1bDh48yNq1azn77LPZsGEDf/jDH0KrT2hJwcyqATc75/qa2WPA7cB659w0M3vJzOoDHQuXOee+CitGEZHSyj/s9fnnn8/s2bPp1asX3bp1IzU1lfnz5/Pqq68yZcoUADp37sz7779PamoqdevW5eWXXyYxMZFatWpx2223ATBy5EiqVq1K165do/rbCcUJs/uoD5D3M0OPAxcAK/z5VUBboFUxZSIiJ63Cw17HxXkfq/Hx8ZQvX56VK1cSHx8frL9//3527NgBQEpKComJiQX2l5OTQ1paGkuXLqVly5Zs3LgxtLpAuEmhCVDXzK4G/ggkAjn+sr1A0lHKijCz/maWaWaZ2dnZ0Y1aROQYCg97ffjwYQAOHTrEjz/+yIUXXsjOnTuD9Q8fPkyNGjUAggQC3rDbhw4dYtGiRbz99ts899xzLF++nIULF4ZanzDPKcQB3zvn3jOzxsBVwF3+sjLAYX+dw4XKinDOjQHGgDdKajSDFpFTS9iXkBYe9nrSpEnMmDGDZcuW8c9//pNq1apx5513Mm7cOPbu3cuIESP44osvWLZsGSNHjuTxxx+nfPny9O7dmzFjxtC6dWvWrl3L+PHjKV++PB07dgy1PmEmhW+Abf70biAXqOjPJwDZwPZiykRETloPPfRQgfk6derQrVs3unXrFpTln87z2WefFZjv169fMN20adMIR1lyYXYf/Quo60+fCwwG8mreAu98w4JiykRETgkTJ05k/fr1LFu2LNah/GyhJQXn3HwAM/sd3iWpY4FLzewG4DPn3Brg9WLKREROCTfeeCNbt26lefPmsQ7lZwv1PgXn3COFigYWWp5TuExERMKjO5pFRCSgpCAiIgElBRERCWjobBE5rQy7qVdE9/fghDeOuTwjI4NBgwbRoUMHmjRpwhdffMGVV15J586di6w7duxYqlevTrdu3Rg6dCgXXXQRvXv3Lna/N998M3feeSeXXnppROpRUkoKIiInoHPnzjRv3pwePXrQrl07du/eTe3atdmxYwcJCQkF1m3WrBnPPvss3bp1Y9CgQVSoUOGo+/373/9OtWrVoh1+Eeo+EhGJoOzsbJKTkylXrlyRZVWrVg2mV6xYwYYNG4rdx48//siKFSvYvHlz1OI8mhK1FMysq3NulpmVBy4Dyjrn0qMbmojIqWPJkiV88803TJs2jcWLFxMfH8/zzz9PlSpVWL16NUOGDCmwfmZmJvv27ePhhx9m5MiR1KxZk5UrV1KlShXat2/PzJkz2bZtG2lpacGQF9u2baNz584cOXKEPn368OSTT7Jv3z5WrlzJ448/HpF6HDMpmNl1QCWgjZnV8IvLAs0AJQUREV/Lli1p164dK1asICMjg6ZNm7Jw4UImTJjA1KlTeeutt7jssssKrD9nzhwAZsyYQUZGBtu3b6dBgwa0aNGC1atXA7B+/Xo+/vhjRo8ezcGDB7nmmmtIT08nNTWV6tWrc+mllzJq1KiI1eN4LYW3gFsA8/8A9gNPRCwCEZHTSOPGjZk5cybgjYianp7OgQMHCnQdFda9e3feeOMNatWqxRVXXFFg2fLlyznnnHMAbyTVdevWBctq1qwJgJkRKcdMCs45B4w1s7edc7vyys3sgohFICJyGqlcuTIbNmzgkksu4YMPPqBLly4AbN26lUOHDhW7TVxcHL16FX/VVOPGjZk1axYABw4coHbt2tEJ3FfSq49+bWbX4HUdGV730cVRi0pE5Gc63iWkkZaRkUFmZiZmRoMGDejatSsTJkxg5cqVpKSkMHbsWCpUqECrVq149dVXWb58OatXr2bixImsW7eOnTt38tVXX3H11VeTkJBAly5d6N69O2+//TZxcXH07duXdu3aMXHiRLZt28bLL79MVlYWy5YtY8KECbRo0YKsrCxmzZpF165dT7g+5jUGjrOS2Z+BacCPftFNzrlhJ/zoEZCamuoyMzMLlDW//9VjbrPsqd9GMyQRCdGaNWto1KhRrMP42datW8f8+fO59dZb+fbbb3n00UdJS0vj4osj8727uOfHzJY551KLW7+kLYWVwP855w77O1xyQlGKiAjg/f7C/v37eeeddzh48CBJSUk0bNgwZvGUNCkMAf5qZrvxuo8acZSfyhQRkZKrUKECd9111/FXDElJk8ITzrn38mbMrEuU4hERKTXnXESvwDldlOT0QGEluqM5f0LwRff0t4hICZUrV45du3b9rA/A05lzjl27dhV7Z/WxlPSO5h3A53hdR9WA7/B+OU1EJKbq1q3L1q1byc7WT7oXVq5cOerWrXv8FfMpafdRz7yf0wQws2tL9SgiIlFStmxZ6tevH+swThslTQpmZm3ypoGmwNvRCUlERGKlpEnhKWCmP30IeDc64YiISCyVNCn0cc5tNLNEINvpjI6IyGmppL+nkGxmH+Dd1bzAzFpGMSYREYmRkrYUznfOBUP3mdmdgO5qFhE5zZS0pZBbaD4n0oGIiEjslbSl8L2ZTcIbEK828Gz0QhIRkVgpaVLoCCwCFgPbgV9HLSIREYmZkiaFxc65qXkzZqakICJyGippUmhqZucC/wc0BOoAk6MWlYiIxERJTzQ/AhwBrgMOAw9GLSIREYmZErUUnHOHgBeiHIuIiMRYSVsKIiJyBlBSEBGRgJKCiIgElBRERCQQelIws3PMbLiZxZvZY2bW08wG+MuKlImISHhi0VL4DZAI/BZY75ybBlxiZvWPUiYiIiEJNSmYWUNgoz/bCljhT68C2h6lTEREQhJ2S6EJsNqfTuSn0Vb3AklHKSvCzPqbWaaZZerHukVEIie0pGBmrYCFhR77sD9dxp8urqwI59wY51yqcy61Ro0aUYpYROTMU9KxjyLhl8D5QHWgAbAPqOgvSwCy8UZgLVwmIiIhCa2l4Jwb65wbB7wBrAemAE39xS2AT4AFxZSJiEhIwmwpYGblgQFAS2Aq0NnMHPCZc26NmW0GhuUvCzM+EZEzXahJwTm3H7jX/wN4r9DyHGBgmDGJiMhPdEeziIgElBRERCSgpCAiIgElBRERCSgpiIhIQElBREQCSgoiIhJQUhARkYCSgoiIBJQUREQkoKQgIiIBJQUREQkoKYiISEBJQUREAkoKIiISUFIQEZGAkoKIiASUFEREJKCkICIiASUFEREJKCmIiEhASUFERAJKCiIiElBSEBGRgJKCiIgElBRERCSgpCAiIgElBRERCSgpiIhIQElBREQCSgoiIhJQUhARkYCSgoiIBJQUREQkcFZYD2Rm8UB/oBxQFXgUGAp8CtR2zo301ylQFlZ8IiISbkuhF/C6c+7vwAXAYGC9c24acImZ1Qd+W0yZiIiEJMyk8Eugjz/9f8DvgBX+/CqgLdCqmDIREQlJaN1HwN/4KQk1xUsMOf78XiAJSCymrAgz64/XFUW9evWiFK6IyJkntJaCc26/cy7HzC4HPgJ+BA77i8v403HFlBW3rzHOuVTnXGqNGjWiHLmIyJkj1KuPzOwc4HLn3JPAdqCivygByD5KmYiIhCTsS1JvAJ40s7OABXjdSAAtgE+OUiYiIiEJ85LU/njnFR7DS0btgVvMzAGfOefWmNlmYFj+srDiExGREJOCc24MMKZQ8cBC6+QULhMRkfDojmYREQkoKYiISEBJQUREAkoKIiISUFIQEZFAmMNcSAg2P3rRMZfXe+izkCKJjVYjWx112cIBC0OMJHxz2xx9qLC28+aGGEn4Rt37zjGX3/30NSFFEr5hN/U65vIHJ7xRqv2ppSAiIgElBRERCSgpiIhIQElBREQCSgoiIhJQUhARkYCSgoiIBJQUREQkoKQgIiIBJQUREQkoKYiISEBJQUREAkoKIiISUFIQEZGAkoKIiASUFEREJKCkICIiASUFEREJKCmIiEhASUFERAJKCiIiElBSEBGRgJKCiIgElBRERCSgpCAiIgElBRERCSgpiIhIQElBREQCJ2VSMLOHzOxaM3sw1rGIiJxJTrqkYGadAHPOvQ2UNbM2sY5JRORMcdIlBaAVsMKfXgF0iGEsIiJnFHPOxTqGAsxsNPCmc262mXUErnfO3V5onf5Af3/2l8AXIYVXHdgZ0mPFgup3alP9Tl1h1+0XzrkaxS04K8QgSioOOOxPl8k3HXDOjQHGhBkUgJllOudSw37csKh+pzbV79R1MtXtZOw+2g5U9KcTgOwYxiIickY5GZPCAqCpP90S+DiGsYiInFFOxqTwL6CGmV0P4JxLj3E8+YXeZRUy1e/Upvqduk6aup10J5pFRCR2TsaWgoiIxIiSgoiIBM6opGBmHc1svpmN8+cTzOwfZjbZzGpF6THrmtm9ZpYSjf37j9HZzD4zs/81s9/7/3seZd1bzaybP/2YmfU+ynr/MrM6pYgh2G8Y8tX5OTPrZ2Z/N7POpdi+k5mNimaMP4eZnWVm95vZzf6xHFTC7Sqb2dJoxydFlea1WNL3n7/8FTO7NFpxH5Vz7oz6A34LLAVS/fkUoHUpto8DepbyMUcA7aJcr3F5jwGUA74FahSz3iXAOH/6HOBsf7pG/hiBpFI+frDfEI9l/jqfC+QCCSXctixQ7RjLrw+zLvke9+Z8r82KwN9LsW2pjtnp8Ber41RMHCV6LR7t/XeUfdYA4sKuyxnVUvAdAf4IPFvaDc3MgFuAyqXc9PvSPtaJcM7lAvuA+sUs/i7f9K+A88ysDHB3XqGZVQZ+ZWaJpXjY746/SlTVALbgvRlLorH/V4T/La/YZSE4B/+SbOdcDjC9JBuZWTIQ/rfKGDKzisBtsY6jGMd6LRZ5/xW3AzOL95fXi3h0x3Ey3tEcdc65BWa2xcz6AosBzKwt0AyoAEwFmgBPOOcuMLMngZrAfXhjM2Fm2cAqYCLwDtAa+G/geeBloAvwsHNub5h182NLBvYC2/3uh41Abedc4e6SVLz6jsN7AVYwsyPAciAN2A/sMLO7gT3ABXj1a4X3fFQGZjvnFkW5SsfS0u/66wn82jn3o5lNpNAxMLMbgKp4wwnMxHvtPwa085vzPwItgCnAFcAvzayPc25yyPWZBCwzs37ANOBZ//LsKngfECOB3+El8UbAh8DrwBvAU8DbZvZroBpQHjgb74PoAWAYXsLZ7v91Bp5yzn0dWu18hWJsDPQAHgcuBOY55z4ys//Cq3d54EtgFzAZGA708ddPMbPbgLl4rb8LgAPAec65/w21UsW/Fu/A+1J4oR9vfnnvv4fNbADeMbnYX/8j4CqgNjDOzP4b7/1YG8jA67GYDPzJ38fFzrkhEalFrJtdMWjm3eT/TwbW4r2x2uJ1KRlelv+nv84c/38KPzX50oC0fPt7GLgVv6sGqOj/vwe4Id867aJcr3HAQ0AvvJZQbb8syV/+ANCxUF3a4X1oFiKlMyIAAAXdSURBVIkxbx7vTTbJL+sG/AKve6oM3hs2vfBzFOKxHMdPTfbhQN/ijgFeMsg7lq2BXxU6vhOAi4A6/vEPnpcYvUar4HUjzca7fn22X94CeCDf8bkdaJNvu7z6PIGX2OKBS/yyVXgfmg2BWX5Z/7znLAZ1LBAjXiLAn1/lPwdT8q2f90E4Dm+QzMT8dfanB+B9MSufV+9YvhbxEsEEv+x64DfHeP9l5KtDV386zf9rAIz2y8rme8+NAy71p+dGqi5nYvcRAM65LXjfsAbjdSlVAK4EmuN9Kzkmv+maZ6NzLm84jmp+Vs/78AzTEufcG865Ec65bXh3hOcNsrUD70PlmArVC7w37FcAzrkZzrlNgMNLPq0Jv45Hk4X3zQqKHoMLgG/AayU651YU2nYUXqvhDbzXAlDscxF1ZtbaOfe9c+4VvJbOL7xi6wLUxa8H8Fe85F+4LuC1Jvrgtfjyjs9u59xBvBbRdr/sR7yWRCwUjvEIgHPuR7xvxBf78eUpD+R1Z250zu3IvzP/WI3De40vwfvwjZW812JToIx/7M4GfjjGNtPNrBfe8f2g0LJmeOcI8Y/h+fmW5R3LiN1wdsYmBd9wvGwdB+xyzqU77w7qvC6DvMH4quXb5iBe18Nl+cqOAJjZL4BX8Log1uQri5XP8VpE4H2grDrGusXVK28f5wGYWZyZNcLr516D/+KNcR3z7ME7P1LkGOC90ermrWhmTQpt29A51x2vKd6Wn56L/zSzsN8jl+Vd9eV/AKwHcvzX5lt435jB+9C5i6JdEgCdnXO/A34NXB1CzD9H4RjLgHf1FV5rYTVeV1+eMvw0DtqRfOUH/W0uA67Fax02w+/mjZE9eO+ZT4Fc/9hN8OeP5oj/he4N51zhQUCz8FqxmNnZwLZoBJ3njDqn4GfsP5rZAefcVOfcfjP7E3AIuNfvf9/s/20FPvT7c8sBzc3sQry+yweBbX7ffXu8fs0svJO7/8b7hueAy4ENeImntpnNL+aAR6JenfH6J52ZZTnnNvuL7gL6mdlXQLZzbqaZPQQ08+tyI3C+meX1s/8GmOnXqwNel9hdZvaRecOV7wPeA1b6dUvx69sIr1XSzMwaO+eyIl3H49R5PTALuAloQ9FjsACvb/5evBOAS8zsZqCBmV2C90H8A16X21S/nrcC5zjnjhCu74Hu/rmd6nhXrp3v903vAj43s+F4XQq9gX/4sacDDf3Xa4p/qWMZYJaZdQUam9mVeN+km/mXOuZdGvmWcy7UiyEKxwjc4J/baQ783jm3y8z+n5mlAZXwWkW/9JcPMLMhzrn9eMm/P94xvgCv+2g/XtdbKI7xWrwY2Ghmt+K9phbiXf1Y3Puvvpm9h9eaSMf74nUtXgJ8HZhjZjfivUZvMbPGeM/FTf6lyI3NrKtzbtYJ18fvjxIRiRkzm+OcaxfrOGLBzM4HLnfOjTWzc/DODY5zzq2MRTxnVEtBRE4+/jfgBmbW3Dm3LNbxxMDXQHkzuwbvRPK/KcF5zWhRS0FERAJn+olmERHJR0lBREQCSgoiIhLQiWaRUjKzCng3u23HuynwG+A/8S6FHOOci/U4UCI/m5KCSOmNBV5zzs0AMLNWePdsVMW7bl5JQU5Z6j4SKQUzawE0yUsIAM65hXh34IZ9A5hIxKmlIFI6bfEGUizsBeA5ADM7F3gab/TcC51zj/l3oAYjeOKNbhrLET1FiqWkIFI6xd7Y45z73vu5DXDO7Tazu51zOWb232aWhDdC7Xd4QyB8Xcy8yElB3UcipTMX7xt+AWbWsVBRE/83HCrhjZ01joIjeBaeFzkpKCmIlIJzLhP4zPL9HrU/gOB3+eb7AH2cc5Pwrk6qB3Sn4AieJ8uIniIFaJgLkVLyfyrxL3gjWn6N95sVm/F+Ne0NvF/Nugl4G2+U1jJ4o7N+j3fZ6h68MfGDeX/IdpGYU1IQEZGAuo9ERCSgpCAiIgElBRERCSgpiIhIQElBREQCSgoiIhJQUhARkcD/B6qPC9weSke5AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7JYol-VeA9pu",
        "outputId": "b9449892-88f2-42aa-893b-b5af6d8e4937"
      },
      "source": [
        "df = pd.read_csv('DataLabeled.csv', error_bad_lines=False)\n",
        "\n",
        "print(df['Text'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0       \"RT @IndrajithLk: \"ඝරසරප\" චිත්‍රපටය, පහළ තියෙන...\n",
            "1       \"@NightWingzzz @lishwebber @Nimezzz මොකක් හරි ...\n",
            "2       \"@dead_mans_creed @Wind_Desika කිමද මේ තරම් කො...\n",
            "3       \"@MrThila_ @maalupaan @punchi_kella @anuwa01 @...\n",
            "4       \"විමලයා ගොනෙක් වගෙ stage එකෙ මෑරෙන්න හදනවා.......\n",
            "                              ...                        \n",
            "1705    \"පීටීඒ සහ අයිසීසීපීආර් පනත යටතේ වැරදි ලෙස ගොනු...\n",
            "1706    \"අරාබි රටවල් මුස්ලිම් නොවන සඳහා සුපිරි වෙළඳසැල...\n",
            "1707    \"22 හැවිරිදි මුස්ලිම් පිරිමි ළමයෙක් ඉස්ලාම් භී...\n",
            "1708    \"# ශ්‍රී ලංකාවේ මුස්ලිම් ප්‍රජාව වෙත එල්ල කරන ...\n",
            "1709    \"රාජ්‍ය අංශයේ කාන්තා ඇඳුම් පැළඳුම් සංග්‍රහය පි...\n",
            "Name: Text, Length: 1710, dtype: object\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4kavsg8Of3iu"
      },
      "source": [
        "Detokenioze"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LOKdSmjbCEMY"
      },
      "source": [
        "stopwords_set = [\"සහ\",\"සමග\",\"සමඟ\",\"අහා\",\"ආහ්\",\"ආ\",\"ඕහෝ\",\"අනේ\",\"අඳෝ\",\"අපොයි\",\"පෝ\",\"අයියෝ\",\"ආයි\",\"ඌයි\",\"චී\",\"චිහ්\",\"චික්\",\"හෝ‍\",\"දෝ\",\n",
        "                 \"දෝහෝ\",\"මෙන්\",\"සේ\",\"වැනි\",\"බඳු\",\"වන්\",\"අයුරු\",\"අයුරින්\",\"ලෙස\",\"වැඩි\",\"ශ්‍රී\",\"හා\",\"ය\",\"නිසා\",\"නිසාවෙන්\",\"බවට\",\"බව\",\"බවෙන්\",\"නම්\",\"වැඩි\",\"සිට\",\n",
        "                 \"දී\",\"මහා\",\"මහ\",\"පමණ\",\"පමණින්\",\"පමන\",\"වන\",\"විට\",\"විටින්\",\"මේ\",\"මෙලෙස\",\"මෙයින්\",\"ඇති\",\"ලෙස\",\"සිදු\",\"වශයෙන්\",\"යන\",\"සඳහා\",\"මගින්\",\"හෝ‍\",\n",
        "                 \"ඉතා\",\"ඒ\",\"එම\",\"ද\",\"අතර\",\"විසින්\",\"සමග\",\"පිළිබඳව\",\"පිළිබඳ\",\"තුළ\",\"බව\",\"වැනි\",\"මහ\",\"මෙම\",\"මෙහි\",\"මේ\",\"වෙත\",\"වෙතින්\",\"වෙතට\",\"වෙනුවෙන්\",\n",
        "                 \"වෙනුවට\",\"වෙන\",\"ගැන\",\"නෑ\",\"අනුව\",\"නව\",\"පිළිබඳ\",\"විශේෂ\",\"දැනට\",\"එහෙන්\",\"මෙහෙන්\",\"එහේ\",\"මෙහේ\",\"ම\",\"තවත්\",\"තව\",\"සහ\",\"දක්වා\",\"ට\",\"ගේ\",\n",
        "                 \"එ\",\"ක\",\"ක්\",\"බවත්\",\"බවද\",\"මත\",\"ඇතුලු\",\"ඇතුළු\",\"මෙසේ\",\"වඩා\",\"වඩාත්ම\",\"නිති\",\"නිතිත්\",\"නිතොර\",\"නිතර\",\"ඉක්බිති\",\"දැන්\",\"යලි\",\"පුන\",\"ඉතින්\",\n",
        "                 \"සිට\",\"සිටන්\",\"පටන්\",\"තෙක්\",\"දක්වා\",\"සා\",\"තාක්\",\"තුවක්\",\"පවා\",\"ද\",\"හෝ‍\",\"වත්\",\"විනා\",\"හැර\",\"මිස\",\"මුත්\",\"කිම\",\"කිම්\",\"ඇයි\",\"මන්ද\",\"හෙවත්\",\n",
        "                 \"නොහොත්\",\"පතා\",\"පාසා\",\"ගානෙ\",\"තව\",\"ඉතා\",\"බොහෝ\",\"වහා\",\"සෙද\",\"සැනින්\",\"හනික\",\"එම්බා\",\"එම්බල\",\"බොල\",\"නම්\",\"වනාහි\",\"කලී\",\"ඉඳුරා\",\n",
        "                 \"අන්න\",\"ඔන්න\",\"මෙන්න\",\"උදෙසා\",\"පිණිස\",\"සඳහා\",\"රබයා\",\"නිසා\",\"එනිසා\",\"එබැවින්\",\"බැවින්\",\"හෙයින්\",\"සේක්\",\"සේක\",\"ගැන\",\"අනුව\",\"පරිදි\",\"විට\",\n",
        "                 \"තෙක්\",\"මෙතෙක්\",\"මේතාක්\",\"තුරු\",\"තුරා\",\"තුරාවට\",\"තුලින්\",\"නමුත්\",\"එනමුත්\",\"වස්\",'මෙන්',\"ලෙස\",\"පරිදි\",\"එහෙත්\"]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1urDRhm59Z0a",
        "outputId": "96a5b123-c500-4d54-96e2-0b461d6ac3a7"
      },
      "source": [
        "print(stopwords_set)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['සහ', 'සමග', 'සමඟ', 'අහා', 'ආහ්', 'ආ', 'ඕහෝ', 'අනේ', 'අඳෝ', 'අපොයි', 'පෝ', 'අයියෝ', 'ආයි', 'ඌයි', 'චී', 'චිහ්', 'චික්', 'හෝ\\u200d', 'දෝ', 'දෝහෝ', 'මෙන්', 'සේ', 'වැනි', 'බඳු', 'වන්', 'අයුරු', 'අයුරින්', 'ලෙස', 'වැඩි', 'ශ්\\u200dරී', 'හා', 'ය', 'නිසා', 'නිසාවෙන්', 'බවට', 'බව', 'බවෙන්', 'නම්', 'වැඩි', 'සිට', 'දී', 'මහා', 'මහ', 'පමණ', 'පමණින්', 'පමන', 'වන', 'විට', 'විටින්', 'මේ', 'මෙලෙස', 'මෙයින්', 'ඇති', 'ලෙස', 'සිදු', 'වශයෙන්', 'යන', 'සඳහා', 'මගින්', 'හෝ\\u200d', 'ඉතා', 'ඒ', 'එම', 'ද', 'අතර', 'විසින්', 'සමග', 'පිළිබඳව', 'පිළිබඳ', 'තුළ', 'බව', 'වැනි', 'මහ', 'මෙම', 'මෙහි', 'මේ', 'වෙත', 'වෙතින්', 'වෙතට', 'වෙනුවෙන්', 'වෙනුවට', 'වෙන', 'ගැන', 'නෑ', 'අනුව', 'නව', 'පිළිබඳ', 'විශේෂ', 'දැනට', 'එහෙන්', 'මෙහෙන්', 'එහේ', 'මෙහේ', 'ම', 'තවත්', 'තව', 'සහ', 'දක්වා', 'ට', 'ගේ', 'එ', 'ක', 'ක්', 'බවත්', 'බවද', 'මත', 'ඇතුලු', 'ඇතුළු', 'මෙසේ', 'වඩා', 'වඩාත්ම', 'නිති', 'නිතිත්', 'නිතොර', 'නිතර', 'ඉක්බිති', 'දැන්', 'යලි', 'පුන', 'ඉතින්', 'සිට', 'සිටන්', 'පටන්', 'තෙක්', 'දක්වා', 'සා', 'තාක්', 'තුවක්', 'පවා', 'ද', 'හෝ\\u200d', 'වත්', 'විනා', 'හැර', 'මිස', 'මුත්', 'කිම', 'කිම්', 'ඇයි', 'මන්ද', 'හෙවත්', 'නොහොත්', 'පතා', 'පාසා', 'ගානෙ', 'තව', 'ඉතා', 'බොහෝ', 'වහා', 'සෙද', 'සැනින්', 'හනික', 'එම්බා', 'එම්බල', 'බොල', 'නම්', 'වනාහි', 'කලී', 'ඉඳුරා', 'අන්න', 'ඔන්න', 'මෙන්න', 'උදෙසා', 'පිණිස', 'සඳහා', 'රබයා', 'නිසා', 'එනිසා', 'එබැවින්', 'බැවින්', 'හෙයින්', 'සේක්', 'සේක', 'ගැන', 'අනුව', 'පරිදි', 'විට', 'තෙක්', 'මෙතෙක්', 'මේතාක්', 'තුරු', 'තුරා', 'තුරාවට', 'තුලින්', 'නමුත්', 'එනමුත්', 'වස්', 'මෙන්', 'ලෙස', 'පරිදි', 'එහෙත්']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "55yESq-Kf-1w"
      },
      "source": [
        "def filter_stop_words(sentences):\n",
        "    filtered_sentences = []\n",
        "    detokenizer = Detok()\n",
        "    for sentence in sentences:\n",
        "        tokenized_sentence = word_tokenize(sentence)\n",
        "        filtered_sentence = [word for word in tokenized_sentence if word not in stopwords_set]\n",
        "        filtered_sentence = [] \n",
        "        for w in tokenized_sentence: \n",
        "            if w not in stopwords_set: \n",
        "                filtered_sentence.append(stem_word(w))\n",
        "        filtered_sentences.append(filtered_sentence)\n",
        "    return filtered_sentences\n",
        "\n",
        "\n",
        "def detokenize(filtered_sentences):\n",
        "    detokenized_sentences = []\n",
        "    for sentence in filtered_sentences:\n",
        "        detokenized_sentences.append(TreebankWordDetokenizer().detokenize(sentence))\n",
        "    return detokenized_sentences\n",
        "\n",
        "\n",
        "filtered_sentences = filter_stop_words(df['Text2'])\n",
        "detokenized_sentences = detokenize(filtered_sentences)\n",
        "\n",
        "df['filtered_sentence'] = detokenized_sentences\n",
        "df['filtered_sentence'] = df['filtered_sentence'].apply(simplify_sinhalese_text).tolist()\n",
        "#df['filtered_sentence'] = df['Text'].apply(remove_english_words).tolist()\n",
        "\n",
        "#df.to_csv('SubjectivityTagged.csv')\n",
        "df"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0Ohx8Co_U1pv"
      },
      "source": [
        "Data Cleaning"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oaKhkWmMq9ed",
        "outputId": "ab4b8a58-f280-4fd0-ba86-a429a9f167d6"
      },
      "source": [
        "#%%data cleaning , tokenizing and POS tagging\n",
        "df = pd.read_csv('SubjectivityTagged.csv')\n",
        "\n",
        "def clean_data(dataframe):\n",
        "    # Drop duplicate rows\n",
        "    dataframe.drop_duplicates(subset='Text', inplace=True)\n",
        "    #replace URL of a text\n",
        "    dataframe['Text_cleaned'] = dataframe['Text'].str.replace('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', '')\n",
        "    #replace mention\n",
        "    dataframe['Text_cleaned'] = dataframe['Text_cleaned'].str.replace('#|@\\w*','')\n",
        "    #remove retweet states in the beginning such as \"RT @sam92ky: \"\n",
        "    dataframe['Text_cleaned'] = dataframe['Text_cleaned'].str.replace('RT : ','')\n",
        "    #remove numbers\n",
        "    dataframe['Text_cleaned'] = dataframe['Text_cleaned'].str.replace('\\d+','')\n",
        "\n",
        "    #punctuation removal \n",
        "    string_text = dataframe['Text_cleaned'].str\n",
        "    dataframe['Text_cleaned'] = string_text.translate(str.maketrans('', '', string.punctuation))\n",
        "    print(string_text)\n",
        "\n",
        "    # coerced entire coloumn to str dtype\n",
        "    dataframe['Text_cleaned'] = dataframe['Text_cleaned'].astype(str)\n",
        "\n",
        "    #translate English to sinhala\n",
        "    df['Text_cleaned'] = df['Text_cleaned'].apply(translate_to_sinhala).tolist()\n",
        "    df['Text_cleaned'] = df['Text_cleaned'].apply(translate_english).tolist()\n",
        "\n",
        "    # simplify sinhala characters\n",
        "    df['Text_cleaned'] = df['Text_cleaned'].apply(simplify_sinhalese_text).tolist()\n",
        "\n",
        "    # pos tagging\n",
        "    #df['Text'] = df['Text'].apply(tagger.predict).tolist()\n",
        "\n",
        "    #print(\"New shape:\", dataframe.shape)\n",
        "    return dataframe.head()\n",
        "\n",
        "clean_data(df)\n",
        "df\n",
        "df.to_csv('SubjectivityTagged.csv')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<pandas.core.strings.StringMethods object at 0x7fe9c14b9ed0>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DToAfWpMb-or",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 434
        },
        "outputId": "2af9b123-6e68-4a09-ac06-c108e4fdf704"
      },
      "source": [
        "#%%data cleaning , tokenizing and POS tagging\n",
        "df = pd.read_csv('DataLabeled.csv')\n",
        "\n",
        "def clean_data(dataframe):\n",
        "    # Drop duplicate rows\n",
        "    #dataframe.drop_duplicates(subset='Text', inplace=True)\n",
        "    #replace URL of a text\n",
        "    dataframe['Text_cleaned'] = dataframe['Text'].str.replace('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', '')\n",
        "    #replace mention\n",
        "    dataframe['Text_cleaned'] = dataframe['Text_cleaned'].str.replace('#|@\\w*','')\n",
        "    #remove retweet states in the beginning such as \"RT @sam92ky: \"\n",
        "    dataframe['Text_cleaned'] = dataframe['Text_cleaned'].str.replace('RT : ','')\n",
        "    #remove numbers\n",
        "    dataframe['Text_cleaned'] = dataframe['Text_cleaned'].str.replace('\\d+','')\n",
        "\n",
        "    #punctuation removal \n",
        "    string_text = dataframe['Text_cleaned'].str\n",
        "    dataframe['Text_cleaned'] = string_text.translate(str.maketrans('', '', string.punctuation))\n",
        "    print(string_text)\n",
        "\n",
        "    # coerced entire coloumn to str dtype\n",
        "    dataframe['Text_cleaned'] = dataframe['Text_cleaned'].astype(str)\n",
        "\n",
        "    #translate English to sinhala\n",
        "    df['Text_cleaned'] = df['Text_cleaned'].apply(translate_to_sinhala).tolist()\n",
        "    df['Text_cleaned'] = df['Text_cleaned'].apply(translate_english).tolist()\n",
        "\n",
        "    # simplify sinhala characters\n",
        "    df['Text_cleaned'] = df['Text_cleaned'].apply(simplify_sinhalese_text).tolist()\n",
        "\n",
        "    # pos tagging\n",
        "    #df['Text'] = df['Text'].apply(tagger.predict).tolist()\n",
        "\n",
        "    #print(\"New shape:\", dataframe.shape)\n",
        "    return dataframe.head()\n",
        "\n",
        "clean_data(df)\n",
        "df\n",
        "#df.to_csv('imotional data (1).csv')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<pandas.core.strings.StringMethods object at 0x7f5341c269d0>\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Text</th>\n",
              "      <th>Class</th>\n",
              "      <th>Text_cleaned</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>\"RT @IndrajithLk: \"ඝරසරප\" චිත්‍රපටය, පහළ තියෙන...</td>\n",
              "      <td>Neutral</td>\n",
              "      <td>ගරසරප චිත්‍රපටය පහල තියෙන සම්බන්ධකය එකෙන් බාගත...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>\"@NightWingzzz @lishwebber @Nimezzz මොකක් හරි ...</td>\n",
              "      <td>Neutral</td>\n",
              "      <td>මොකක් හරි නාට්‍යයක් තියෙන්නත් එපැයි ඉතින්</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>\"@dead_mans_creed @Wind_Desika කිමද මේ තරම් කො...</td>\n",
              "      <td>Neutral</td>\n",
              "      <td>කිමද මේ තරම් කොහේදෝ ඇති විදපු ගලක් හා උරන මා...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>\"@MrThila_ @maalupaan @punchi_kella @anuwa01 @...</td>\n",
              "      <td>Neutral</td>\n",
              "      <td>එහෙම කොහොමද උනේ 😂😂 පරය අපිව අයින් කරල නේ 😂</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>\"විමලයා ගොනෙක් වගෙ stage එකෙ මෑරෙන්න හදනවා.......</td>\n",
              "      <td>Political</td>\n",
              "      <td>විමලයා ගොනෙක් වගෙ ක්‍රියාවලියක වැදගත් කාලයක් හ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1705</th>\n",
              "      <td>\"පීටීඒ සහ අයිසීසීපීආර් පනත යටතේ වැරදි ලෙස ගොනු...</td>\n",
              "      <td>Racist</td>\n",
              "      <td>පීටීඑ සහ අයිසීසීපීඅර් පනත යටතේ වැරදි ලෙස ගොනු ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1706</th>\n",
              "      <td>\"අරාබි රටවල් මුස්ලිම් නොවන සඳහා සුපිරි වෙළඳසැල...</td>\n",
              "      <td>Racist</td>\n",
              "      <td>අරාබි රටවල් මුස්ලිම් නොවන සදහා සුපිරි වෙලදසැල්...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1707</th>\n",
              "      <td>\"22 හැවිරිදි මුස්ලිම් පිරිමි ළමයෙක් ඉස්ලාම් භී...</td>\n",
              "      <td>Racist</td>\n",
              "      <td>හැවිරිදි මුස්ලිම් පිරිමි ලමයෙක් ඉස්ලාම් බීතික...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1708</th>\n",
              "      <td>\"# ශ්‍රී ලංකාවේ මුස්ලිම් ප්‍රජාව වෙත එල්ල කරන ...</td>\n",
              "      <td>Racist</td>\n",
              "      <td>ෂ්‍රී ලංකාවේ මුස්ලිම් ප්‍රජාව වෙත එල්ල කරන දේ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1709</th>\n",
              "      <td>\"රාජ්‍ය අංශයේ කාන්තා ඇඳුම් පැළඳුම් සංග්‍රහය පි...</td>\n",
              "      <td>Racist</td>\n",
              "      <td>රාජ්‍ය අංෂයේ කාන්තා ඇදුම් පැලදුම් සංග්‍රහය පිල...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1710 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                   Text  ...                                       Text_cleaned\n",
              "0     \"RT @IndrajithLk: \"ඝරසරප\" චිත්‍රපටය, පහළ තියෙන...  ...  ගරසරප චිත්‍රපටය පහල තියෙන සම්බන්ධකය එකෙන් බාගත...\n",
              "1     \"@NightWingzzz @lishwebber @Nimezzz මොකක් හරි ...  ...          මොකක් හරි නාට්‍යයක් තියෙන්නත් එපැයි ඉතින්\n",
              "2     \"@dead_mans_creed @Wind_Desika කිමද මේ තරම් කො...  ...    කිමද මේ තරම් කොහේදෝ ඇති විදපු ගලක් හා උරන මා...\n",
              "3     \"@MrThila_ @maalupaan @punchi_kella @anuwa01 @...  ...         එහෙම කොහොමද උනේ 😂😂 පරය අපිව අයින් කරල නේ 😂\n",
              "4     \"විමලයා ගොනෙක් වගෙ stage එකෙ මෑරෙන්න හදනවා.......  ...  විමලයා ගොනෙක් වගෙ ක්‍රියාවලියක වැදගත් කාලයක් හ...\n",
              "...                                                 ...  ...                                                ...\n",
              "1705  \"පීටීඒ සහ අයිසීසීපීආර් පනත යටතේ වැරදි ලෙස ගොනු...  ...  පීටීඑ සහ අයිසීසීපීඅර් පනත යටතේ වැරදි ලෙස ගොනු ...\n",
              "1706  \"අරාබි රටවල් මුස්ලිම් නොවන සඳහා සුපිරි වෙළඳසැල...  ...  අරාබි රටවල් මුස්ලිම් නොවන සදහා සුපිරි වෙලදසැල්...\n",
              "1707  \"22 හැවිරිදි මුස්ලිම් පිරිමි ළමයෙක් ඉස්ලාම් භී...  ...   හැවිරිදි මුස්ලිම් පිරිමි ලමයෙක් ඉස්ලාම් බීතික...\n",
              "1708  \"# ශ්‍රී ලංකාවේ මුස්ලිම් ප්‍රජාව වෙත එල්ල කරන ...  ...   ෂ්‍රී ලංකාවේ මුස්ලිම් ප්‍රජාව වෙත එල්ල කරන දේ...\n",
              "1709  \"රාජ්‍ය අංශයේ කාන්තා ඇඳුම් පැළඳුම් සංග්‍රහය පි...  ...  රාජ්‍ය අංෂයේ කාන්තා ඇදුම් පැලදුම් සංග්‍රහය පිල...\n",
              "\n",
              "[1710 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v_g_cQMAjVX_"
      },
      "source": [
        "Simplify sinhalese text"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M3mwN73nsOXd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "1028f1d6-8f25-43ae-fe74-0c105691d730"
      },
      "source": [
        "sinhalese_chars = [\n",
        "    \"අ\", \"ආ\", \"ඇ\", \"ඈ\", \"ඉ\", \"ඊ\",\n",
        "    \"උ\", \"ඌ\", \"ඍ\", \"ඎ\", \"ඏ\", \"ඐ\",\n",
        "    \"එ\", \"ඒ\", \"ඓ\", \"ඔ\", \"ඕ\", \"ඖ\",\n",
        "    \"ං\", \"ඃ\",\n",
        "    \"ක\", \"ඛ\", \"ග\", \"ඝ\", \"ඞ\", \"ඟ\",\n",
        "    \"ච\", \"ඡ\", \"ජ\", \"ඣ\", \"ඤ\", \"ඥ\", \"ඦ\",\n",
        "    \"ට\", \"ඨ\", \"ඩ\", \"ඪ\", \"ණ\", \"ඬ\",\n",
        "    \"ත\", \"ථ\", \"ද\", \"ධ\", \"න\", \"ඳ\",\n",
        "    \"ප\", \"ඵ\", \"බ\", \"භ\", \"ම\", \"ඹ\",\n",
        "    \"ය\", \"ර\", \"ල\", \"ව\",\n",
        "    \"ශ\", \"ෂ\", \"ස\", \"හ\", \"ළ\", \"ෆ\",\n",
        "    \"෴\", \"\\u200d\"\n",
        "]\n",
        "# \"\\u200d\" is used with \"යංශය\" - කාව්‍ය, \"රේඵය\" - වර්‍තමාන, \"Both\" - මහාචාර්‍ය්‍ය, \"රකාරාංශය\" - මුද්‍රණය\n",
        "\n",
        "sinhalese_vowel_signs = [\"්\", \"ා\", \"ැ\", \"ෑ\", \"ි\", \"ී\", \"ු\", \"ූ\", \"ෘ\", \"ෙ\", \"ේ\", \"ෛ\", \"ො\", \"ෝ\",\n",
        "                         \"ෞ\", \"ෟ\", \"ෲ\", \"ෳ\", \"ර්‍\"]\n",
        "\n",
        "# dictionary that maps wrong usage of vowels to correct vowels\n",
        "vowel_sign_fix_dict = {\n",
        "    \"ෑ\": \"ැ\",\n",
        "    \"ෙ\" + \"්\": \"ේ\",\n",
        "    \"්\" + \"ෙ\": \"ේ\",\n",
        "\n",
        "    \"ෙ\" + \"ා\": \"ො\",\n",
        "    \"ා\" + \"ෙ\": \"ො\",\n",
        "\n",
        "    \"ේ\" + \"ා\": \"ෝ\",\n",
        "    \"ො\" + \"්\": \"ෝ\",\n",
        "\n",
        "    \"ෙෙ\": \"ෛ\",\n",
        "    \"ෘෘ\": \"ෲ\",\n",
        "\n",
        "    \"ෙ\" + \"ෟ\": \"ෞ\",\n",
        "    \"ෟ\" + \"ෙ\": \"ෞ\",\n",
        "\n",
        "    \"ි\" + \"ී\": \"ී\",\n",
        "    \"ී\" + \"ි\": \"ී\",\n",
        "\n",
        "    # duplicating same symbol\n",
        "    \"ේ\" + \"්\": \"ේ\",\n",
        "    \"ේ\" + \"ෙ\": \"ේ\",\n",
        "\n",
        "    \"ො\" + \"ා\": \"ො\",\n",
        "    \"ො\" + \"ෙ\": \"ො\",\n",
        "\n",
        "    \"ෝ\" + \"ා\": \"ෝ\",\n",
        "    \"ෝ\" + \"්\": \"ෝ\",\n",
        "    \"ෝ\" + \"ෙ\": \"ෝ\",\n",
        "    \"ෝ\" + \"ේ\": \"ෝ\",\n",
        "    \"ෝ\" + \"ො\": \"ෝ\",\n",
        "\n",
        "    \"ෞ\" + \"ෟ\": \"ෞ\",\n",
        "    \"ෞ\" + \"ෙ\": \"ෞ\",\n",
        "\n",
        "    # special cases - may be typing mistakes\n",
        "    \"ො\" + \"ෟ\": \"ෞ\",\n",
        "    \"ෟ\" + \"ො\": \"ෞ\",\n",
        "}\n",
        "\n",
        "simplify_characters_dict = {\n",
        "    # Consonant\n",
        "    \"ඛ\": \"ක\",\n",
        "    \"ඝ\": \"ග\",\n",
        "    \"ඟ\": \"ග\",\n",
        "    \"ඡ\": \"ච\",\n",
        "    \"ඣ\": \"ජ\",\n",
        "    \"ඦ\": \"ජ\",\n",
        "    \"ඤ\": \"ඥ\",\n",
        "    \"ඨ\": \"ට\",\n",
        "    \"ඪ\": \"ඩ\",\n",
        "    \"ණ\": \"න\",\n",
        "    \"ඳ\": \"ද\",\n",
        "    \"ඵ\": \"ප\",\n",
        "    \"භ\": \"බ\",\n",
        "    \"ඹ\": \"බ\",\n",
        "    \"ශ\": \"ෂ\",\n",
        "    \"ළ\": \"ල\",\n",
        "\n",
        "    # Vowels\n",
        "    \"ආ\": \"අ\",\n",
        "    \"ඈ\": \"ඇ\",\n",
        "    \"ඊ\": \"ඉ\",\n",
        "    \"ඌ\": \"උ\",\n",
        "    \"ඒ\": \"එ\",\n",
        "    \"ඕ\": \"ඔ\",\n",
        "\n",
        "}\n",
        "\n",
        "def is_sinhalese_letter(char: str) -> bool:\n",
        "    return char in sinhalese_chars\n",
        "\n",
        "\n",
        "def is_sinhalese_vowel(char: str) -> bool:\n",
        "    return char in sinhalese_vowel_signs\n",
        "\n",
        "\n",
        "def get_fixed_vowel(vowel: str) -> str:\n",
        "    return vowel_sign_fix_dict[vowel]\n",
        "\n",
        "\n",
        "def get_simplified_character(character: str) -> str:\n",
        "    if len(character) != 1:\n",
        "        raise TypeError(\"character should be a string with length 1\")\n",
        "    try:\n",
        "        return simplify_characters_dict[character]\n",
        "    except KeyError:\n",
        "        return character\n",
        "\n",
        "def simplify_sinhalese_text(text: str) -> str:\n",
        "    \"\"\"\n",
        "    simplify\n",
        "    :param text:\n",
        "    :return:\n",
        "    \"\"\"\n",
        "    modified_text = \"\"\n",
        "    for c in text:\n",
        "        modified_text += get_simplified_character(c)\n",
        "    return modified_text\n",
        "\n",
        "simplify_sinhalese_text(\"ඝරසරප චිත්‍රපටය පහළ තියෙන සම්බන්ධකය එකෙන් බාගත කරගන්න\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'ගරසරප චිත්\\u200dරපටය පහල තියෙන සම්බන්ධකය එකෙන් බාගත කරගන්න'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SmjdJg3yWcVz",
        "outputId": "02848810-5f48-4d42-91fa-58dc1c2c9b56"
      },
      "source": [
        "class SinhaleseVowelLetterFixer:\n",
        "    \"\"\"\n",
        "    Sinhalese Language Vowel Letter Fixer\n",
        "    \"\"\"\n",
        "\n",
        "    @staticmethod\n",
        "    def get_fixed_text(text: str) -> str:\n",
        "        \"\"\"\n",
        "        Fix wrong usage of vowels\n",
        "        :param text: text to be fixed\n",
        "        :return: fixed text with proper vowels\n",
        "        \"\"\"\n",
        "        fixed_text = \"\"\n",
        "        last_letter = \"\"\n",
        "        last_vowel = \"\"\n",
        "\n",
        "        for letter in text:\n",
        "            if is_sinhalese_letter(letter):\n",
        "                fixed_text += (last_letter + last_vowel)\n",
        "                last_letter = letter\n",
        "                last_vowel = \"\"\n",
        "            elif is_sinhalese_vowel(letter):\n",
        "                if last_letter == \"\":\n",
        "                    print(\"Error : First letter can't be a vowel sign : \" + letter)\n",
        "                if last_vowel == \"\":\n",
        "                    last_vowel = letter\n",
        "                else:\n",
        "                    try:\n",
        "                        last_vowel = get_fixed_vowel(last_vowel + letter)\n",
        "                        print(last_vowel)\n",
        "                    except KeyError:\n",
        "                        # fix error of mistakenly duplicate vowel\n",
        "                        if last_vowel == letter:\n",
        "                            continue\n",
        "                        else:\n",
        "                            print(\"Error : can't fix vowel combination \" + last_vowel + \" + \" + letter)\n",
        "            else:\n",
        "                fixed_text += (last_letter + last_vowel + letter)\n",
        "                last_letter = \"\"\n",
        "                last_vowel = \"\"\n",
        "\n",
        "        fixed_text += last_letter + last_vowel\n",
        "        return fixed_text\n",
        "\n",
        "\n",
        "corrected_text = SinhaleseVowelLetterFixer.get_fixed_text(\"මෑරෙන්න\")\n",
        "#assert correct_text == corrected_text\n",
        "print(corrected_text)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "මෑරෙන්න\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oZyHtyH3jHXy"
      },
      "source": [
        "Translate to sinhala"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sibgKOtOEl8m"
      },
      "source": [
        "# english-sinhala dictionary\n",
        "dictionary = {}\n",
        "df= pd.read_csv('en-sinhala dictionary.csv')\n",
        "dictionary_file = df[\"En,sinhala\"]\n",
        "\n",
        "for line in dictionary_file:\n",
        "    key, value = line.strip().split(\",\")\n",
        "    dictionary[key] = value\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PWfMc94pHEYl"
      },
      "source": [
        "# to check whether the string contains English words(any)\n",
        "def translate_english(x):\n",
        "    for word1 in x.split():\n",
        "        new_word = ''.join(i for i in word1 if not i.isdigit())\n",
        "        x = x.replace(word1, new_word)\n",
        "    for word in x.split():\n",
        "        word2 = \"\".join(l for l in word if l not in string.punctuation)\n",
        "        if re.match('[a-zA-Z]', word2) is not None:\n",
        "            word1 = word2.lower()\n",
        "            translated_word = dictionary.get(word1)\n",
        "            if translated_word is None:\n",
        "                translated_word = ''\n",
        "            x = x.replace(word, translated_word)\n",
        "    return x\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bLl-coSsxvTn"
      },
      "source": [
        "translate_words_dict = {\n",
        "    \"unp\": \"එක්සත් ජාතික පක්ෂය\",\n",
        "    \"muslim\": \"මුස්ලිම්\",\n",
        "    \"srilankanpolitics\": \"ශ්‍රී ලංකන් දේශපාලනය\",\n",
        "    \"council\": \"සභාව\",\n",
        "    \"sinhala\": \"සිංහල\",\n",
        "    \"buddhist\": \"බෞද්ධ\",\n",
        "    \"buddhism\": \"බුද්ධාගම\",\n",
        "    \"srilanka\": \"ශ්‍රී ලංකාව\",\n",
        "    \"racist\": \"ජාතිවාදී\",\n",
        "    \"presidentialfirst\": \"පළමු ජනාධිපති\",\n",
        "    \"feeling\": \"හැඟීම\",\n",
        "    \"feminist\": \"ස්ත්‍රීවාදී\",\n",
        "    \"loved\": \"ආදරය කළා\",\n",
        "    \"team\": \"කණ්ඩායම\",\n",
        "    \"tclsl\":\"ට්විටර් ක්‍රිකට් ලීගය ශ්‍රී ලංකාව\",\n",
        "    \"pongal\": \"පොංගල්\",\n",
        "    \"pongalfestival\": \"පොංගල් උත්සවය\",\n",
        "    \"women\": \"කාන්තා\",\n",
        "    \"nextpresidentinsl\": \"ශ්‍රී ලංකාවේ මීළඟ ජනාධිපති \",\n",
        "    \"seventhexecutivepresident\": \"හත්වන විධායක සභාපති\",\n",
        "    \"hate\": \"වෛරය\",\n",
        "    \"love\": \"ආදරය\",\n",
        "    \"angry\": \"තරහයි\",\n",
        "    \"doctor\": \"ඩොක්ටර්\",\n",
        "    \"ltte\": \"එල්ටීටීඊය\",\n",
        "    \"lka\": \"‍ශ්‍රී ලංකාව\",\n",
        "    \"hurt\": \"රිදෙනවා\",\n",
        "    \"typo\": \"යතුරු ලියනය\",\n",
        "    \"racial\": \"වාර්ගික\",\n",
        "    \"hatred\": \"වෛරය\",\n",
        "    \"halal\": \"හලාල්\",\n",
        "    \"wicket\": \"කඩුල්ල\",\n",
        "    \"taker\": \"ටේකර්\",\n",
        "    \"indoor\": \"ගෘහස්ථ\",\n",
        "    \"attacker\": \"ප්‍රහාරකයා\",\n",
        "    \"attack\": \"ප්රහාරය\",\n",
        "    \"spikers\": \"ස්පිකර්ස්\",\n",
        "    \"training\": \"පුහුණුව\",\n",
        "    \"final\": \"අවසාන\",\n",
        "    \"match\": \"තරගය\",\n",
        "    \"tournament\": \"තරඟාවලිය\",\n",
        "    \"youth\": \"තරුණ\",\n",
        "    \"amen\": \"ආමෙන්\",\n",
        "    \"enough\": \"ඇති\",\n",
        "    \"standagainstracism\": \"ජාතිවාදයට එරෙහිව නැගී සිටින්න\"\n",
        "}\n",
        "\n",
        "\n",
        "def translate_to_sinhala(word: str) -> str:\n",
        "  word = word.lower()\n",
        "  if word in translate_words_dict:\n",
        "        return translate_words_dict[word]\n",
        "  return word\n",
        "\n",
        "#translate_to_sinhala(\"unp\")\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "0fhEqKIbHcyI",
        "outputId": "4c2be918-af23-4994-f460-fc749f31a8ae"
      },
      "source": [
        "translate_english(\"දේශපාලන අර්බුදය Match fixing හා වීරයන් ඒ සඳහා කැපකිරීමක් කිරීමට ඉඩ දෙන්නේ නැත.\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'දේශපාලන අර්බුදය තරඟය කලම්ප හා වීරයන් ඒ සඳහා කැපකිරීමක් කිරීමට ඉඩ දෙන්නේ නැත.'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "skw-4f_bje9o"
      },
      "source": [
        "Stem words"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7DJhfuC7exv7"
      },
      "source": [
        "import os\n",
        "\n",
        "import pygtrie as trie\n",
        "\n",
        "from sinling.config import RESOURCE_PATH\n",
        "from sinling.core import Stemmer\n",
        "\n",
        "__all__ = [\n",
        "    'SinhalaStemmer'\n",
        "]\n",
        "\n",
        "\n",
        "def _load_stem_dictionary():\n",
        "    stem_dict = dict()\n",
        "    with open(os.path.join(RESOURCE_PATH, 'stem_dictionary.txt'), 'r', encoding='utf-8') as fp:\n",
        "        for line in fp.read().split('\\n'):\n",
        "            try:\n",
        "                base, suffix = line.strip().split('\\t')\n",
        "                stem_dict[f'{base}{suffix}'] = (base, suffix)\n",
        "            except ValueError as _:\n",
        "                pass\n",
        "    return stem_dict\n",
        "\n",
        "\n",
        "def _load_suffixes():\n",
        "    suffixes = trie.Trie()\n",
        "    with open(os.path.join(RESOURCE_PATH, 'suffixes_list.txt'), 'r', encoding='utf-8') as fp:\n",
        "        for suffix in fp.read().split('\\n'):\n",
        "            suffixes[suffix[::-1]] = suffix\n",
        "    return suffixes\n",
        "\n",
        "\n",
        "class SinhalaStemmer(Stemmer):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.stem_dictionary = _load_stem_dictionary()\n",
        "        self.suffixes = _load_suffixes()\n",
        "\n",
        "    def stem(self, word):\n",
        "        if word in self.stem_dictionary:\n",
        "            return self.stem_dictionary[word]\n",
        "        else:\n",
        "            suffix = self.suffixes.longest_prefix(word[::-1]).key\n",
        "            if suffix is not None:\n",
        "                return word[0:-len(suffix)], word[len(word) - len(suffix):]\n",
        "            else:\n",
        "                return word, ''\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T1MBcxIotE2X",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a25f97f2-bbba-420b-848d-b473b58c241e"
      },
      "source": [
        "\n",
        "stemmer = stemmer()\n",
        "\n",
        "def stem_word(word: str) -> str:\n",
        "    word= translate_to_sinhala(word)\n",
        "    \"\"\"\n",
        "    Stemming words\n",
        "    :param word: word\n",
        "    :return: stemmed word\n",
        "    \"\"\"\n",
        "    if len(word) < 4:\n",
        "        return word\n",
        "\n",
        "    # remove 'ට'\n",
        "    if word[-1] == 'ට':\n",
        "        return word[:-1]\n",
        "\n",
        "    # remove 'ම'\n",
        "    if word[-1] == 'ම':\n",
        "        return word[:-1]\n",
        "\n",
        "    # remove 'ද'\n",
        "    if word[-1] == 'ද':\n",
        "        return word[:-1]\n",
        "\n",
        "    # remove 'ටත්'\n",
        "    if word[-3:] == 'ටත්':\n",
        "        return word[:-3]\n",
        "\n",
        "    # remove 'එක්'\n",
        "    if word[-3:] == 'ෙක්':\n",
        "        return word[:-3]\n",
        "\n",
        "    # remove 'යේ'\n",
        "    if word[-2:] == 'යේ':\n",
        "        return word[:-2]\n",
        "\n",
        "    # remove 'ගෙ' (instead of ගේ because this step comes after simplifying text)\n",
        "    if word[-2:] == 'ගෙ':\n",
        "        return word[:-2]\n",
        "\n",
        "    # remove 'එ'\n",
        "    if word[-1:] == 'ෙ':\n",
        "        return word[:-1]\n",
        "\n",
        "    # remove 'ක්'\n",
        "    if word[-2:] == 'ක්':\n",
        "        return word[:-2]\n",
        "\n",
        "    # remove 'වත්'\n",
        "    if word[-3:] == 'වත්':\n",
        "        return word[:-3]\n",
        "\n",
        "    word=stemmer.stem(word)\n",
        "    word=word[0]\n",
        "\n",
        "\n",
        "    # else\n",
        "    return word\n",
        "\n",
        "\n",
        "print(stem_word('ගරසරප'))\n",
        "print(stem_word('චිත්‍රපටය'))\n",
        "print(stem_word('පහල'))\n",
        "print(stem_word('තියෙන'))\n",
        "print(stem_word('සම්බන්ධකය'))\n",
        "print(stem_word('එකෙන්'))\n",
        "print(stem_word('බාගත'))\n",
        "print(stem_word('කරගන්න'))\n",
        "#විපක්ෂයේ,විපක්ෂව,විපක්ෂකම,විපක්ෂය,විපක්ෂයා \n",
        "\n",
        "print(stem_word('ගරසරප චිත්‍රපටය පහල තියෙන සම්බන්ධකය එකෙන් බාගත කරගන්න'))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "ගරසරප\n",
            "චිත්‍රපට\n",
            "පහල\n",
            "තියෙ\n",
            "සම්බන්ධක\n",
            "එ\n",
            "බාගත\n",
            "කරග\n",
            "ගරසරප චිත්‍රපටය පහල තියෙන සම්බන්ධකය එකෙන් බාගත කරග\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jPwq3B5NI4FG"
      },
      "source": [
        "නායකයා නායකයෝ නායකයන් නායකයාට නායකයන්ට නායකයාගෙන් නායකයන්ගෙන් නායකයාගේ නායකයන්ගේ නායකයනි"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HOFXIiv-TQWH",
        "outputId": "0da9d76e-d160-41de-c716-23f73e6e1e46"
      },
      "source": [
        "from sinling import SinhalaTokenizer as tokenizer,SinhalaStemmer as stemmer, POSTagger,preprocess, word_joiner,word_splitter\n",
        "stemmer = stemmer()\n",
        "\n",
        "#words = [\"නායකයා\",\"නායකයෝ\",\"නායකයන්\",\"නායකයාට\",\"නායකයන්ට\",\"නායකයාගෙන්\",\"නායකයන්ගෙන්\",\"නායකයාගේ\",\"නායකයන්ගේ\",\"නායකයනි\"]\n",
        "words = [\"ගරසරප\",\"චිත්‍රපටය\",\"පහල\",\"තියෙන\",\"සම්බන්ධකය\",\"එකෙන්\",\"බාගත\",\"කරගන්න\"]\n",
        "\n",
        "for word in words:\n",
        "  word=stemmer.stem(word)\n",
        "  word=word[0]\n",
        "  print(word)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "ගරසරප\n",
            "චිත්‍රපට\n",
            "පහ\n",
            "තියෙ\n",
            "සම්බන්ධක\n",
            "එ\n",
            "බාගත\n",
            "කරග\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Tc-5-4ZriVK9"
      },
      "source": [
        "POS Tagging"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FbsFF84CiVnv",
        "outputId": "00caa58a-7632-4691-9af2-c67e669a6027"
      },
      "source": [
        "import os\n",
        "from typing import List, Text, Tuple\n",
        "\n",
        "import joblib\n",
        "\n",
        "from sinling.sinhala.stemmer import SinhalaStemmer\n",
        "from sinling.config import RESOURCE_PATH\n",
        "\n",
        "from typing import List, Text, Tuple\n",
        "\n",
        "__all__ = ['Tagger']\n",
        "\n",
        "\n",
        "class Tagger:\n",
        "    def predict(self, tokens: List[Text]) -> List[Tuple[Text, Text]]:\n",
        "        raise NotImplementedError\n",
        "\n",
        "\n",
        "__all__ = [\n",
        "    'POSTagger'\n",
        "]\n",
        "\n",
        "\n",
        "class POSTagger(Tagger):\n",
        "    def __init__(self):\n",
        "        self._model = joblib.load(os.path.join(RESOURCE_PATH, 'pos-tagger-crf-sinling.joblib'))\n",
        "        self._stemmer = SinhalaStemmer()\n",
        "\n",
        "    def predict(self, x: List[List[Text]]) -> List[List[Tuple[Text, Text]]]:\n",
        "        features = [[self._word2features(ts, i) for i in range(len(ts))] for ts in x]\n",
        "        pos_tags = self._model.predict(features)\n",
        "        return [list(zip(x[ix], pos_tags[ix])) for ix in range(len(x))]\n",
        "\n",
        "    def _word2features(self, sent, i):\n",
        "        word = sent[i]\n",
        "        stem, suff = self._stemmer.stem(word)\n",
        "        features = {\n",
        "            'bias': 1.0,\n",
        "            word: True,\n",
        "            f'STEM': stem,\n",
        "            f'SUFF': suff,\n",
        "            'len(word)': len(word),\n",
        "            'word.isdigit()': word.isdigit(),\n",
        "        }\n",
        "        if i > 0:\n",
        "            word_prev = sent[i - 1]\n",
        "            features.update({\n",
        "                f'-1:word': word_prev,\n",
        "                '-1:word.isdigit()': word_prev.isdigit(),\n",
        "            })\n",
        "        else:\n",
        "            features['BOS'] = True\n",
        "        if i < len(sent) - 1:\n",
        "            word_next = sent[i + 1]\n",
        "            features.update({\n",
        "                f'+1:word': word_next,\n",
        "                '+1:word.isdigit()': word_next.isdigit(),\n",
        "            })\n",
        "        else:\n",
        "            features['EOS'] = True\n",
        "        return features\n",
        "\n",
        "\n",
        "if __name__ == '__main__':\n",
        "    from sinling import SinhalaTokenizer\n",
        "\n",
        "    tokenizer = SinhalaTokenizer()\n",
        "\n",
        "    document = 'ගරසරප චිත්‍රපට පහල තියෙ සම්බන්ධක එක  බාගත කරග' \\\n",
        "               'ඇතැම් වෛරස් රෝග වලට වැක්සීන හෙවත් එන්නත් ද වෛරස් නාශක ඖෂධ ද තිබුනද සියලූ‍ වෛරස් ' \\\n",
        "               'සම්බන්ධයෙන් ඒ න්‍යාය වැඩ කරන්නේ නැත. වසූරිය වෛරසය මිනිසා විසින් මිහිමතින් තුරන් කර තිබේ.'\n",
        "\n",
        "    tokenized_sentences = [tokenizer.tokenize(f'{ss}.') for ss in tokenizer.split_sentences(document)]\n",
        "    print(tokenized_sentences)\n",
        "    tagger = POSTagger()\n",
        "\n",
        "    pos_tags = tagger.predict(tokenized_sentences)\n",
        "\n",
        "    for sent in pos_tags:\n",
        "        print(sent)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[['ගරසරප', 'චිත්\\u200dරපට', 'පහල', 'තියෙ', 'සම්බන්ධක', 'එක', 'බාගත', 'කරග', '.']]\n",
            "[('ගරසරප', 'JJ'), ('චිත්\\u200dරපට', 'NNC'), ('පහල', 'JJ'), ('තියෙ', 'NNC'), ('සම්බන්ධක', 'NNC'), ('එක', 'NUM'), ('බාගත', 'NNC'), ('කරග', 'VFM'), ('.', 'FS')]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T_ZTgZxrjzRw"
      },
      "source": [
        "Remove english words"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aB9IHJDCw5Ae",
        "outputId": "778b6347-a185-413d-ba90-a0795eb35322"
      },
      "source": [
        "import nltk\n",
        "nltk.download('words')\n",
        "import string\n",
        "\n",
        "def remove_english_words(sent: str) -> str:\n",
        "  \n",
        "  words = set(nltk.corpus.words.words())\n",
        "  string_text = \"\".join(w for w in sent \\\n",
        "\n",
        "         if not w in words)\n",
        "\n",
        "  string_text = string_text.translate(str.maketrans('', '', string.punctuation))\n",
        "\n",
        "  return string_text\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package words to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/words.zip.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bkfPoBBkibat"
      },
      "source": [
        "remove neutral data rows"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ski2H9Xkih-S",
        "outputId": "c5fe1cd0-c3b2-4b14-f9f6-50c1f116f71c"
      },
      "source": [
        "# importing pandas as pd\n",
        "import pandas as pd\n",
        "  \n",
        "# Read the csv file and construct the \n",
        "# dataframe\n",
        "df = pd.read_csv('cleaned data.csv')\n",
        "  \n",
        "# First filter out those rows which\n",
        "# does not contain any data\n",
        "df = df.dropna(how = 'all')\n",
        "  \n",
        "# Filter all rows for which the player's\n",
        "# age is greater than or equal to 25\n",
        "df.drop(df[df['Class'] == 'Neutral'].index, inplace = True)\n",
        "  \n",
        "# Print the modified dataframe\n",
        "print(df.head(15))\n",
        "  \n",
        "# Print the shape of the dataframe\n",
        "print(df.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-05-21 11:24:39,463 | INFO | utils.py:157 | _init_num_threads | NumExpr defaulting to 2 threads.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "    Id  ...                                  Filtered_sentence\n",
            "0    0  ...  දින එකාබද්ධ විපක්ෂ මාධ්‍ ලේක රේනුක පෙරේර මහත න...\n",
            "1    1  ...  මා එක්සත් ජාතික පක්ෂ එකාබද්ධ විපක්ෂ පවති කිසි ...\n",
            "4    4  ...   විමල ගොන වගෙ  එකෙ මෑරෙ හද පරය රටට වෑඩ කර යකො හොර\n",
            "5    5  ...  ලැජ්ජ ස⁣හෝදර විදිහ දැක්කා ඔබ කුලල්කෑ වාසි ග තම...\n",
            "6    6  ...  ලැජ්ජ ස⁣හෝදර විදිහ දැක්කා ඔබ කුලල්කෑ වාසි ග තම...\n",
            "8    8  ...  ඉටි බං උතුර ඉල්ලනසෙට එක නැගෙනහිරපඉල්ල තම්බි සෙ...\n",
            "9    9  ...  පේනවාන තම්බි කවද ඉ උන් වාසි පැත්ත එහෙ ඉදගෙ උ ෂ...\n",
            "11  12  ...                      ඉස්ලාම ත්‍රස්තවාදී එහි තේරු එ\n",
            "12  13  ...  පල පල තම්බි හු තම වැඩ බලාගෙ හිට අපි අපේ රට බලා...\n",
            "13  14  ...  අඩෝ තම්බි පර කට වහගෙ හිට අල්ලාහ් හොර ගැනි අයේෂ...\n",
            "14  15  ...  කාද යකෝ ඔසදී ඩීප්ෆ්‍රීසර එකෙ හිර කරල තිබ්බ පරය 👹️\n",
            "15  17  ...  එක පැත්ත බැලුවා උඩතලවි තම්බි දන ගැස්ස චරිතය තම...\n",
            "16  18  ...  සිංහලයා සිංහ පක්ෂ නැතිමු දෙමලා ඉලම පක්ෂ ඇත තම්...\n",
            "17  19  ...  මුස්ලි ලෝකයට පිලිලය වු මරක්ක කරන වහබ්වාද ෂ්‍රී...\n",
            "20  22  ...  මුරලිදර ඉන්ටර්වී සීරිස එකේ දෙ එක මයු රහීම්ස ගෙ...\n",
            "\n",
            "[15 rows x 4 columns]\n",
            "(735, 4)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        },
        "id": "2xtVVLVHjVs4",
        "outputId": "84fea889-bbf1-4282-b259-025b603a0df9"
      },
      "source": [
        "plt.rcParams['font.sans-serif']= \"cmr10\"\n",
        "sns.countplot(x='Class',hue='Class',data=df)\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEGCAYAAABsLkJ6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXwV9b3/8dcHS9iRNawiXNGCCyKN0RogCIIsFbGyaNsrXKvUvRar99cWvGrVXrUWLXivlyqiFmWzyiJYRQticGEJsoQooFEWhYig7LJ8fn/MJCQhwCFyzkmY9/PxOA9m+c7MZ4aT85nvd2a+Y+6OiIhET6VkByAiIsmhBCAiElFKACIiEaUEICISUUoAIiIRpQQgIhJRP0h2AMeiQYMG3rJly2SHISJSYSxatOgrd29Y2rwKlQBatmzJwoULkx2GiEiFYWafHW6emoBERCJKCUBEJKKUAEREIkoJQEQkopQAREQiSglARCSilABERCJKCUBEJKIq1INgIiJRMvqO6TGXveXRy455/aoBiIhE1AlRA/jRnc/FXHbRI9fEMRIRkYpDNQARkYg6IWoAUjaf33dOzGVb3L0sjpFULBmjMmIql3VrVpwjqTjmds6MuWzm23PjGIkUpRqAiEhEKQGIiESUEoCISEQpAYiIRFRcE4CZVTKzkSWmtTaz/wyH65nZPWb2MzP7RTxjERGR4uKWAMysHnA7UPLy/6+AauHwMOBf7v4CcJWZVY9XPCIiUlzcEoC7f+3ufwG+LZhmZhcCi4sUywCyw+FVQHq84hERkeISdg3AzE4CUoEvikxOBXaEw9uBxomKR0Qk6hJ5EfhS4PWS23f3/eHwScD+EvMxs6FmttDMFubn58c7RhGRyEjkk8CtCc742wBtzKwdsNHMarj7DqA2cMgvvLuPAcYApKWleQLjFRE5oSUsAbj7XwHMrAuw292Xmtk7QDvgXeBcYHii4hERibp43gVU08x+A7Q1s2HheCowBLjEzM4DngD6m9m1wHh33xqveEREpLi41QDcfTswMvwU2E6QAIq6I14xiIjI4elJYBGRiFICEBGJKCUAEZGIUgIQEYkoJQARkYhSAhARiSglABGRiFICEBGJKCUAEZGIUgIQEYkoJQARkYhSAhARiSglABGRiFICEBGJKCUAEZGIUgIQEYkoJQARkYhSAhARiSglABGRiIrbO4EBzKwS8Ki7/8bM6gI/BxoCm9z9CTOrB9wGfAxUcve/xzMeERE5KG4JIPxxHwJkhpMGAf/n7nvN7AMzexm4CXjD3eea2Qwz+4e774xXTCIiclDcmoDc/Wt3/wvwbTjpQqB7OJwHNAUygOxw2iogPV7xiIhIcXFtAirh18BOMzOgJZADpAI7wvnbgcYJjEdEJNISdhHY3b9x970E1wFGhk09ldx9f1jkJGB/yeXMbKiZLTSzhfn5+YkKV0TkhJfQu4DM7N+A2u7+Yjhpo5nVCIdrA4f8wrv7GHdPc/e0hg0bJipUEZETXsISgJmlAN3d/X/MrKmZtQPeAdqFRc4FliQqHhGRqIvnXUA1geuBtmY2DGgO/IeZPQBUDsefAH5rZm2B8e6+NV7xiIhIcXFLAO6+HRgZfgoMK1FsG3BHvGIQEZHD05PAIiIRpQQgIhJRSgAiIhGlBCAiElFKACIiEaUEICISUUoAIiIRpQQgIhJRSgAiIhGlBCAiElFKACIiEaUEICISUUoAIiIRpQQgIhJRSgAiIhGlBCAiElFKACIiEaUEICISUXFNAGZWycxGhsMpZvZHM7vSzG493DQREUmMuCUAM6sH3A5khpOuAVa7+0tAezNrdZhpIiKSAHFLAO7+tbv/Bfg2nJQBZIfDSwkSQ2nTREQkARJ5DSAV2BEObwcaH2aaiIgkQCITQCVgfzh8Ujhc2rRizGyomS00s4X5+fkJCVREJAoSmQA2AjXC4dpA/mGmFePuY9w9zd3TGjZsmJBARUSiIJEJ4B2gXTh8PvD+YaaJiEgC/CBeKzazmsD1QFszGwY8DdxrZg4sc/eVZvY58EDRafGKR0REiotbAnD37cDI8FPg9hJldpScJiIiiaEngUVEIkoJQEQkopQAREQiSglARCSilABERCJKCUBEJKKUAEREIkoJQEQkopQAREQiSglARCSilABERCJKCUBEJKKUAEREIkoJQEQkopQAREQiSglARCSiYkoAZtYr/LeamXUzs57xDUtEROLtiG8EM7OfAjWBzmZW8Eb2ykAH4LU4xyYiInF0tBrAywRJwop8dgEPxjkuERGJsyPWANzdgbFmNtXdNxdMN7M2x7ohMzsJuJoggTQGngKGA0uApu4+6ljXKSIiZRfrS+F/bGaXETT/GEET0LnHuK2ewHJ3X2JmVwK/Bla7+0tm9rSZtXL3T49xnSIiUkaxJoCzgT8D34XjvyjDtrYB95rZz4GmQFsOXkdYCmQCSgAiIgkS622gHwKfuPtn7v4Z8MGxbsjd3wa+BlYAO4DU8F+A7QTNQiIikiCx1gCGA38ys68JmoDacow/2GbWBJgPvAPcB6wC9oezTyoyXHK5ocBQgBYtWhzLJkVE5AhiTQAPuvurBSNlfA7geuBP7r7XzD4FpgE1wnm1gfzSFnL3McAYgLS0NC/DdkVEpBQxNQEV/fEPNS3DtgxICYeXAg8B7cLx84H3y7BOEREpo5hqAGa2CVhO8CNeH9gKjD3Gbf0VuNnMvgAc+AvwgJk5sMzdVx7j+kRE5HuItQnoSnefVzBiZpcf64bcfQvwcInJtx/rekRE5PiINQGYmXUuGCZoupkan5BERCQRYk0AjwAzw+F9wIz4hCMiIokSawIY5O55ZpYK5IddRIiISAUW64Ngp5jZ68BLwDtmlh7HmEREJAFirQGc7u49CkbM7CbK8DSwiIiUH7HWAHaXGN9RaikREakwYq0BfGNmEwg6g2sKPBa/kEREJBFiTQDdCPrxeRfYCPw4bhGJiEhCxJoA3nX3yQUjZqYEICJSwcWaANqZWT3gE+AMoBkwMW5RiYhI3MV6Efhe4ADwU4Jum/8Qt4hERCQhYqoBuPs+4G9xjkUkZnv37mXdunXs3l3yBrX4uz/9/pjKrVyZnP4Nq1atSvPmzalcuXJSti8VR6xNQCLlyrp166hVqxYtW7bEzBK6bdsY2/baNGoT50gO5e5s3ryZdevW0apVq4RvXyqWWJuARMqV3bt3U79+/YT/+Jd3Zkb9+vWTUjOSikcJQCos/fiXTsdFYqUmIJEj2LlzJyNGjKBRo0akpqbSpEkTZsyeQZWqVRj47wOpfXLtZIcoUmZKACJHcO211/Kzn/2Mvn37ApCVlcX6tev59ttv6dW3lxKAVGhqAhI5jAULFrB8+fLCH3+AjIwMWv+wNbVq1UpiZCLHh2oAIocxd+5c2rQ59E6eAb8YwIPDHwRg65atPHzPw3Tp0YXVH63mpmE3sfqj1Xyy+hPWNFrDmjVruOSSS8jNzaVKlSqsWbOG2267LdG7IlKqhCYAM+tJ8CTxAeB54LfAEqCpu49KZCwiR3O4i6m1ah88+69Ttw7DHxxO9RrVmTZlGvmb8nlv3nvUOrkWXbt2pVmzZrz55pvUqVOHXr160axZs0SFL3JUCWsCMrP6wGB3/yvQCLgBWO3uLwHtzUw3LUu5kpmZSW5u7iHT3533brHxVR+t4tVXXmXnjp18t/s7rrjqCpYvWU56ejp5eXkMGTKEBQsWFI6LlBeJvAYwCHg/HL4faANkh+NLgcwExiJyVGlpaZxzzjlMmzatcNratWupXfvghd+Zr8xk1tRZ9OnXh/oN6rNh/QZmz5rN7+//PYsXLyYrK4upU6fy+OOPF46LlBeJbAI6G9hpZn2Ac4BUDr5YZjvQOIGxiMTk2Wef5d5772XlypU0a9aMBg0aUKV6FXKW5fDK5Fe4IOMCdu3cxdzZc2nSrAlz3phDk6ZNmDp5Kqc1Po1u3bqxatUqnn/+eapVq0a3bt2SvUsihRKZACoB37j7q2Z2JtAbuDmcdxJBJ3OHMLOhwFCAFi1aJCJOkUIpKSk88MADxablbsxl6r+mFo6nXZgGQOYlxSuxBV1B9OzZM85RipRNIpuAvgA2hMNfE7xmskY4XhvIL20hdx/j7mnuntawYcP4RykiEhGJTABvAc3D4XrAXUC7cPx8Dl4fEBGRBEhYAnD3eQBm9kuC20DHAheY2VXAMndPTt+5IiIRldDnANz93hKTbk/k9kVE5CB1BSEiElFKACJlkDU3i75d+vLAHx5g4nMTeeAPD/D6jNdLLTt27NjCZwlGjBjBpEmTSi3XtWtX1q9fH3MMRdcrUhbqC0hOCD+687njur5Fj1xzxPkZmRmc1e4suvfuTnpGOnt276HzuZ1JuzCNeg3qFSvboUMHHnvsMfr27cuwYcOoXr06APn5+axYsYIuXboA8MILL9C4ceyPwxRdr0hZqAYgchxUqVqFqtWqsu7zdYfMq1OnTuFwdnY2a9asYf/+/YwePbpw+rZt28jOzmbTpk0xb7PoekXKQjUAkePgi/VfUL1Gdeo3rM+4J8fR9JSmzN45m1tuuaVYuYULF7Jz506GDBlCdnY2O3fupFKlSnTo0IFx48ZRrVo1UlNTGT16NLVq1SI3N5cbb7yRrKwsNm7cyLZt2+jWrRsXXXRRkvZUTiRKACLfw4L3FrDl6y18seELnn3pWUY+OJJhw4fRMLUh/3j6H7z55pucdtppheXT09OZM2cOLVu2pEOHDnTp0oXOnTsD0LZtWwByc3N55513mDBhAtOmTcPdueKKK6hcuTLbt29n0KBBvPbaa0nZXzmxKAGIfA/tzmtHp66dCseXZS+jbr26AKSmprJgwYJiCaA0O3bsoEaNGoXjS5YsoVWroHPcgvb9PXv2MGXKFGrWrKkXvstxo2sAIsfR6W1O58v1XwKwbt062rVrd9iylStXZt++fcyfP7/Y9LPPPps1a9YAcODAAVauXEm/fv1o27YtPXr0AOCzzz6L0x5IlKgGIFIGWXOzWP7hcsyM0844jabNmwIw4k8jmDx+Ms1bNKdhw4b07t2b++67j8WLF7NixQrGjx/PqlWr+Oqrr+jduzcvvPACvXv3Zu3atbz11lvk5+fzxBNPcPHFFzNmzBiqV69Onz59OPfcc5k3bx55eXk0btyYlStX8sEHH7B48WJycnI488wzk3xEpCIyd092DDFLS0vzhQsXHjL9WG4BPNrtfVHy+X3nxFy2xd3L4hjJsVu5cmVhm3mi5W489CUxpSnoDTQZknl8SjO3c+yv+8h8e24cI6lYRt8xPeaytzx6WanTzWyRu6eVNk9NQCIiEaUEICISUUoAIiIRpQQgIhJRSgAiIhGlBCAiElF6DkBOCMdyS2ssjnbba9bcLB76r4e4sOOFtG7TmrzVeWRcnEFGZkZM6589ezavvPJKsQ7hRBJNCUCkDAq6g76k1yWkZ6SzdctWMttnkrU8i5q1ah51+czMTM4777zDzp88eTIDBgw4niGLHEJNQCLHwZbNW2jctDEpVVJiKp+Tk0NOTk6p8954443DzhM5nhJeAzCzusDvgOHACGAJ0NTdRyU6FpHva2n2UvI35fP6jNeZMGMCKSkp3HnTnVxx1RU89f5T3HPPPdSsWZMJEyawdevWwi4g9u3bx4gRI5gzZw7Tpk0jJSWFBQsWMHDgQF5//XU++ugjJk6cyKBBg5K9i3ICS0YN4GdAKnANsNrdXwLam1mrJMQi8r20O68dfa7oQ/NTmzP/7aBTt3sfuZeLOl/EKaecwowZM9i6dStPPvkkN9xwA126dMHMSE9PL1zHpEmTaNasGddeey316tWjT58+dOjQQT/+EncJTQBmdgaQF45mANnh8FIg9s5CRMqZ1me0Zu6bQR82W7dsZerkqXz22Wfs3r2b3NxcmjRpAkDHjh0Pafu/5ZZbGDFiBP3796dSpYN/kjt27EjcDkgkJboGcDawIhxOBQq+4duB2F+GKlLO1KhZg7V5a1m/dj2/u+139Orbq7AztiZNmrBu3cFXRS5fvrzYsh9//DGvvPIKDz30EHPnzi3sJvq9997jwIEDCd0PiZaEXQMwswwgC6gWTqoE7A+HTyoyXHK5ocBQgBYtWsQ5SqmoEt1badHuoFu0akGnrp2Y9tI0Fr63kAapDZj3r3lUsSrMmzePjh07cvvtt/Poo49yyimnkJ6ezrPPPsvq1atZsmQJ8+fPp3bt2mzYsIEBAwZQvXp1xo4dy5YtW4rVCESOt0ReBP4hcDrQAGgN7AQKXoNUG8gvbSF3HwOMgaA76PiHKXJ0GZkZTJ9bvKve0c8E9/RfPuByIOgO+rrrrgOgdevWxcq2bNmSwYMHA/Dkk08esv6nn376uMcsUlLCEoC7jwUws5YETUHvAO0ImoTOB+5JVCwiIpLg20DNrBpwK5AOTAa6m5kDy9x9ZSJjERGJuoQmAHffBdwRfgBeTeT2RUTkIF1hEhGJKCUAEZGIUmdwImWwb98+nv2/Z6nfoD579uxh185dDLlhyFGX27ZtG127dmXBggXxD1LkKJQA5ISQMSq2bphjlXVr1hHnz3hpBhdkXMDZ7c9m546djH4ktm6da9WqxfTp049eUCQB1AQkUgbffPMNH+V8BED1GtXp1qtbTMutXbuW999/P56hicRMNQCRMujdrzf9e/Rn8vjJ9OjTg2uGXsNr015j27ZtfLHuC37xy1/w38/8N6NHj2blypVccsklXH311fTv358777yTyy+/nHfffZfNmzeza9cu9uzZQ506dXjwwQf5wx/+wNKlS2nUqBGNGjXijTfe4M4776RZs2bJ3m05wSgBiJRBw9SGzHh7Bm/OepOpk6eS90kea/PW8syUZ1iWvYzJ4ycz8sGR7N69m/Hjx/PII4/QuXNnAJo2bQrA9OnT6dKlCz169CAnJ4f27dvz+9//nh49enD66afz61//mlmzZrF+/Xrefvttrr766mTuspyA1AQkUgaL3l9Erdq16DeoH2NeHMOGdRtwd+a9NY8vv/iShqkNAfjd737HyJEjS33716233srEiRPp0KEDu3fvBqBevXpUrlyZlJQUGjVqBEBKSgp79uxJ3M5JZCgBiJRB9oJsNn6xEYDKlSvTomULqlWvRqeunejeuzsXZV4EwMyZM3niiScYPnz4Iet44403ePrpp3n33Xd59VU9EymJpyYgkTKoVbsWs2fNplKlSmz5eguDfzWYzz75jBfHvUidunU4vc3p3HXXXaxevZpJkyZxww03ULt2bXr27MnHH3/M5MmTycvLY9KkSezfv59evXoxa9YscnJy+Oc//8mCBQtYvHgx77//PtOmTQPgiiuu4OSTT07ynsuJRAlATghHu23zeBt0zaFv6zq11anFxn/S+SeFw6tXry4c3rBhw2HXu2nTJgAuvfTSwlrDP/7xj+8Vq8jhqAlIRCSilABERCJKCUBEJKKUAEREIkoJQEQkopQAREQiSreByglhbufM47q+zLfnHtf1iZRHqgGIJNHkyZOTHYJEWMISgJmlmNktZvZbM7s/HP+jmV1pZrcmKg6R8mLHjh089dRTyQ5DIiyRTUD9gRfdfbOZTQHuAla7+0tm9rSZtXL3TxMYj0iZZS/MZuvXW9mzew9rPl7D7FmzueH2G1j90WrSLkzjgo4XMGPGDL755ht27drFGWecQf369Rk0aBB33XUXEydOZPjw4eTl5fHUU0+RmZnJ3r17yc3NpUqVKqxZs4bbbrst2bspJ7hEJoAfAnWA/wE+AX4JXB7OWwpkAkoAUiHMeX0O6Relk9Elg1P/7VTefeddLr3sUi6+9GIGXDqAv0/9O8899xyTJk0CoHv37vzzn/8kLS2N5s2b88wzz5CamkqTJk247rrrABg1ahR16tShV69e6vtfEiKR1wD+G3g2HG5HkAR2hOPbgcalLWRmQ81soZktzM/Pj3+UIjH4+S9/zsypM7my+5Xs2bOHShb8KaWkpFC1alVyV+SSkpJSWH7Xrl2F/fy0bNmS1NTUYuvbsWMHQ4YMYcGCBaSnp5OXl5ewfZHoSlgCcPdd7r7DzDoB/wK+A/aHs08qMlxyuTHunubuaQ0bNkxQtCJHNn/ufB4Y+QATXp3A3NlzOXDgABC8LH7v3r20/mFrvvrqq8Ly+/fvp+D7W6nSwT+7ypUrs2/fPubPn8/UqVN5/PHHWbx4MVlZie3cTqIpobeBmlldoJO7P2hmbYEa4azagE7vpcwSfdvm+rXrmTV1FgcOHKBz187MfGUmb/3zLVZ8uIJ7Hr6HuvXqctNNNzFu3Di2b9/OyJEj+eijj1i0aBGjRo3i/vvvp1q1agwcOJAxY8bQsWNHcnNzef7556lWrRrdusX2jmGR7yPRzwFcBTxkZj8A3iFoCloBnA/ck+BYRMrs5jtuLjbeqHEjul7ala6Xdi2c1rdv30OWW7ZsWbHx66+/vnC4Xbt2xzlKkSNL5G2gQwmuA2wENgELgAvM7CpgmbuvTFQsIsfT9Jem83ne56z4cEWyQxE5JgmrAbj7GGBMicm3J2r7IvFy2ZWXcdmVlyU7DJFjpieBRUQiSglARCSilABERCJKCUBEJKLUHbScEEbfMf24ru+WR498UTdrbhYP/ddDXNjxQlq3aU3e6jwyLs4gIzPjkLJjx46lQYMG9O3blxEjRnDOOecwcODAUtc7ePBgbrrpJi644ILjsh8iR6IEIFIGGZkZnNXuLC7pdQnpGels3bKVzPaZZC3PomatmsXKdujQgccee4y+ffsybNgwqlevftj1/vnPf6Z+/frxDl8EUBOQyHGxZfMWGjdtTEqVlEPm1alTp3A4OzubNWvWlLqO7777juzsbD7//PO4xSlSlGoAIt/D0uyl5G/K5/UZrzNhxgRSUlJ4cdyL1Kpdi+fXPc/w4cOLlV+4cCE7d+7knnvuYdSoUTRq1IgPP/yQk08+mYsvvpiZM2eyYcMGhgwZUtgtxIYNG+jevTsHDhxg0KBBPPTQQ+zcuZMPP/yQ+++/P0l7LicC1QBEvod257WjzxV9aH5qc+a/PZ9VuavIXpDNT376E9q3b8/LL79crHx6enrh8LRp0xg4cCCNGzfmnHPO4fzzz6d9+/YArF69mvfee4/+/ftz44038pvf/IazzjqLtLQ0GjRowIABA5g3b15C91VOPKoBiBwHrc9ozdw3gw7p9u/fz7y35lFtf7VizT8l9evXjylTptCkSRN69OhRbN7ixYupW7cuEPQYumrVqsJ5jRo1AsDMjvduSMQoAYgcBzVq1mBt3lranNWGrDlZdOraiTaN2rBu3Tr27dtX6jKVKlWif//+pc4788wzmTVrFgB79uyhadOmcYtdoksJQE4IR7tt83jLmpvF8g+XY2a0aNWCTl07Me2laeTm5NLslGa89MJLnNbkNDIyMnjuuedYvHgxK1asYPz48axatYqvvvqKTz/9lD59+lC7dm169uxJv379mDp1KpUqVeLqq6+mS5cujB8/ng0bNvDMM8+Qk5PDokWL+Pvf/875559PTk4Os2bNolevXgnddzlxKAGIlEFGZgbT5xZ/9mD0M6OLjbdp1AaAu+++m7vvvhuAv/3tbwCsWrWKNm3a8PDDD7Nlyxbuu+++Q64ZDB48+JDtFu1OuuANYyJlpQQgkgTNmjVj165dTJ8+nb1799K4cWPOOOOMZIclEaMEIJIE1atX5+abbz56QZE40m2gUmG5e7JDKJd0XCRWSgBSIVWtWpXNmzfrx64Ed2fz5s1UrVo12aFIBaAmIKmQmjdvzrp168jPz0/4tr/89suYyvnXyUlOVatWpXnz5knZtlQsSU8AZnY38CFwtrs/kOx4pGKoXLkyrVq1Ssq2rxt1XUzlsm7NinMkIt9PUpuAzOwSwNx9KlDZzDonMx4RkShJ9jWADCA7HM4GuiYxFhGRSEl2AkgFdoTD24HGSYxFRCRSLJl3UZjZ/wIT3X2OmfUALnf3m0uUGQoMDUd/CHyU4DBLagB8leQYygsdi4N0LA7SsTioPByLU929YWkzkn0ReCNQIxyuDRxyS4e7jwHGJDKoIzGzhe6eluw4ygMdi4N0LA7SsTiovB+LZDcBvQO0C4fTgfeSGIuISKQkOwG8BTQ0swEA7v5akuMREYmMpDYBufsBYFg4OjmZsRyDctMcVQ7oWBykY3GQjsVB5fpYJPUisIiIJE+ym4BERCRJIpMAzKy7mS0zs7+a2a/Cf688TNlrzaxvOPxHMxt4mHJvmVmzY4ihcL3lTZHj87iZXW9mfzaz7sew/CVmNvroJcsvM/uBmd1pZoPD78iwoy8FZlbLzBbEOz6R487dI/MBxgFdwuGqwBagYSnl2gPjwuG6QJVwuGHB8uF442PcfuF6y+OnxPGpB+wGase4bGWg/hHmD0j2/sWwD4OBtHC4BvDnY1j2mL4LUftUhP//EvF2B5YBjwPXA38Guh+m7LVA33D4j8DAI6z3WeCCZO9fwScyNYCS3H03sBMorUexrUWGzwNOM7OTgFsKJppZLeA8M0s9hs1uPXqRcqMhsJYgCcTizPBziLAmUeq8cqYu4W3J7r4DeCWWhczsFOCCOMZVoZlZDSC2HvTKCXd/A1gEvOzufwMeBKabWe1Sii8GfhoO/wWYeoRV/xYoN7XFZD8IljThH+12YGNY1c8Dmrp7yWaMNKA6wdnxeUB1MztA8J8+BNgFbDKzW4BtQBvgfwn6OWoE1ALedPf5cd6l4yXdzJoAVwI/dvfvzGw88AzQE7jH3beb2VVAHYInHWcSfJf+CHQJm7m+A84HJgE9gB+a2SB3n5j4XYrZBGCRmV0PvAQ8Ft6ifDLQAhgF/JLgRKAtMBt4EZgCPAJMNbMfA/WBakAVgqT/e+ABguSyMfx0Bx5x9/UJ27syKLE/ZwJXAPcDZwFvu/u/zOwnBMeoGvAxsBmYCDwMDArLtzSz64C5BLXFNsAe4DR3/2tCd6psjnRCVPKE8Usgp2QhM0sJ539M8HuTdFFMAJ3NrAFwCnAxQWb/f+7+pZn93sy6AWuKlP+AoFkkz8wWA3Pc/W0AM1sZ/tsG6OjuV4U/fga8DOwFapWf0PEAAAXRSURBVBL8MfRM0P59Xx940DXHeQQ/Ui8CQ919h5mdBfzEzF4DbnD3LmbWEXB3/8DMCtYxEHiIoAr9HfAqsKOc//gTfgfOBPoB1wBnEPxAdTOz84Hr3f1BM6sK/By4s8h3YUO4msuAOcDrwJnuvsTMHgzHVwGPu3uv8NpRZ4LjW54V3Z9VQDd3n2Jm04CFZtYJuMbdBwKY2RvApcBCYB3wH+6+ycy+cPenwjK3EvxozgLKdQKk9BOiG4FvCJLg/SXKF5ww3hPu50bg3LD8v4DeQFNgnJn9O8EJZFPgDYJrshOB/wzXca67D4/nzkWxCegDd5/i7iPdfQPBE8gFfXVsIjhrPaKwSltUe+BTAHef5u6fAQ70BzoSXG+oaHIIvqwA9cMv66kE+9IG+ALA3d9x9+wSy44mqA1MAQ4UTCzluJUrZtbR3b9x92cJEvapwWTrCTQn3GfgT8BvONiTbVGjCM56F3Pw//1rd99LkAw3htO+I6ghlHcl9+cAgLt/R/DjdS7BvhSoRtDJI0Ceu28qurLwOzCO4O/sA6Bl/EI/Lj5w9xeBT4Du4UlQhru/ACwhqBEVK19kuK+7TyKoESxz9wXhMphZa+BCd59C0GIw0t1XECTOr9x9MtApnjsG0UwAJS0nqA1A8Ee+9Ahl9xLUmi4qZR2nAZhZJTNrS9B+vJLgzAkzO/U4xpwI2wiufZxKcOFqEsH+QPBDWPjKKTM7u8SyZ7h7P4IzmUwOHrcLzaw8f+cuKrirK/zBXk1Qc3nN3V8mOEuDIDHezKFnfxBcKPwl8GOgTwJijreS+3MSBHdMASnACoJmwAIncbBPrwNFpu8Nl7kIuBz4NdCBoKm0Iig4IWoHnBSeFFQBvj3CMq+YWX+Cv5fXS8zrQHATSsF37fQi8wpOEuL+kFZkmoDCC5FpgJtZjrt/Hs66GbjezD4F8t19ZviWsg5htv85cHrYbDQT+BkwM7yG0JXgLqKbzexfYc+lOwmaPD4kyOAtCc4A2hLUNjqY2ZnufkgbYTKVOD6rCarnvyBopviS4IzYCfbpHYL28TsI2kU/MLPBQGsza0/wQ/otQdV2MsExuRao68HT3+XVN0C/8BpPA2Akwf/9jQTt2svN7GGgNUEz15Phfr4GnBFeL2gZ3jZ8EjDLzHoBZ5rZpQRnvR3M7AKg4Dbjl939m8Tu5jEptj9AQTPnj4BfuftmM/sfMxtC0Nz5G4Jee38E3Gpmw919F8EJxFCC704boKD5481E71AZbSM4yVsC9PCw2xoza87hf0cPhGf4pckBeoXrqAJsOEy5uNKTwCISMzOb4+5dkh1HvIUnRCMJ7tgZAXwNjCdoo/8hwYnPTiAL+A+C5t6rgdsJzub7A3cRXCcoOEl4haD56wDBieRVwD6CE6WXCWpUE8PPAuB5YLC7z4rbfioBiEgszOznBBf3L3f3RcmOpzwzs9OBTu4+1szqAncTPAP0YZJDK0YJQETkODOz6gQ1g88Jbns9Hfhr2BxWbigBiIhEVHm+I0NEROJICUBEJKIicxuoSFmEbbl/JLg3exPBPd0XEtzCOMbdK1L/TiLFKAGIHNlY4AV3nwZgZhkEz3bUIbi3XQlAKiw1AYkcRtj/z9kFP/4A7p5F8PRreX54SyQmqgGIHF4mkFvK9L8R9BOPmdUDHgWmA2e5+x/DDuUKe7sk6DW0ovV+KRGgBCByeKXeI+3u3xT0fOruX5vZLWFvqf9uZo2BbhTv7bLkuEi5oCYgkcObS3DmXkzYZXhRZ4fvR6hJ0GPmOIr3dllyXKRcUAIQOQx3XwgssyLvcQ47AdxaZHwQMMjdJxDcJdSC4H0CRXu7rIi9X0oE6ElgkSMI3+L0XwQdeq0neHfE5wRvD5tC8JKPXxC8BrATQa+ZawkuEu8i6EXy9KLjBT1JiiSbEoCISESpCUhEJKKUAEREIkoJQEQkopQAREQiSglARCSilABERCJKCUBEJKKUAEREIur/AwHzRH94tMttAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7ASZ7Wbskr2t"
      },
      "source": [
        "df.to_csv('cleaned data.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2Tg-34cX4dew"
      },
      "source": [
        "balance data-Undersample"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 316
        },
        "id": "aqTCu3ed4dK8",
        "outputId": "46f04d42-e215-4e57-ee2b-049dc2b6214e"
      },
      "source": [
        "from imblearn.under_sampling import RandomUnderSampler\n",
        "under_sampler = RandomUnderSampler(random_state=42)\n",
        "X_res, y_res = under_sampler.fit_resample(X_train, y_train)\n",
        "print(f\"Training target statistics: {Counter(y_res)}\")\n",
        "print(f\"Testing target statistics: {Counter(y_test)}\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/externals/six.py:31: FutureWarning: The module is deprecated in version 0.21 and will be removed in version 0.23 since we've dropped support for Python 2.7. Please rely on the official version of six (https://pypi.org/project/six/).\n",
            "  \"(https://pypi.org/project/six/).\", FutureWarning)\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/deprecation.py:144: FutureWarning: The sklearn.neighbors.base module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.neighbors. Anything that cannot be imported from sklearn.neighbors is now part of the private API.\n",
            "  warnings.warn(message, FutureWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-1-e21f554e231d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mimblearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munder_sampling\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mRandomUnderSampler\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0munder_sampler\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mRandomUnderSampler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrandom_state\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m42\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mX_res\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_res\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0munder_sampler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_resample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"Training target statistics: {Counter(y_res)}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"Testing target statistics: {Counter(y_test)}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'X_train' is not defined"
          ]
        }
      ]
    }
  ]
}