singleton classifier training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 414,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models import Word2Vec\n",
    "from string import punctuation\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from stem import IndonesianStemmer\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.layers import Dense, Embedding, Flatten, Input, Dropout, Concatenate, Reshape, Conv2D, MaxPool2D\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.model_selection import train_test_split\n",
    "from functools import reduce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_vector = Word2Vec.load('word2vec/id.bin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "stemmer = IndonesianStemmer()\n",
    "\n",
    "def is_number(word):\n",
    "    return word.replace(',', '.').replace('.', '').replace('-', '', 1).isdigit()\n",
    "    \n",
    "def preprocess_word(word):\n",
    "    if '\\\\' in word:\n",
    "        word = word.split('\\\\')[0]\n",
    "    \n",
    "    while word[-1] in punctuation and len(word) > 1:\n",
    "        word = word[:-1]\n",
    "    \n",
    "    while word[0] in punctuation and len(word) > 1:\n",
    "        word = word[1:]\n",
    "        \n",
    "    word = word.lower()\n",
    "        \n",
    "    if word not in word_vector.wv:\n",
    "        tmp = word.split('-')\n",
    "        if len(tmp) == 2 and tmp[0] == tmp[1]:\n",
    "            word = tmp[0]\n",
    "            \n",
    "    if word not in word_vector.wv:\n",
    "        word = stemmer.stem(word)\n",
    "\n",
    "    if word not in word_vector.wv:\n",
    "        tmp = word.split('-')\n",
    "        if len(tmp) == 2 and tmp[0] == tmp[1]:\n",
    "            word = tmp[0]\n",
    "            \n",
    "    if word not in word_vector.wv:\n",
    "        word = stemmer.stem(word)\n",
    "        \n",
    "    if is_number(word):\n",
    "        word = '<angka>'\n",
    "        \n",
    "    return word\n",
    "\n",
    "def preprocess_sentence(sentence):\n",
    "    return ' '.join([preprocess_word(word) for word in sentence.split() if preprocess_word(word) != ''])\n",
    "\n",
    "def preprocess_arr(arr):\n",
    "    return [preprocess_word(word) for word in arr if preprocess_word(word) != '']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_entity_types(labels):\n",
    "    entity_types = set()\n",
    "    \n",
    "    for label in labels:\n",
    "        for entity_type in label.split('|'):\n",
    "            entity_types.add(entity_type)\n",
    "    \n",
    "    return list(entity_types)\n",
    "\n",
    "def entity_to_bow(entities):\n",
    "    idx = {entities[i]: i for i in range(len(entities))}\n",
    "    \n",
    "    def f(label):\n",
    "        bow = [0 for _ in entities]\n",
    "        \n",
    "        for entity_type in label.split('|'):\n",
    "            bow[idx[entity_type]] = 1\n",
    "        \n",
    "        return bow\n",
    "    \n",
    "    return f\n",
    "\n",
    "def entity_to_id(entities):\n",
    "    idx = {entities[i]: i for i in range(len(entities))}\n",
    "    \n",
    "    def f(label):\n",
    "        return idx[label]\n",
    "    \n",
    "    return f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [],
   "source": [
    "markables = pd.read_csv('markables.csv')\n",
    "markables.previous_words = markables.previous_words.map(lambda x: preprocess_arr(eval(x)))\n",
    "markables.next_words = markables.next_words.map(lambda x: preprocess_arr(eval(x)))\n",
    "markables.entity = markables.entity.map(entity_to_bow(get_entity_types(markables.entity)))\n",
    "markables.first_pos_tag = markables.first_pos_tag.map(entity_to_id(get_entity_types(markables.first_pos_tag)))\n",
    "markables.is_singleton = markables.is_singleton.map(int)\n",
    "markables.text = markables.text.map(preprocess_sentence)\n",
    "\n",
    "tokenizer = Tokenizer()\n",
    "tokenizer.fit_on_texts(markables.text)\n",
    "tokenizer.fit_on_texts(markables.previous_words)\n",
    "tokenizer.fit_on_texts(markables.next_words)\n",
    "\n",
    "markables.previous_words = tokenizer.texts_to_sequences(markables.previous_words)\n",
    "markables.next_words = tokenizer.texts_to_sequences(markables.next_words)\n",
    "markables.text = tokenizer.texts_to_sequences(markables.text)\n",
    "\n",
    "markables.is_singleton = markables.is_singleton.map(lambda x: to_categorical(x, num_classes=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>num_words</th>\n",
       "      <th>first_pos_tag</th>\n",
       "      <th>entity</th>\n",
       "      <th>is_singleton</th>\n",
       "      <th>previous_words</th>\n",
       "      <th>next_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>[3440]</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>[0, 0, 1, 0, 0, 0, 0, 0, 0, 0]</td>\n",
       "      <td>[0.0, 1.0]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[12, 731, 1955, 621, 38, 1057, 778, 3289, 283,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>[1955, 621]</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 1, 0, 0, 0]</td>\n",
       "      <td>[0.0, 1.0]</td>\n",
       "      <td>[3440, 12, 731]</td>\n",
       "      <td>[38, 1057, 778, 3289, 283, 12, 1091, 283, 132, 2]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>[38, 1057, 778]</td>\n",
       "      <td>3</td>\n",
       "      <td>16</td>\n",
       "      <td>[1, 0, 0, 0, 0, 0, 0, 0, 0, 1]</td>\n",
       "      <td>[1.0, 0.0]</td>\n",
       "      <td>[3440, 12, 731, 1955, 621]</td>\n",
       "      <td>[3289, 283, 12, 1091, 283, 132, 2, 1926, 41, 302]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>[283]</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 1, 0, 0, 0]</td>\n",
       "      <td>[0.0, 1.0]</td>\n",
       "      <td>[3440, 12, 731, 1955, 621, 38, 1057, 778, 3289]</td>\n",
       "      <td>[12, 1091, 283, 132, 2, 1926, 41, 302, 9, 1754]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>[283, 132]</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 1, 0, 0, 0]</td>\n",
       "      <td>[0.0, 1.0]</td>\n",
       "      <td>[731, 1955, 621, 38, 1057, 778, 3289, 283, 12,...</td>\n",
       "      <td>[2, 1926, 41, 302, 9, 1754, 1955, 621, 3441, 214]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id             text  num_words  first_pos_tag  \\\n",
       "0   1           [3440]          1             15   \n",
       "1   2      [1955, 621]          2             15   \n",
       "2   3  [38, 1057, 778]          3             16   \n",
       "3   4            [283]          1             15   \n",
       "4   5       [283, 132]          2             15   \n",
       "\n",
       "                           entity is_singleton  \\\n",
       "0  [0, 0, 1, 0, 0, 0, 0, 0, 0, 0]   [0.0, 1.0]   \n",
       "1  [0, 0, 0, 0, 0, 0, 1, 0, 0, 0]   [0.0, 1.0]   \n",
       "2  [1, 0, 0, 0, 0, 0, 0, 0, 0, 1]   [1.0, 0.0]   \n",
       "3  [0, 0, 0, 0, 0, 0, 1, 0, 0, 0]   [0.0, 1.0]   \n",
       "4  [0, 0, 0, 0, 0, 0, 1, 0, 0, 0]   [0.0, 1.0]   \n",
       "\n",
       "                                      previous_words  \\\n",
       "0                                                 []   \n",
       "1                                    [3440, 12, 731]   \n",
       "2                         [3440, 12, 731, 1955, 621]   \n",
       "3    [3440, 12, 731, 1955, 621, 38, 1057, 778, 3289]   \n",
       "4  [731, 1955, 621, 38, 1057, 778, 3289, 283, 12,...   \n",
       "\n",
       "                                          next_words  \n",
       "0  [12, 731, 1955, 621, 38, 1057, 778, 3289, 283,...  \n",
       "1  [38, 1057, 778, 3289, 283, 12, 1091, 283, 132, 2]  \n",
       "2  [3289, 283, 12, 1091, 283, 132, 2, 1926, 41, 302]  \n",
       "3    [12, 1091, 283, 132, 2, 1926, 41, 302, 9, 1754]  \n",
       "4  [2, 1926, 41, 302, 9, 1754, 1955, 621, 3441, 214]  "
      ]
     },
     "execution_count": 257,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "markables.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "found: 3065\n",
      "not found: 570\n",
      "total found: 140989\n",
      "total not found: 16018\n"
     ]
    }
   ],
   "source": [
    "reverse_word_map = dict(map(reversed, tokenizer.word_index.items()))\n",
    "\n",
    "def sequence_to_text(sequence):\n",
    "    return ' '.join([reverse_word_map[word_id] for word_id in sequence if word_id > 0])\n",
    "\n",
    "not_found = set()\n",
    "found = set()\n",
    "total_found = 0\n",
    "total_not_found = 0\n",
    "\n",
    "for sentence in markables.text.map(sequence_to_text):\n",
    "    for word in sentence.split():\n",
    "        if word not in word_vector.wv:\n",
    "            not_found.add(word)\n",
    "            total_not_found += 1\n",
    "        else:\n",
    "            found.add(word)\n",
    "            total_found += 1\n",
    "\n",
    "for sentence in list(markables.previous_words.map(sequence_to_text)) + list(markables.next_words.map(sequence_to_text)):\n",
    "    for word in sentence.split():\n",
    "        if word not in word_vector.wv:\n",
    "            not_found.add(word)\n",
    "            total_not_found += 1\n",
    "        else:\n",
    "            found.add(word)\n",
    "            total_found += 1\n",
    "\n",
    "print(\"found: %d\" % len(found))\n",
    "print(\"not found: %d\" % len(not_found))\n",
    "print(\"total found: %d\" % total_found)\n",
    "print(\"total not found: %d\" % total_not_found)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_val = min(map(min, word_vector.wv.vectors))\n",
    "max_val = max(map(max, word_vector.wv.vectors))\n",
    "\n",
    "vocab_size = len(tokenizer.word_index) + 1\n",
    "vector_size = word_vector.wv.vector_size\n",
    "\n",
    "embedding_matrix = np.zeros((vocab_size, vector_size))\n",
    "\n",
    "for word, i in tokenizer.word_index.items():\n",
    "    if word in word_vector.wv:\n",
    "        embedding_matrix[i] = word_vector.wv[word]\n",
    "    else:\n",
    "        embedding_matrix[i] = np.random.rand(300) * (max_val - min_val) + min_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_text_length = 10\n",
    "max_prev_words_length = 5\n",
    "max_next_words_length = 5\n",
    "\n",
    "padded_text = pad_sequences(markables.text, maxlen=max_text_length, padding='post')\n",
    "padded_prev_words = pad_sequences(markables.previous_words.map(lambda seq: seq[(-1*max_prev_words_length):]), maxlen=max_prev_words_length, padding='pre')\n",
    "padded_next_words = pad_sequences(markables.next_words.map(lambda seq: seq[:max_next_words_length]), maxlen=max_next_words_length, padding='post')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 381,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_idx, test_idx = train_test_split([i for i in range(len(padded_text))], test_size=0.3)\n",
    "\n",
    "padded_text_train = padded_text[train_idx]\n",
    "padded_text_test = padded_text[test_idx]\n",
    "\n",
    "padded_prev_words_train = padded_prev_words[train_idx]\n",
    "padded_prev_words_test = padded_prev_words[test_idx]\n",
    "\n",
    "padded_next_words_train = padded_next_words[train_idx]\n",
    "padded_next_words_test = padded_next_words[test_idx]\n",
    "\n",
    "numeric_train = markables[['num_words', 'first_pos_tag', 'entity']].iloc[train_idx]\n",
    "numeric_test = markables[['num_words', 'first_pos_tag', 'entity']].iloc[test_idx]\n",
    "\n",
    "label_train = markables.is_singleton[train_idx]\n",
    "label_test = markables.is_singleton[test_idx]\n",
    "\n",
    "\n",
    "numeric_train = np.array(list(map(lambda p: reduce(lambda x,y: x + y, [i if type(i) is list else [i] for i in p]), numeric_train.values)))\n",
    "numeric_test = np.array(list(map(lambda p: reduce(lambda x,y: x + y, [i if type(i) is list else [i] for i in p]), numeric_test.values)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 534,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_184 (InputLayer)          (None, 5)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_185 (InputLayer)          (None, 5)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_161 (Embedding)       (None, 5, 300)       1090800     input_184[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "embedding_162 (Embedding)       (None, 5, 300)       1090800     input_185[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "input_183 (InputLayer)          (None, 10)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "reshape_22 (Reshape)            (None, 5, 300, 1)    0           embedding_161[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "reshape_23 (Reshape)            (None, 5, 300, 1)    0           embedding_162[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "embedding_160 (Embedding)       (None, 10, 300)      1090800     input_183[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_63 (Conv2D)              (None, 4, 1, 64)     38464       reshape_22[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_64 (Conv2D)              (None, 3, 1, 64)     57664       reshape_22[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_65 (Conv2D)              (None, 2, 1, 64)     76864       reshape_22[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_66 (Conv2D)              (None, 4, 1, 64)     38464       reshape_23[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_67 (Conv2D)              (None, 3, 1, 64)     57664       reshape_23[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_68 (Conv2D)              (None, 2, 1, 64)     76864       reshape_23[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "reshape_21 (Reshape)            (None, 10, 300, 1)   0           embedding_160[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_61 (MaxPooling2D) (None, 1, 1, 64)     0           conv2d_63[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_62 (MaxPooling2D) (None, 1, 1, 64)     0           conv2d_64[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_63 (MaxPooling2D) (None, 1, 1, 64)     0           conv2d_65[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_64 (MaxPooling2D) (None, 1, 1, 64)     0           conv2d_66[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_65 (MaxPooling2D) (None, 1, 1, 64)     0           conv2d_67[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_66 (MaxPooling2D) (None, 1, 1, 64)     0           conv2d_68[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "input_186 (InputLayer)          (None, 12)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_60 (Conv2D)              (None, 9, 1, 64)     38464       reshape_21[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_61 (Conv2D)              (None, 8, 1, 64)     57664       reshape_21[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_62 (Conv2D)              (None, 7, 1, 64)     76864       reshape_21[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_82 (Concatenate)    (None, 3, 1, 64)     0           max_pooling2d_61[0][0]           \n",
      "                                                                 max_pooling2d_62[0][0]           \n",
      "                                                                 max_pooling2d_63[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_83 (Concatenate)    (None, 3, 1, 64)     0           max_pooling2d_64[0][0]           \n",
      "                                                                 max_pooling2d_65[0][0]           \n",
      "                                                                 max_pooling2d_66[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dense_628 (Dense)               (None, 64)           832         input_186[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_58 (MaxPooling2D) (None, 1, 1, 64)     0           conv2d_60[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_59 (MaxPooling2D) (None, 1, 1, 64)     0           conv2d_61[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_60 (MaxPooling2D) (None, 1, 1, 64)     0           conv2d_62[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_159 (Flatten)           (None, 192)          0           concatenate_82[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "flatten_160 (Flatten)           (None, 192)          0           concatenate_83[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_489 (Dropout)           (None, 64)           0           dense_628[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_81 (Concatenate)    (None, 3, 1, 64)     0           max_pooling2d_58[0][0]           \n",
      "                                                                 max_pooling2d_59[0][0]           \n",
      "                                                                 max_pooling2d_60[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dense_625 (Dense)               (None, 16)           3088        flatten_159[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_626 (Dense)               (None, 16)           3088        flatten_160[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_629 (Dense)               (None, 32)           2080        dropout_489[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "flatten_158 (Flatten)           (None, 192)          0           concatenate_81[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_84 (Concatenate)    (None, 32)           0           dense_625[0][0]                  \n",
      "                                                                 dense_626[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_490 (Dropout)           (None, 32)           0           dense_629[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_624 (Dense)               (None, 16)           3088        flatten_158[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_627 (Dense)               (None, 16)           528         concatenate_84[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dense_630 (Dense)               (None, 16)           528         dropout_490[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_85 (Concatenate)    (None, 48)           0           dense_624[0][0]                  \n",
      "                                                                 dense_627[0][0]                  \n",
      "                                                                 dense_630[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_631 (Dense)               (None, 32)           1568        concatenate_85[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_491 (Dropout)           (None, 32)           0           dense_631[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_632 (Dense)               (None, 8)            264         dropout_491[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dropout_492 (Dropout)           (None, 8)            0           dense_632[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_633 (Dense)               (None, 2)            18          dropout_492[0][0]                \n",
      "==================================================================================================\n",
      "Total params: 3,806,458\n",
      "Trainable params: 534,058\n",
      "Non-trainable params: 3,272,400\n",
      "__________________________________________________________________________________________________\n",
      "None\n",
      "Epoch 1/10\n",
      "4984/4984 [==============================] - 25s 5ms/step - loss: 0.3795 - acc: 0.8423\n",
      "Epoch 2/10\n",
      "4984/4984 [==============================] - 6s 1ms/step - loss: 0.2705 - acc: 0.8784\n",
      "Epoch 3/10\n",
      "4984/4984 [==============================] - 7s 1ms/step - loss: 0.2007 - acc: 0.9163\n",
      "Epoch 4/10\n",
      "4984/4984 [==============================] - 7s 1ms/step - loss: 0.1432 - acc: 0.9454\n",
      "Epoch 5/10\n",
      "4984/4984 [==============================] - 6s 1ms/step - loss: 0.0972 - acc: 0.9675\n",
      "Epoch 6/10\n",
      "4984/4984 [==============================] - 7s 1ms/step - loss: 0.0751 - acc: 0.9785\n",
      "Epoch 7/10\n",
      "4984/4984 [==============================] - 7s 1ms/step - loss: 0.0472 - acc: 0.9866\n",
      "Epoch 8/10\n",
      "4984/4984 [==============================] - 7s 1ms/step - loss: 0.0423 - acc: 0.9896\n",
      "Epoch 9/10\n",
      "4984/4984 [==============================] - 7s 1ms/step - loss: 0.0356 - acc: 0.9910\n",
      "Epoch 10/10\n",
      "4984/4984 [==============================] - 7s 1ms/step - loss: 0.0237 - acc: 0.9946\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x1afadead68>"
      ]
     },
     "execution_count": 534,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filter_sizes = [2,3,4]\n",
    "num_filters = 64\n",
    "\n",
    "words_input = Input(shape=(max_text_length,))\n",
    "words_embedding = Embedding(vocab_size, vector_size, weights=[embedding_matrix], input_length=10, trainable=False)(words_input)\n",
    "reshape_words = Reshape((max_text_length,vector_size,1))(words_embedding)\n",
    "conv_0_words = Conv2D(num_filters, kernel_size=(filter_sizes[0], vector_size), padding='valid', kernel_initializer='normal', activation='relu')(reshape_words)\n",
    "conv_1_words = Conv2D(num_filters, kernel_size=(filter_sizes[1], vector_size), padding='valid', kernel_initializer='normal', activation='relu')(reshape_words)\n",
    "conv_2_words = Conv2D(num_filters, kernel_size=(filter_sizes[2], vector_size), padding='valid', kernel_initializer='normal', activation='relu')(reshape_words)\n",
    "maxpool_0_words = MaxPool2D(pool_size=(max_text_length - filter_sizes[0] + 1, 1), strides=(1,1), padding='valid')(conv_0_words)\n",
    "maxpool_1_words = MaxPool2D(pool_size=(max_text_length - filter_sizes[1] + 1, 1), strides=(1,1), padding='valid')(conv_1_words)\n",
    "maxpool_2_words = MaxPool2D(pool_size=(max_text_length - filter_sizes[2] + 1, 1), strides=(1,1), padding='valid')(conv_2_words)\n",
    "words_representation = Concatenate(axis=1)([maxpool_0_words, maxpool_1_words, maxpool_2_words])\n",
    "words_representation = Flatten()(words_representation)\n",
    "words_representation = Dense(16, activation='sigmoid')(words_representation)\n",
    "\n",
    "prev_words_input = Input(shape=(max_prev_words_length,))\n",
    "prev_words_embedding = Embedding(vocab_size, vector_size, weights=[embedding_matrix], input_length=10, trainable=False)(prev_words_input)\n",
    "reshape_prev_words = Reshape((max_prev_words_length,vector_size,1))(prev_words_embedding)\n",
    "conv_0_prev_words = Conv2D(num_filters, kernel_size=(filter_sizes[0], vector_size), padding='valid', kernel_initializer='normal', activation='relu')(reshape_prev_words)\n",
    "conv_1_prev_words = Conv2D(num_filters, kernel_size=(filter_sizes[1], vector_size), padding='valid', kernel_initializer='normal', activation='relu')(reshape_prev_words)\n",
    "conv_2_prev_words = Conv2D(num_filters, kernel_size=(filter_sizes[2], vector_size), padding='valid', kernel_initializer='normal', activation='relu')(reshape_prev_words)\n",
    "maxpool_0_prev_words = MaxPool2D(pool_size=(max_prev_words_length - filter_sizes[0] + 1, 1), strides=(1,1), padding='valid')(conv_0_prev_words)\n",
    "maxpool_1_prev_words = MaxPool2D(pool_size=(max_prev_words_length - filter_sizes[1] + 1, 1), strides=(1,1), padding='valid')(conv_1_prev_words)\n",
    "maxpool_2_prev_words = MaxPool2D(pool_size=(max_prev_words_length - filter_sizes[2] + 1, 1), strides=(1,1), padding='valid')(conv_2_prev_words)\n",
    "prev_words_representation = Concatenate(axis=1)([maxpool_0_prev_words, maxpool_1_prev_words, maxpool_2_prev_words])\n",
    "prev_words_representation = Flatten()(prev_words_representation)\n",
    "prev_words_representation = Dense(16, activation='sigmoid')(prev_words_representation)\n",
    "\n",
    "next_words_input = Input(shape=(max_next_words_length,))\n",
    "next_words_embedding = Embedding(vocab_size, vector_size, weights=[embedding_matrix], input_length=10, trainable=False)(next_words_input)\n",
    "reshape_next_words = Reshape((max_next_words_length,vector_size,1))(next_words_embedding)\n",
    "conv_0_next_words = Conv2D(num_filters, kernel_size=(filter_sizes[0], vector_size), padding='valid', kernel_initializer='normal', activation='relu')(reshape_next_words)\n",
    "conv_1_next_words = Conv2D(num_filters, kernel_size=(filter_sizes[1], vector_size), padding='valid', kernel_initializer='normal', activation='relu')(reshape_next_words)\n",
    "conv_2_next_words = Conv2D(num_filters, kernel_size=(filter_sizes[2], vector_size), padding='valid', kernel_initializer='normal', activation='relu')(reshape_next_words)\n",
    "maxpool_0_next_words = MaxPool2D(pool_size=(max_next_words_length - filter_sizes[0] + 1, 1), strides=(1,1), padding='valid')(conv_0_next_words)\n",
    "maxpool_1_next_words = MaxPool2D(pool_size=(max_next_words_length - filter_sizes[1] + 1, 1), strides=(1,1), padding='valid')(conv_1_next_words)\n",
    "maxpool_2_next_words = MaxPool2D(pool_size=(max_next_words_length - filter_sizes[2] + 1, 1), strides=(1,1), padding='valid')(conv_2_next_words)\n",
    "next_words_representation = Concatenate(axis=1)([maxpool_0_next_words, maxpool_1_next_words, maxpool_2_next_words])\n",
    "next_words_representation = Flatten()(next_words_representation)\n",
    "next_words_representation = Dense(16, activation='sigmoid')(next_words_representation)\n",
    "\n",
    "context_representation = Concatenate()([prev_words_representation, next_words_representation])\n",
    "context_representation = Dense(16, activation='sigmoid')(context_representation)\n",
    "\n",
    "numeric_input = Input(shape=(12,))\n",
    "numeric_representation = Dense(64, activation='relu')(numeric_input)\n",
    "numeric_representation = Dropout(0.2)(numeric_representation)\n",
    "numeric_representation = Dense(32, activation='relu')(numeric_representation)\n",
    "numeric_representation = Dropout(0.2)(numeric_representation)\n",
    "numeric_representation = Dense(16, activation='sigmoid')(numeric_representation)\n",
    "\n",
    "markable_representation = Concatenate()([words_representation, context_representation, numeric_representation])\n",
    "# markable_representation = Dense(64, activation='relu')(markable_representation)\n",
    "# markable_representation = Dropout(0.2)(markable_representation)\n",
    "markable_representation = Dense(32, activation='relu')(markable_representation)\n",
    "markable_representation = Dropout(0.2)(markable_representation)\n",
    "# markable_representation = Dense(16, activation='relu')(markable_representation)\n",
    "# markable_representation = Dropout(0.2)(markable_representation)\n",
    "markable_representation = Dense(8, activation='relu')(markable_representation)\n",
    "markable_representation = Dropout(0.2)(markable_representation)\n",
    "\n",
    "output_layer = Dense(2, activation='softmax')(markable_representation)\n",
    "\n",
    "model = Model(inputs=[words_input, prev_words_input, next_words_input, numeric_input], outputs=[output_layer])\n",
    "\n",
    "model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "# summarize the model\n",
    "print(model.summary())\n",
    "# fit the model\n",
    "model.fit([padded_text_train, padded_prev_words_train, padded_next_words_train, numeric_train], np.stack(label_train.values), epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 536,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "threshold 0.100000:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.59      0.49      0.54       278\n",
      "           1       0.93      0.95      0.94      1858\n",
      "\n",
      "   micro avg       0.89      0.89      0.89      2136\n",
      "   macro avg       0.76      0.72      0.74      2136\n",
      "weighted avg       0.88      0.89      0.88      2136\n",
      "\n",
      "threshold 0.200000:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.57      0.53      0.55       278\n",
      "           1       0.93      0.94      0.93      1858\n",
      "\n",
      "   micro avg       0.89      0.89      0.89      2136\n",
      "   macro avg       0.75      0.73      0.74      2136\n",
      "weighted avg       0.88      0.89      0.88      2136\n",
      "\n",
      "threshold 0.300000:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.55      0.54      0.55       278\n",
      "           1       0.93      0.93      0.93      1858\n",
      "\n",
      "   micro avg       0.88      0.88      0.88      2136\n",
      "   macro avg       0.74      0.74      0.74      2136\n",
      "weighted avg       0.88      0.88      0.88      2136\n",
      "\n",
      "threshold 0.400000:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.54      0.55      0.55       278\n",
      "           1       0.93      0.93      0.93      1858\n",
      "\n",
      "   micro avg       0.88      0.88      0.88      2136\n",
      "   macro avg       0.74      0.74      0.74      2136\n",
      "weighted avg       0.88      0.88      0.88      2136\n",
      "\n",
      "threshold 0.500000:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.54      0.57      0.55       278\n",
      "           1       0.93      0.93      0.93      1858\n",
      "\n",
      "   micro avg       0.88      0.88      0.88      2136\n",
      "   macro avg       0.74      0.75      0.74      2136\n",
      "weighted avg       0.88      0.88      0.88      2136\n",
      "\n",
      "threshold 0.600000:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.53      0.58      0.56       278\n",
      "           1       0.94      0.92      0.93      1858\n",
      "\n",
      "   micro avg       0.88      0.88      0.88      2136\n",
      "   macro avg       0.74      0.75      0.74      2136\n",
      "weighted avg       0.88      0.88      0.88      2136\n",
      "\n",
      "threshold 0.700000:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.52      0.60      0.56       278\n",
      "           1       0.94      0.92      0.93      1858\n",
      "\n",
      "   micro avg       0.88      0.88      0.88      2136\n",
      "   macro avg       0.73      0.76      0.74      2136\n",
      "weighted avg       0.88      0.88      0.88      2136\n",
      "\n",
      "threshold 0.800000:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.52      0.62      0.56       278\n",
      "           1       0.94      0.91      0.93      1858\n",
      "\n",
      "   micro avg       0.88      0.88      0.88      2136\n",
      "   macro avg       0.73      0.77      0.75      2136\n",
      "weighted avg       0.89      0.88      0.88      2136\n",
      "\n",
      "threshold 0.900000:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.49      0.65      0.56       278\n",
      "           1       0.95      0.90      0.92      1858\n",
      "\n",
      "   micro avg       0.87      0.87      0.87      2136\n",
      "   macro avg       0.72      0.78      0.74      2136\n",
      "weighted avg       0.89      0.87      0.88      2136\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def get_classes(output, threshold=0.5):\n",
    "    return list(map(lambda x: 1 if x[1] > threshold else 0, output))\n",
    "\n",
    "pred = model.predict([padded_text_test, padded_prev_words_test, padded_next_words_test, numeric_test])\n",
    "\n",
    "for i in range(1, 10):\n",
    "    print('threshold %f:' % (i * 0.1))\n",
    "    print(classification_report(get_classes(label_test), list(get_classes(pred, i*0.1))))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 540,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "perusahaan minyak asing | label: 1 | prediksi: 1\n",
      "nya | label: 1 | prediksi: 1\n",
      "majalah forbes | label: 1 | prediksi: 1\n",
      "pendiri facebook | label: 0 | prediksi: 0\n",
      "country director | label: 0 | prediksi: 1\n",
      "semester pertama | label: 1 | prediksi: 1\n",
      "nya | label: 1 | prediksi: 0\n",
      "stabilitas nilai tukar rupiah | label: 1 | prediksi: 1\n",
      "rp angka triliun | label: 1 | prediksi: 1\n",
      "nya | label: 1 | prediksi: 0\n",
      "kesepakatan | label: 1 | prediksi: 1\n",
      "nya | label: 0 | prediksi: 0\n",
      "jakarta | label: 1 | prediksi: 1\n",
      "stimulus fiskal | label: 1 | prediksi: 1\n",
      "hari minggu | label: 1 | prediksi: 1\n",
      "neraca pembayaran angka | label: 1 | prediksi: 1\n",
      "itu | label: 1 | prediksi: 1\n",
      "icmi | label: 0 | prediksi: 0\n",
      "peningkatan produksi ini | label: 1 | prediksi: 1\n",
      "pemda papua | label: 1 | prediksi: 1\n"
     ]
    }
   ],
   "source": [
    "from random import randint\n",
    "\n",
    "n = 20\n",
    "\n",
    "start = randint(0, len(label_test) - n)\n",
    "\n",
    "pred = get_classes(model.predict([padded_text_test, padded_prev_words_test, padded_next_words_test, numeric_test])[start:start+n], 0.6)\n",
    "lab = get_classes(label_test[start:start+n])\n",
    "\n",
    "for a, b, c in zip(list(map(sequence_to_text, padded_text_test))[start:start+n], lab, pred):\n",
    "    print(a,'| label:', b, '| prediksi:', c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 541,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('singleton_classifier.model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python3.6-conda",
   "language": "python",
   "name": "python3.6-conda"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}