model_lstm_5fold.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "dYxpVTQDDex5",
    "outputId": "4754ea58-e3f9-402b-f93f-dcdb4dbb0939"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import warnings\n",
    "from gensim.models import Word2Vec\n",
    "from tqdm import tqdm\n",
    "import random\n",
    "import tensorflow as tf\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.model_selection import KFold, StratifiedKFold\n",
    "\n",
    "from keras import backend as K\n",
    "from keras.preprocessing import text, sequence\n",
    "from keras import Model\n",
    "from keras.layers import Conv1D, Embedding, Input, Bidirectional, CuDNNLSTM, Dense, Concatenate, Masking, LSTM, SpatialDropout1D\n",
    "from keras.layers import BatchNormalization, Dropout, Activation\n",
    "from keras.layers import GlobalMaxPool1D, GlobalAveragePooling1D, GlobalAvgPool1D, GlobalMaxPooling1D\n",
    "from keras.optimizers import Adam\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau, Callback\n",
    "from keras.utils import to_categorical\n",
    "from keras_radam import RAdam\n",
    "from keras_lookahead import Lookahead\n",
    "\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = '1'\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_rows', None)\n",
    "\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install keras-rectified-adam\n",
    "# !pip install keras-lookahead"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fix_seed(seed):\n",
    "    np.random.seed(seed)\n",
    "    tf.set_random_seed(seed)\n",
    "\n",
    "seed = 2020\n",
    "fix_seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "c7LMAeCXD3Tr"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(250000, 2)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train = pd.read_csv('raw_data/train_set.csv', sep='\\t')\n",
    "df_test = pd.read_csv('raw_data/test_a.csv', sep='\\t')\n",
    "df_data = df_train.append(df_test)\n",
    "df_data = df_data.reset_index(drop=True)\n",
    "df_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2967 6758 339 2021 1854 3731 4109 3792 4149 15...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11.0</td>\n",
       "      <td>4464 486 6352 5619 2465 4802 1452 3137 5778 54...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.0</td>\n",
       "      <td>7346 4068 5074 3747 5681 6093 1777 2226 7354 6...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2.0</td>\n",
       "      <td>7159 948 4866 2109 5520 2490 211 3956 5520 549...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3.0</td>\n",
       "      <td>3646 3055 3055 2490 4659 6065 3370 5814 2465 5...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   label                                               text\n",
       "0    2.0  2967 6758 339 2021 1854 3731 4109 3792 4149 15...\n",
       "1   11.0  4464 486 6352 5619 2465 4802 1452 3137 5778 54...\n",
       "2    3.0  7346 4068 5074 3747 5681 6093 1777 2226 7354 6...\n",
       "3    2.0  7159 948 4866 2109 5520 2490 211 3956 5520 549...\n",
       "4    3.0  3646 3055 3055 2490 4659 6065 3370 5814 2465 5..."
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "pwyczuurF9zl",
    "outputId": "2c9cff1e-5961-4d5f-8a64-582ac0e3e6db"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generate seqs\n"
     ]
    }
   ],
   "source": [
    "max_words_num = None\n",
    "seq_len = 2000\n",
    "embedding_dim = 128\n",
    "col = 'text'\n",
    "\n",
    "print('Generate seqs')\n",
    "os.makedirs('seqs', exist_ok=True)\n",
    "seq_path = 'seqs/seqs_{}_{}.npy'.format(max_words_num, seq_len)\n",
    "word_index_path = 'seqs/word_index_{}_{}.npy'.format(max_words_num, seq_len)\n",
    "if not os.path.exists(seq_path) or not os.path.exists(word_index_path):\n",
    "    tokenizer = text.Tokenizer(num_words=max_words_num, lower=False, filters='')\n",
    "    tokenizer.fit_on_texts(df_data[col].values.tolist())\n",
    "    seqs = sequence.pad_sequences(tokenizer.texts_to_sequences(df_data[col].values.tolist()), maxlen=seq_len,\n",
    "                                  padding='post', truncating='pre')\n",
    "    word_index = tokenizer.word_index\n",
    "        \n",
    "    np.save(seq_path, seqs)\n",
    "    np.save(word_index_path, word_index)\n",
    "\n",
    "else:\n",
    "    seqs = np.load(seq_path)\n",
    "    word_index = np.load(word_index_path, allow_pickle=True).item()\n",
    "    \n",
    "# print('Generate embedding')\n",
    "# os.makedirs('embedding', exist_ok=True)\n",
    "# embedding_path = 'embedding/w2v_{}_{}.m'.format(col, embedding_dim)\n",
    "# if not os.path.exists(embedding_path):\n",
    "#     print('Training w2v')\n",
    "#     model = Word2Vec([[word for word in senetnce.split(' ')] for senetnce in df_data[col].values],\n",
    "#                       size=embedding_dim, window=20, workers=32, seed=seed, min_count=1, sg=1, hs=1)\n",
    "\n",
    "#     model.save(embedding_path)\n",
    "# else:\n",
    "#     model = Word2Vec.load(embedding_path)\n",
    "\n",
    "embedding = np.zeros((len(word_index) + 1, embedding_dim))\n",
    "# for word, i in tqdm(word_index.items()):\n",
    "#     embedding_vector = model[word] if word in model else None\n",
    "#     if embedding_vector is not None:\n",
    "#         embedding[i] = embedding_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0     38918\n",
       "1.0     36945\n",
       "2.0     31425\n",
       "3.0     22133\n",
       "4.0     15016\n",
       "5.0     12232\n",
       "6.0      9985\n",
       "7.0      8841\n",
       "8.0      7847\n",
       "9.0      5878\n",
       "10.0     4920\n",
       "11.0     3131\n",
       "12.0     1821\n",
       "13.0      908\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_data['label'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 模型训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs('model', exist_ok=True)\n",
    "os.makedirs('sub', exist_ok=True)\n",
    "os.makedirs('prob', exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_index = df_data[df_data['label'].notnull()].index.tolist()\n",
    "test_index = df_data[df_data['label'].isnull()].index.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model(emb, seq_len):\n",
    "    emb_layer = Embedding(\n",
    "        input_dim=emb.shape[0],\n",
    "        output_dim=emb.shape[1],\n",
    "#         weights=[emb],\n",
    "        input_length=seq_len,\n",
    "#         trainable=False\n",
    "    )\n",
    "    \n",
    "    seq = Input(shape=(seq_len, ))\n",
    "    seq_emb = emb_layer(seq)\n",
    "    \n",
    "    seq_emb = SpatialDropout1D(rate=0.2)(seq_emb)\n",
    "\n",
    "    lstm = Bidirectional(CuDNNLSTM(200, return_sequences=True))(seq_emb)\n",
    "    lstm_avg_pool = GlobalAveragePooling1D()(lstm)\n",
    "    lstm_max_pool = GlobalMaxPooling1D()(lstm)\n",
    "    x = Concatenate()([lstm_avg_pool, lstm_max_pool])\n",
    "    \n",
    "    x = Dropout(0.2)(Activation(activation='relu')(BatchNormalization()(Dense(1024)(x))))\n",
    "    out = Dense(14, activation='softmax')(x)\n",
    "    \n",
    "    model = Model(inputs=seq, outputs=out)\n",
    "    model.compile(loss='sparse_categorical_crossentropy', optimizer=Lookahead(RAdam()), metrics=['accuracy'])\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Evaluator(Callback):\n",
    "    def __init__(self, validation_data):\n",
    "        super().__init__()\n",
    "        self.best_val_f1 = 0.\n",
    "        self.x_val = validation_data[0]\n",
    "        self.y_val = validation_data[1]\n",
    "\n",
    "    def evaluate(self):\n",
    "        y_true = self.y_val\n",
    "        y_pred = self.model.predict(self.x_val).argmax(axis=1)\n",
    "        f1 = f1_score(y_true, y_pred, average='macro')\n",
    "        return f1\n",
    "\n",
    "    def on_epoch_end(self, epoch, logs=None):\n",
    "        val_f1 = self.evaluate()\n",
    "        if val_f1 > self.best_val_f1:\n",
    "            self.best_val_f1 = val_f1\n",
    "        logs['val_f1'] = val_f1\n",
    "        print(f'val_f1: {val_f1:.5f}, best_val_f1: {self.best_val_f1:.5f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /opt/conda/envs/tensorflow_py3/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n",
      "WARNING:tensorflow:From /opt/conda/envs/tensorflow_py3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:422: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
      "\n",
      "Train on 160000 samples, validate on 40000 samples\n",
      "Epoch 1/30\n",
      "160000/160000 [==============================] - 525s 3ms/step - loss: 0.7921 - accuracy: 0.7599 - val_loss: 0.4447 - val_accuracy: 0.8900\n",
      "val_f1: 0.85424, best_val_f1: 0.85424\n",
      "\n",
      "Epoch 00001: val_f1 improved from -inf to 0.85424, saving model to model/lstm_0.h5\n",
      "Epoch 2/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.1947 - accuracy: 0.9392 - val_loss: 0.3392 - val_accuracy: 0.8970\n",
      "val_f1: 0.88194, best_val_f1: 0.88194\n",
      "\n",
      "Epoch 00002: val_f1 improved from 0.85424 to 0.88194, saving model to model/lstm_0.h5\n",
      "Epoch 3/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.1451 - accuracy: 0.9539 - val_loss: 0.2601 - val_accuracy: 0.9172\n",
      "val_f1: 0.90292, best_val_f1: 0.90292\n",
      "\n",
      "Epoch 00003: val_f1 improved from 0.88194 to 0.90292, saving model to model/lstm_0.h5\n",
      "Epoch 4/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.1147 - accuracy: 0.9633 - val_loss: 0.2991 - val_accuracy: 0.9128\n",
      "val_f1: 0.88405, best_val_f1: 0.90292\n",
      "\n",
      "Epoch 00004: val_f1 did not improve from 0.90292\n",
      "Epoch 5/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0916 - accuracy: 0.9697 - val_loss: 0.3025 - val_accuracy: 0.9148\n",
      "val_f1: 0.90855, best_val_f1: 0.90855\n",
      "\n",
      "Epoch 00005: val_f1 improved from 0.90292 to 0.90855, saving model to model/lstm_0.h5\n",
      "Epoch 6/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0744 - accuracy: 0.9750 - val_loss: 0.3079 - val_accuracy: 0.9179\n",
      "val_f1: 0.88079, best_val_f1: 0.90855\n",
      "\n",
      "Epoch 00006: val_f1 did not improve from 0.90855\n",
      "Epoch 7/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0602 - accuracy: 0.9794 - val_loss: 0.3233 - val_accuracy: 0.9193\n",
      "val_f1: 0.90472, best_val_f1: 0.90855\n",
      "\n",
      "Epoch 00007: val_f1 did not improve from 0.90855\n",
      "\n",
      "Epoch 00007: ReduceLROnPlateau reducing learning rate to 0.0005000000237487257.\n",
      "Epoch 8/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0368 - accuracy: 0.9878 - val_loss: 0.2302 - val_accuracy: 0.9469\n",
      "val_f1: 0.93827, best_val_f1: 0.93827\n",
      "\n",
      "Epoch 00008: val_f1 improved from 0.90855 to 0.93827, saving model to model/lstm_0.h5\n",
      "Epoch 9/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0260 - accuracy: 0.9914 - val_loss: 0.2171 - val_accuracy: 0.9495\n",
      "val_f1: 0.94223, best_val_f1: 0.94223\n",
      "\n",
      "Epoch 00009: val_f1 improved from 0.93827 to 0.94223, saving model to model/lstm_0.h5\n",
      "Epoch 10/30\n",
      "160000/160000 [==============================] - 523s 3ms/step - loss: 0.0215 - accuracy: 0.9930 - val_loss: 0.2485 - val_accuracy: 0.9486\n",
      "val_f1: 0.93816, best_val_f1: 0.94223\n",
      "\n",
      "Epoch 00010: val_f1 did not improve from 0.94223\n",
      "Epoch 11/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0197 - accuracy: 0.9934 - val_loss: 0.2839 - val_accuracy: 0.9416\n",
      "val_f1: 0.93274, best_val_f1: 0.94223\n",
      "\n",
      "Epoch 00011: val_f1 did not improve from 0.94223\n",
      "\n",
      "Epoch 00011: ReduceLROnPlateau reducing learning rate to 0.0002500000118743628.\n",
      "Epoch 12/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0138 - accuracy: 0.9956 - val_loss: 0.2250 - val_accuracy: 0.9526\n",
      "val_f1: 0.94530, best_val_f1: 0.94530\n",
      "\n",
      "Epoch 00012: val_f1 improved from 0.94223 to 0.94530, saving model to model/lstm_0.h5\n",
      "Epoch 13/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0104 - accuracy: 0.9969 - val_loss: 0.2631 - val_accuracy: 0.9477\n",
      "val_f1: 0.93936, best_val_f1: 0.94530\n",
      "\n",
      "Epoch 00013: val_f1 did not improve from 0.94530\n",
      "Epoch 14/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0093 - accuracy: 0.9972 - val_loss: 0.2533 - val_accuracy: 0.9513\n",
      "val_f1: 0.94536, best_val_f1: 0.94536\n",
      "\n",
      "Epoch 00014: val_f1 improved from 0.94530 to 0.94536, saving model to model/lstm_0.h5\n",
      "\n",
      "Epoch 00014: ReduceLROnPlateau reducing learning rate to 0.0001250000059371814.\n",
      "Epoch 15/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0074 - accuracy: 0.9979 - val_loss: 0.2433 - val_accuracy: 0.9538\n",
      "val_f1: 0.94603, best_val_f1: 0.94603\n",
      "\n",
      "Epoch 00015: val_f1 improved from 0.94536 to 0.94603, saving model to model/lstm_0.h5\n",
      "Epoch 16/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0064 - accuracy: 0.9982 - val_loss: 0.2501 - val_accuracy: 0.9538\n",
      "val_f1: 0.94644, best_val_f1: 0.94644\n",
      "\n",
      "Epoch 00016: val_f1 improved from 0.94603 to 0.94644, saving model to model/lstm_0.h5\n",
      "Epoch 17/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0062 - accuracy: 0.9983 - val_loss: 0.2645 - val_accuracy: 0.9524\n",
      "val_f1: 0.94426, best_val_f1: 0.94644\n",
      "\n",
      "Epoch 00017: val_f1 did not improve from 0.94644\n",
      "Epoch 18/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0056 - accuracy: 0.9984 - val_loss: 0.2663 - val_accuracy: 0.9531\n",
      "val_f1: 0.94652, best_val_f1: 0.94652\n",
      "\n",
      "Epoch 00018: val_f1 improved from 0.94644 to 0.94652, saving model to model/lstm_0.h5\n",
      "\n",
      "Epoch 00018: ReduceLROnPlateau reducing learning rate to 6.25000029685907e-05.\n",
      "Epoch 19/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0049 - accuracy: 0.9987 - val_loss: 0.2602 - val_accuracy: 0.9543\n",
      "val_f1: 0.94765, best_val_f1: 0.94765\n",
      "\n",
      "Epoch 00019: val_f1 improved from 0.94652 to 0.94765, saving model to model/lstm_0.h5\n",
      "Epoch 20/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0045 - accuracy: 0.9987 - val_loss: 0.2560 - val_accuracy: 0.9542\n",
      "val_f1: 0.94755, best_val_f1: 0.94765\n",
      "\n",
      "Epoch 00020: val_f1 did not improve from 0.94765\n",
      "Epoch 21/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0043 - accuracy: 0.9989 - val_loss: 0.2559 - val_accuracy: 0.9547\n",
      "val_f1: 0.94778, best_val_f1: 0.94778\n",
      "\n",
      "Epoch 00021: val_f1 improved from 0.94765 to 0.94778, saving model to model/lstm_0.h5\n",
      "Epoch 22/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0039 - accuracy: 0.9990 - val_loss: 0.2668 - val_accuracy: 0.9543\n",
      "val_f1: 0.94698, best_val_f1: 0.94778\n",
      "\n",
      "Epoch 00022: val_f1 did not improve from 0.94778\n",
      "Epoch 23/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0042 - accuracy: 0.9989 - val_loss: 0.2639 - val_accuracy: 0.9542\n",
      "val_f1: 0.94756, best_val_f1: 0.94778\n",
      "\n",
      "Epoch 00023: val_f1 did not improve from 0.94778\n",
      "\n",
      "Epoch 00023: ReduceLROnPlateau reducing learning rate to 3.125000148429535e-05.\n",
      "Epoch 24/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0037 - accuracy: 0.9992 - val_loss: 0.2651 - val_accuracy: 0.9543\n",
      "val_f1: 0.94715, best_val_f1: 0.94778\n",
      "\n",
      "Epoch 00024: val_f1 did not improve from 0.94778\n",
      "Epoch 25/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0034 - accuracy: 0.9992 - val_loss: 0.2666 - val_accuracy: 0.9543\n",
      "val_f1: 0.94667, best_val_f1: 0.94778\n",
      "\n",
      "Epoch 00025: val_f1 did not improve from 0.94778\n",
      "\n",
      "Epoch 00025: ReduceLROnPlateau reducing learning rate to 1.5625000742147677e-05.\n",
      "Epoch 26/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0033 - accuracy: 0.9992 - val_loss: 0.2662 - val_accuracy: 0.9548\n",
      "val_f1: 0.94737, best_val_f1: 0.94778\n",
      "\n",
      "Epoch 00026: val_f1 did not improve from 0.94778\n",
      "Epoch 00026: early stopping\n",
      "Train on 160000 samples, validate on 40000 samples\n",
      "Epoch 1/30\n",
      "160000/160000 [==============================] - 524s 3ms/step - loss: 0.7780 - accuracy: 0.7650 - val_loss: 0.4899 - val_accuracy: 0.8577\n",
      "val_f1: 0.78691, best_val_f1: 0.78691\n",
      "\n",
      "Epoch 00001: val_f1 improved from -inf to 0.78691, saving model to model/lstm_1.h5\n",
      "Epoch 2/30\n",
      "160000/160000 [==============================] - 523s 3ms/step - loss: 0.1960 - accuracy: 0.9390 - val_loss: 0.2180 - val_accuracy: 0.9322\n",
      "val_f1: 0.92158, best_val_f1: 0.92158\n",
      "\n",
      "Epoch 00002: val_f1 improved from 0.78691 to 0.92158, saving model to model/lstm_1.h5\n",
      "Epoch 3/30\n",
      "160000/160000 [==============================] - 523s 3ms/step - loss: 0.1423 - accuracy: 0.9541 - val_loss: 0.2106 - val_accuracy: 0.9358\n",
      "val_f1: 0.92603, best_val_f1: 0.92603\n",
      "\n",
      "Epoch 00003: val_f1 improved from 0.92158 to 0.92603, saving model to model/lstm_1.h5\n",
      "Epoch 4/30\n",
      "160000/160000 [==============================] - 523s 3ms/step - loss: 0.1135 - accuracy: 0.9633 - val_loss: 0.2658 - val_accuracy: 0.9216\n",
      "val_f1: 0.90489, best_val_f1: 0.92603\n",
      "\n",
      "Epoch 00004: val_f1 did not improve from 0.92603\n",
      "Epoch 5/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0912 - accuracy: 0.9699 - val_loss: 0.4222 - val_accuracy: 0.8900\n",
      "val_f1: 0.87179, best_val_f1: 0.92603\n",
      "\n",
      "Epoch 00005: val_f1 did not improve from 0.92603\n",
      "\n",
      "Epoch 00005: ReduceLROnPlateau reducing learning rate to 0.0005000000237487257.\n",
      "Epoch 6/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0591 - accuracy: 0.9807 - val_loss: 0.2131 - val_accuracy: 0.9444\n",
      "val_f1: 0.93741, best_val_f1: 0.93741\n",
      "\n",
      "Epoch 00006: val_f1 improved from 0.92603 to 0.93741, saving model to model/lstm_1.h5\n",
      "Epoch 7/30\n",
      "160000/160000 [==============================] - 523s 3ms/step - loss: 0.0455 - accuracy: 0.9854 - val_loss: 0.2181 - val_accuracy: 0.9474\n",
      "val_f1: 0.94051, best_val_f1: 0.94051\n",
      "\n",
      "Epoch 00007: val_f1 improved from 0.93741 to 0.94051, saving model to model/lstm_1.h5\n",
      "Epoch 8/30\n",
      "160000/160000 [==============================] - 523s 3ms/step - loss: 0.0368 - accuracy: 0.9879 - val_loss: 0.2435 - val_accuracy: 0.9412\n",
      "val_f1: 0.93642, best_val_f1: 0.94051\n",
      "\n",
      "Epoch 00008: val_f1 did not improve from 0.94051\n",
      "Epoch 9/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0306 - accuracy: 0.9898 - val_loss: 0.2865 - val_accuracy: 0.9385\n",
      "val_f1: 0.92998, best_val_f1: 0.94051\n",
      "\n",
      "Epoch 00009: val_f1 did not improve from 0.94051\n",
      "\n",
      "Epoch 00009: ReduceLROnPlateau reducing learning rate to 0.0002500000118743628.\n",
      "Epoch 10/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0213 - accuracy: 0.9932 - val_loss: 0.2192 - val_accuracy: 0.9514\n",
      "val_f1: 0.94523, best_val_f1: 0.94523\n",
      "\n",
      "Epoch 00010: val_f1 improved from 0.94051 to 0.94523, saving model to model/lstm_1.h5\n",
      "Epoch 11/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0165 - accuracy: 0.9950 - val_loss: 0.2274 - val_accuracy: 0.9512\n",
      "val_f1: 0.94618, best_val_f1: 0.94618\n",
      "\n",
      "Epoch 00011: val_f1 improved from 0.94523 to 0.94618, saving model to model/lstm_1.h5\n",
      "Epoch 12/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0144 - accuracy: 0.9955 - val_loss: 0.2553 - val_accuracy: 0.9487\n",
      "val_f1: 0.94390, best_val_f1: 0.94618\n",
      "\n",
      "Epoch 00012: val_f1 did not improve from 0.94618\n",
      "Epoch 13/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0128 - accuracy: 0.9961 - val_loss: 0.2476 - val_accuracy: 0.9511\n",
      "val_f1: 0.94646, best_val_f1: 0.94646\n",
      "\n",
      "Epoch 00013: val_f1 improved from 0.94618 to 0.94646, saving model to model/lstm_1.h5\n",
      "Epoch 14/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0115 - accuracy: 0.9964 - val_loss: 0.2491 - val_accuracy: 0.9491\n",
      "val_f1: 0.94467, best_val_f1: 0.94646\n",
      "\n",
      "Epoch 00014: val_f1 did not improve from 0.94646\n",
      "Epoch 15/30\n",
      "160000/160000 [==============================] - 523s 3ms/step - loss: 0.0108 - accuracy: 0.9968 - val_loss: 0.2929 - val_accuracy: 0.9449\n",
      "val_f1: 0.93847, best_val_f1: 0.94646\n",
      "\n",
      "Epoch 00015: val_f1 did not improve from 0.94646\n",
      "\n",
      "Epoch 00015: ReduceLROnPlateau reducing learning rate to 0.0001250000059371814.\n",
      "Epoch 16/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0082 - accuracy: 0.9977 - val_loss: 0.2578 - val_accuracy: 0.9529\n",
      "val_f1: 0.94671, best_val_f1: 0.94671\n",
      "\n",
      "Epoch 00016: val_f1 improved from 0.94646 to 0.94671, saving model to model/lstm_1.h5\n",
      "Epoch 17/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0073 - accuracy: 0.9978 - val_loss: 0.2510 - val_accuracy: 0.9533\n",
      "val_f1: 0.94791, best_val_f1: 0.94791\n",
      "\n",
      "Epoch 00017: val_f1 improved from 0.94671 to 0.94791, saving model to model/lstm_1.h5\n",
      "Epoch 18/30\n",
      "160000/160000 [==============================] - 523s 3ms/step - loss: 0.0067 - accuracy: 0.9980 - val_loss: 0.2590 - val_accuracy: 0.9536\n",
      "val_f1: 0.94821, best_val_f1: 0.94821\n",
      "\n",
      "Epoch 00018: val_f1 improved from 0.94791 to 0.94821, saving model to model/lstm_1.h5\n",
      "Epoch 19/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0057 - accuracy: 0.9985 - val_loss: 0.2644 - val_accuracy: 0.9537\n",
      "val_f1: 0.94783, best_val_f1: 0.94821\n",
      "\n",
      "Epoch 00019: val_f1 did not improve from 0.94821\n",
      "Epoch 20/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0061 - accuracy: 0.9981 - val_loss: 0.2827 - val_accuracy: 0.9517\n",
      "val_f1: 0.94699, best_val_f1: 0.94821\n",
      "\n",
      "Epoch 00020: val_f1 did not improve from 0.94821\n",
      "\n",
      "Epoch 00020: ReduceLROnPlateau reducing learning rate to 6.25000029685907e-05.\n",
      "Epoch 21/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0052 - accuracy: 0.9986 - val_loss: 0.2705 - val_accuracy: 0.9544\n",
      "val_f1: 0.94913, best_val_f1: 0.94913\n",
      "\n",
      "Epoch 00021: val_f1 improved from 0.94821 to 0.94913, saving model to model/lstm_1.h5\n",
      "Epoch 22/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0046 - accuracy: 0.9987 - val_loss: 0.2670 - val_accuracy: 0.9546\n",
      "val_f1: 0.94949, best_val_f1: 0.94949\n",
      "\n",
      "Epoch 00022: val_f1 improved from 0.94913 to 0.94949, saving model to model/lstm_1.h5\n",
      "Epoch 23/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0045 - accuracy: 0.9988 - val_loss: 0.2675 - val_accuracy: 0.9543\n",
      "val_f1: 0.94939, best_val_f1: 0.94949\n",
      "\n",
      "Epoch 00023: val_f1 did not improve from 0.94949\n",
      "Epoch 24/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0044 - accuracy: 0.9988 - val_loss: 0.2760 - val_accuracy: 0.9543\n",
      "val_f1: 0.94946, best_val_f1: 0.94949\n",
      "\n",
      "Epoch 00024: val_f1 did not improve from 0.94949\n",
      "\n",
      "Epoch 00024: ReduceLROnPlateau reducing learning rate to 3.125000148429535e-05.\n",
      "Epoch 25/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0042 - accuracy: 0.9989 - val_loss: 0.2683 - val_accuracy: 0.9547\n",
      "val_f1: 0.95023, best_val_f1: 0.95023\n",
      "\n",
      "Epoch 00025: val_f1 improved from 0.94949 to 0.95023, saving model to model/lstm_1.h5\n",
      "Epoch 26/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0039 - accuracy: 0.9990 - val_loss: 0.2698 - val_accuracy: 0.9551\n",
      "val_f1: 0.95004, best_val_f1: 0.95023\n",
      "\n",
      "Epoch 00026: val_f1 did not improve from 0.95023\n",
      "Epoch 27/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0037 - accuracy: 0.9990 - val_loss: 0.2718 - val_accuracy: 0.9541\n",
      "val_f1: 0.94954, best_val_f1: 0.95023\n",
      "\n",
      "Epoch 00027: val_f1 did not improve from 0.95023\n",
      "\n",
      "Epoch 00027: ReduceLROnPlateau reducing learning rate to 1.5625000742147677e-05.\n",
      "Epoch 28/30\n",
      "160000/160000 [==============================] - 523s 3ms/step - loss: 0.0036 - accuracy: 0.9992 - val_loss: 0.2706 - val_accuracy: 0.9547\n",
      "val_f1: 0.95002, best_val_f1: 0.95023\n",
      "\n",
      "Epoch 00028: val_f1 did not improve from 0.95023\n",
      "Epoch 29/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0035 - accuracy: 0.9991 - val_loss: 0.2704 - val_accuracy: 0.9555\n",
      "val_f1: 0.95082, best_val_f1: 0.95082\n",
      "\n",
      "Epoch 00029: val_f1 improved from 0.95023 to 0.95082, saving model to model/lstm_1.h5\n",
      "Epoch 30/30\n",
      "160000/160000 [==============================] - 522s 3ms/step - loss: 0.0032 - accuracy: 0.9992 - val_loss: 0.2717 - val_accuracy: 0.9550\n",
      "val_f1: 0.95016, best_val_f1: 0.95082\n",
      "\n",
      "Epoch 00030: val_f1 did not improve from 0.95082\n",
      "Train on 160000 samples, validate on 40000 samples\n",
      "Epoch 1/30\n",
      "160000/160000 [==============================] - 523s 3ms/step - loss: 0.7874 - accuracy: 0.7597 - val_loss: 0.4472 - val_accuracy: 0.8865\n",
      "val_f1: 0.84835, best_val_f1: 0.84835\n",
      "\n",
      "Epoch 00001: val_f1 improved from -inf to 0.84835, saving model to model/lstm_2.h5\n",
      "Epoch 2/30\n",
      "160000/160000 [==============================] - 524s 3ms/step - loss: 0.1898 - accuracy: 0.9400 - val_loss: 0.2081 - val_accuracy: 0.9373\n",
      "val_f1: 0.92127, best_val_f1: 0.92127\n",
      "\n",
      "Epoch 00002: val_f1 improved from 0.84835 to 0.92127, saving model to model/lstm_2.h5\n",
      "Epoch 3/30\n",
      "160000/160000 [==============================] - 524s 3ms/step - loss: 0.1419 - accuracy: 0.9546 - val_loss: 0.5733 - val_accuracy: 0.8331\n",
      "val_f1: 0.82215, best_val_f1: 0.92127\n",
      "\n",
      "Epoch 00003: val_f1 did not improve from 0.92127\n",
      "Epoch 4/30\n",
      "160000/160000 [==============================] - 524s 3ms/step - loss: 0.1119 - accuracy: 0.9638 - val_loss: 0.3771 - val_accuracy: 0.8925\n",
      "val_f1: 0.88417, best_val_f1: 0.92127\n",
      "\n",
      "Epoch 00004: val_f1 did not improve from 0.92127\n",
      "\n",
      "Epoch 00004: ReduceLROnPlateau reducing learning rate to 0.0005000000237487257.\n",
      "Epoch 5/30\n",
      "160000/160000 [==============================] - 524s 3ms/step - loss: 0.0763 - accuracy: 0.9752 - val_loss: 0.2042 - val_accuracy: 0.9427\n",
      "val_f1: 0.93492, best_val_f1: 0.93492\n",
      "\n",
      "Epoch 00005: val_f1 improved from 0.92127 to 0.93492, saving model to model/lstm_2.h5\n",
      "Epoch 6/30\n",
      "160000/160000 [==============================] - 524s 3ms/step - loss: 0.0607 - accuracy: 0.9801 - val_loss: 0.2368 - val_accuracy: 0.9428\n",
      "val_f1: 0.93808, best_val_f1: 0.93808\n",
      "\n",
      "Epoch 00006: val_f1 improved from 0.93492 to 0.93808, saving model to model/lstm_2.h5\n",
      "Epoch 7/30\n",
      "160000/160000 [==============================] - 524s 3ms/step - loss: 0.0494 - accuracy: 0.9838 - val_loss: 0.2941 - val_accuracy: 0.9240\n",
      "val_f1: 0.91073, best_val_f1: 0.93808\n",
      "\n",
      "Epoch 00007: val_f1 did not improve from 0.93808\n",
      "Epoch 8/30\n",
      "160000/160000 [==============================] - 524s 3ms/step - loss: 0.0400 - accuracy: 0.9868 - val_loss: 0.2704 - val_accuracy: 0.9352\n",
      "val_f1: 0.93076, best_val_f1: 0.93808\n",
      "\n",
      "Epoch 00008: val_f1 did not improve from 0.93808\n",
      "\n",
      "Epoch 00008: ReduceLROnPlateau reducing learning rate to 0.0002500000118743628.\n",
      "Epoch 9/30\n",
      "160000/160000 [==============================] - 524s 3ms/step - loss: 0.0271 - accuracy: 0.9910 - val_loss: 0.2088 - val_accuracy: 0.9502\n",
      "val_f1: 0.94606, best_val_f1: 0.94606\n",
      "\n",
      "Epoch 00009: val_f1 improved from 0.93808 to 0.94606, saving model to model/lstm_2.h5\n",
      "Epoch 10/30\n",
      "160000/160000 [==============================] - 524s 3ms/step - loss: 0.0213 - accuracy: 0.9934 - val_loss: 0.2335 - val_accuracy: 0.9482\n",
      "val_f1: 0.94249, best_val_f1: 0.94606\n",
      "\n",
      "Epoch 00010: val_f1 did not improve from 0.94606\n",
      "Epoch 11/30\n",
      "160000/160000 [==============================] - 524s 3ms/step - loss: 0.0189 - accuracy: 0.9939 - val_loss: 0.2359 - val_accuracy: 0.9489\n",
      "val_f1: 0.94428, best_val_f1: 0.94606\n",
      "\n",
      "Epoch 00011: val_f1 did not improve from 0.94606\n",
      "\n",
      "Epoch 00011: ReduceLROnPlateau reducing learning rate to 0.0001250000059371814.\n",
      "Epoch 12/30\n",
      "160000/160000 [==============================] - 524s 3ms/step - loss: 0.0143 - accuracy: 0.9958 - val_loss: 0.2539 - val_accuracy: 0.9490\n",
      "val_f1: 0.94486, best_val_f1: 0.94606\n",
      "\n",
      "Epoch 00012: val_f1 did not improve from 0.94606\n",
      "Epoch 13/30\n",
      "160000/160000 [==============================] - 523s 3ms/step - loss: 0.0124 - accuracy: 0.9964 - val_loss: 0.2476 - val_accuracy: 0.9503\n",
      "val_f1: 0.94600, best_val_f1: 0.94606\n",
      "\n",
      "Epoch 00013: val_f1 did not improve from 0.94606\n",
      "\n",
      "Epoch 00013: ReduceLROnPlateau reducing learning rate to 6.25000029685907e-05.\n",
      "Epoch 14/30\n",
      "160000/160000 [==============================] - 524s 3ms/step - loss: 0.0106 - accuracy: 0.9971 - val_loss: 0.2413 - val_accuracy: 0.9530\n",
      "val_f1: 0.94929, best_val_f1: 0.94929\n",
      "\n",
      "Epoch 00014: val_f1 improved from 0.94606 to 0.94929, saving model to model/lstm_2.h5\n",
      "Epoch 15/30\n",
      " 30208/160000 [====>.........................] - ETA: 6:36 - loss: 0.0087 - accuracy: 0.9977"
     ]
    }
   ],
   "source": [
    "bs = 256\n",
    "monitor = 'val_f1'\n",
    "\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=seed)\n",
    "for fold_id, (train_index, val_index) in enumerate(kfold.split(all_index, df_data.iloc[all_index]['label'])):\n",
    "    train_x = seqs[train_index]\n",
    "    val_x = seqs[val_index]\n",
    "\n",
    "    label = df_data['label'].values\n",
    "    train_y = label[train_index]\n",
    "    val_y = label[val_index]\n",
    "    \n",
    "    model_path = 'model/lstm_{}.h5'.format(fold_id)\n",
    "    checkpoint = ModelCheckpoint(model_path, monitor=monitor, verbose=1, save_best_only=True, mode='max', save_weights_only=True)\n",
    "    earlystopping = EarlyStopping(monitor=monitor, patience=5, verbose=1, mode='max')\n",
    "    reduce_lr = ReduceLROnPlateau(monitor=monitor, factor=0.5, patience=2, mode='max', verbose=1)\n",
    "    \n",
    "    model = build_model(embedding, seq_len)\n",
    "    model.fit(train_x, train_y, batch_size=bs, epochs=30,\n",
    "              validation_data=(val_x, val_y),\n",
    "              callbacks=[Evaluator(validation_data=(val_x, val_y)), checkpoint, reduce_lr, earlystopping], verbose=1, shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 模型预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40000/40000 [==============================] - 35s 882us/step\n",
      "50000/50000 [==============================] - 43s 859us/step\n",
      "40000/40000 [==============================] - 35s 884us/step\n",
      "50000/50000 [==============================] - 43s 863us/step\n",
      "40000/40000 [==============================] - 35s 881us/step\n",
      "50000/50000 [==============================] - 43s 859us/step\n",
      "40000/40000 [==============================] - 35s 877us/step\n",
      "50000/50000 [==============================] - 43s 855us/step\n",
      "40000/40000 [==============================] - 35s 882us/step\n",
      "50000/50000 [==============================] - 43s 858us/step\n"
     ]
    }
   ],
   "source": [
    "oof_pred = np.zeros((len(all_index), 14))\n",
    "test_pred = np.zeros((len(test_index), 14))\n",
    "\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=seed)\n",
    "for fold_id, (train_index, val_index) in enumerate(kfold.split(all_index, df_data.iloc[all_index]['label'])):\n",
    "    model = build_model(embedding, seq_len)\n",
    "    model_path = 'model/lstm_{}.h5'.format(fold_id)\n",
    "    model.load_weights(model_path)\n",
    "    \n",
    "    val_x = seqs[val_index]\n",
    "    prob = model.predict(val_x, batch_size=bs, verbose=1)\n",
    "    oof_pred[val_index] = prob\n",
    "    \n",
    "    test_x = seqs[test_index]\n",
    "    prob = model.predict(test_x, batch_size=bs, verbose=1)\n",
    "    test_pred += prob / 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9485614776279795\n"
     ]
    }
   ],
   "source": [
    "df_oof = df_data.loc[all_index][['label']]\n",
    "df_oof['predict'] = np.argmax(oof_pred, axis=1)\n",
    "f1score = f1_score(df_oof['label'], df_oof['predict'], average='macro')\n",
    "print(f1score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9485614776279795"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('prob/sub_5fold_lstm_{}.npy'.format(f1score), test_pred)\n",
    "np.save('prob/oof_5fold_lstm_{}.npy'.format(f1score), oof_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub = pd.DataFrame()\n",
    "sub['label'] = np.argmax(test_pred, axis=1)\n",
    "sub.to_csv('sub/lstm_{}.csv'.format(f1score), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "lstm",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "conda_tensorflow_py3",
   "language": "python",
   "name": "conda_tensorflow_py3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}