Chapter 3.5 新闻分类-多分类问题.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "本节你会构建一个网络，将路透社新闻划分为46个互斥的主题。因为有多个类别，所以这是多分类（multiclass  classification）问题的一个例子。  \n",
    "\n",
    "因为每个数据点只能划分到一个类别，所以更具体地说，这是单标签、多分类（single-label, multiclass classification）问题的一个例子。  \n",
    "\n",
    "如果每个数据点可以划分到多个类别（主题），那它就是一个多标签、多分类（multilabel,multiclass classification）问题。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.5.1 The Reuters dataset 路透社数据\n",
    "它是一个简单的、广泛使用的文本分类数据集。它包括46个不同的主题：某些主题的样本更多，但训练集中每个主题都有至少10个样本"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import models\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras.datasets import reuters\n",
    "from tensorflow.keras import losses\n",
    "from tensorflow.keras import metrics\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/reuters.npz\n",
      "2113536/2110848 [==============================] - 0s 0us/step\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yihuawang/conda/lib/python3.8/site-packages/tensorflow/python/keras/datasets/reuters.py:148: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n",
      "  x_train, y_train = np.array(xs[:idx]), np.array(labels[:idx])\n",
      "/Users/yihuawang/conda/lib/python3.8/site-packages/tensorflow/python/keras/datasets/reuters.py:149: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n",
      "  x_test, y_test = np.array(xs[idx:]), np.array(labels[idx:])\n"
     ]
    }
   ],
   "source": [
    "(train_data, train_labels), (test_data, test_labels) = reuters.load_data(\n",
    "    num_words=10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8982"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2246"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1,\n",
       " 245,\n",
       " 273,\n",
       " 207,\n",
       " 156,\n",
       " 53,\n",
       " 74,\n",
       " 160,\n",
       " 26,\n",
       " 14,\n",
       " 46,\n",
       " 296,\n",
       " 26,\n",
       " 39,\n",
       " 74,\n",
       " 2979,\n",
       " 3554,\n",
       " 14,\n",
       " 46,\n",
       " 4689,\n",
       " 4329,\n",
       " 86,\n",
       " 61,\n",
       " 3499,\n",
       " 4795,\n",
       " 14,\n",
       " 61,\n",
       " 451,\n",
       " 4329,\n",
       " 17,\n",
       " 12]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/reuters_word_index.json\n",
      "557056/550378 [==============================] - 0s 0us/step\n"
     ]
    }
   ],
   "source": [
    "word_index = reuters.get_word_index()\n",
    "reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])\n",
    "decoded_newswire = \" \".join([reverse_word_index.get(i - 3, \"?\") for i in\n",
    "    train_data[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "def vectorize_sequences(sequences, dimension=10000): # 取10,000是因为我们之前限定了10,000个最常见的单词，因此单词索引不会超过10,000\n",
    "    # 以train_data[0]为例子\n",
    "    results = np.zeros((len(sequences), dimension)) #产生一个 218* 10,000的矩阵\n",
    "    for i, sequence in enumerate(sequences): #enumerate() 会产生[(0, 1),(1, 14),...,(217, 32)], \n",
    "                                             #前一个数字代表position, 后一个数字代表单词索引\n",
    "            results[i, sequence] = 1.\n",
    "    return results\n",
    "x_train = vectorize_sequences(train_data)\n",
    "x_test = vectorize_sequences(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 1., 1., ..., 0., 0., 0.],\n",
       "       [0., 1., 1., ..., 0., 0., 0.],\n",
       "       [0., 1., 1., ..., 0., 0., 0.],\n",
       "       ...,\n",
       "       [0., 1., 1., ..., 0., 0., 0.],\n",
       "       [0., 1., 1., ..., 0., 0., 0.],\n",
       "       [0., 1., 1., ..., 0., 0., 0.]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encoding the labels\n",
    "将标签向量化有两种方法：你可以将标签列表转换为整数张量，或者使用one-hot编码。one-hot编码是分类数据广泛使用的一种格式，也叫分类编码（categorical  encoding）。  \n",
    "\n",
    "在这个例子中，标签的 one-hot编码就是将每个标签表示为全零向量，只有标签索引对应的元素为 1。其代码实现如下"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_one_hot(labels, dimension=46): #因为数据集包含46个不同的主题\n",
    "    results = np.zeros((len(labels), dimension))\n",
    "    for i, label in enumerate(labels):\n",
    "        results[i, label] = 1.\n",
    "    return results\n",
    "y_train = to_one_hot(train_labels)\n",
    "y_test = to_one_hot(test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.utils import to_categorical\n",
    "y_train = to_categorical(train_labels)\n",
    "y_test = to_categorical(test_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.5.3 Building your model 构建网络\n",
    "\n",
    "Dense层的堆叠，每层只能访问上一层输出的信息。如果某一层丢失了与分类问题相关的一些信息，那么这些信息无法被后面的层找回，也就是说，每一层都可能成为信息瓶颈。上一个例子使用了16维的中间层，但对这个例子来说16维空间可能太小了，无法学会区分46个不同的类别。这种维度较小的层可能成为信息瓶颈，永久地丢失相关信息。  \n",
    "dense层数一般都是2^n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = models.Sequential()\n",
    "model.add(layers.Dense(64, activation='relu', input_shape=(10000,)))\n",
    "model.add(layers.Dense(64, activation='relu'))\n",
    "model.add(layers.Dense(46, activation='softmax')) \n",
    "# 网络的最后一层是大小为46的Dense层。这意味着，对于每个输入样本，网络都会输出一个46维向量。这个向量的每个元素（即每个维度）代表不同的输出类别\n",
    "#最后一层使用了 softmax激活。你在MNIST例子中见过这种用法。网络将输出在46个不同输出类别上的概率分布——对于每一个输入样本，\n",
    "#网络都会输出一个46维向量，其中output[i]是样本属于第i个类别的概率。46个概率的总和为 1。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 多分类--categorical_crossentropy\n",
    "model.compile(optimizer='rmsprop',\n",
    "        loss='categorical_crossentropy',\n",
    "        metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.5.4 Validating your approach 验证\n",
    "同样从训练集中留出1000个作为验证集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_val = x_train[:1000]\n",
    "partial_x_train = x_train[1000:]\n",
    "y_val = y_train[:1000]\n",
    "partial_y_train = y_train[1000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "16/16 [==============================] - 1s 58ms/step - loss: 3.2019 - accuracy: 0.4083 - val_loss: 1.8418 - val_accuracy: 0.6060\n",
      "Epoch 2/20\n",
      "16/16 [==============================] - 0s 19ms/step - loss: 1.5128 - accuracy: 0.6794 - val_loss: 1.3494 - val_accuracy: 0.7080\n",
      "Epoch 3/20\n",
      "16/16 [==============================] - 0s 19ms/step - loss: 1.1048 - accuracy: 0.7580 - val_loss: 1.1489 - val_accuracy: 0.7520\n",
      "Epoch 4/20\n",
      "16/16 [==============================] - 0s 18ms/step - loss: 0.8511 - accuracy: 0.8181 - val_loss: 1.0594 - val_accuracy: 0.7750\n",
      "Epoch 5/20\n",
      "16/16 [==============================] - 0s 18ms/step - loss: 0.6535 - accuracy: 0.8610 - val_loss: 0.9721 - val_accuracy: 0.7980\n",
      "Epoch 6/20\n",
      "16/16 [==============================] - 0s 19ms/step - loss: 0.5268 - accuracy: 0.8890 - val_loss: 0.9180 - val_accuracy: 0.8050\n",
      "Epoch 7/20\n",
      "16/16 [==============================] - 0s 20ms/step - loss: 0.4357 - accuracy: 0.9127 - val_loss: 0.8845 - val_accuracy: 0.8140\n",
      "Epoch 8/20\n",
      "16/16 [==============================] - 0s 19ms/step - loss: 0.3506 - accuracy: 0.9303 - val_loss: 0.8867 - val_accuracy: 0.8090\n",
      "Epoch 9/20\n",
      "16/16 [==============================] - 0s 21ms/step - loss: 0.2779 - accuracy: 0.9394 - val_loss: 0.8733 - val_accuracy: 0.8210\n",
      "Epoch 10/20\n",
      "16/16 [==============================] - 0s 19ms/step - loss: 0.2404 - accuracy: 0.9430 - val_loss: 0.8902 - val_accuracy: 0.8150\n",
      "Epoch 11/20\n",
      "16/16 [==============================] - 0s 20ms/step - loss: 0.1892 - accuracy: 0.9538 - val_loss: 0.9419 - val_accuracy: 0.8050\n",
      "Epoch 12/20\n",
      "16/16 [==============================] - 0s 22ms/step - loss: 0.1785 - accuracy: 0.9563 - val_loss: 0.9375 - val_accuracy: 0.8120\n",
      "Epoch 13/20\n",
      "16/16 [==============================] - 0s 19ms/step - loss: 0.1632 - accuracy: 0.9559 - val_loss: 0.9424 - val_accuracy: 0.8070\n",
      "Epoch 14/20\n",
      "16/16 [==============================] - 0s 21ms/step - loss: 0.1377 - accuracy: 0.9615 - val_loss: 0.9610 - val_accuracy: 0.8020\n",
      "Epoch 15/20\n",
      "16/16 [==============================] - 0s 20ms/step - loss: 0.1299 - accuracy: 0.9584 - val_loss: 0.9556 - val_accuracy: 0.8190\n",
      "Epoch 16/20\n",
      "16/16 [==============================] - 0s 19ms/step - loss: 0.1135 - accuracy: 0.9608 - val_loss: 0.9794 - val_accuracy: 0.8220\n",
      "Epoch 17/20\n",
      "16/16 [==============================] - 0s 21ms/step - loss: 0.1161 - accuracy: 0.9582 - val_loss: 1.0443 - val_accuracy: 0.7910\n",
      "Epoch 18/20\n",
      "16/16 [==============================] - 0s 19ms/step - loss: 0.1065 - accuracy: 0.9636 - val_loss: 1.0595 - val_accuracy: 0.8060\n",
      "Epoch 19/20\n",
      "16/16 [==============================] - 0s 22ms/step - loss: 0.1029 - accuracy: 0.9625 - val_loss: 1.0202 - val_accuracy: 0.8090\n",
      "Epoch 20/20\n",
      "16/16 [==============================] - 0s 18ms/step - loss: 0.1040 - accuracy: 0.9614 - val_loss: 1.0494 - val_accuracy: 0.8030\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(partial_x_train,\n",
    "                    partial_y_train,\n",
    "                    epochs=20,\n",
    "                    batch_size=512,\n",
    "                    validation_data=(x_val, y_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZwU1bn/8c/DJuAgIqASkBlIVASBAUdFUMQlvysuiIRcJUREEhHXuMRo4o1yzeV3f0lMrhfFBfckRDQxEhc0CQqiMS4D4oJgXAKKIiLKJiAMPL8/TjU0TfcsTFd3z/T3/XrVq2vvp2t66ulzTtUpc3dERKR4Ncl3ACIikl9KBCIiRU6JQESkyCkRiIgUOSUCEZEip0QgIlLklAgkq8zsSTM7J9vr5pOZLTGzE2PYr5vZN6Lx283sp7VZdzfeZ7SZ/XV346xmv0PMbFm29yu51yzfAUj+mdn6pMnWwFfA1mj6fHefVtt9ufvQONZt7Nx9Qjb2Y2ZlwL+A5u5eFe17GlDrv6EUHyUCwd1LEuNmtgT4vrvPSl3PzJolTi4i0nioakgyShT9zexqM/sEuNfM2pnZ42a20sy+iMa7JG0zx8y+H42PNbPnzezGaN1/mdnQ3Vy3m5nNNbN1ZjbLzKaY2e8yxF2bGH9mZn+P9vdXM+uQtPxsM1tqZqvM7Npqjs8AM/vEzJomzTvDzF6Pxo8ws3+Y2WozW25mt5hZiwz7us/M/itp+qpom4/NbFzKuqeY2atmttbMPjSziUmL50avq81svZkdlTi2SdsPNLNXzGxN9DqwtsemOmZ2SLT9ajNbaGbDkpadbGZvRfv8yMx+GM3vEP19VpvZ52b2nJnpvJRjOuBSk/2BfYBSYDzhO3NvNN0V2AjcUs32RwJvAx2AXwB3m5ntxrq/B14G2gMTgbOrec/axPgd4FxgX6AFkDgx9QRui/b/tej9upCGu78IfAkcn7Lf30fjW4HLo89zFHACcGE1cRPFcFIUzzeBA4HU9okvgTHA3sApwAVmNjxaNjh63dvdS9z9Hyn73gd4ApgcfbZfA0+YWfuUz7DLsakh5ubAY8Bfo+0uAaaZ2cHRKncTqhnbAIcCz0TzrwSWAR2B/YCfAOr3JseUCKQm24Dr3f0rd9/o7qvc/WF33+Du64BJwLHVbL/U3e90963A/UAnwj98rdc1s67A4cB17r7Z3Z8HHs30hrWM8V53/6e7bwQeAsqj+SOBx919rrt/Bfw0OgaZPACMAjCzNsDJ0TzcfZ67v+juVe6+BLgjTRzp/HsU35vu/iUh8SV/vjnu/oa7b3P316P3q81+ISSOd9z9t1FcDwCLgdOS1sl0bKozACgB/l/0N3oGeJzo2ABbgJ5mtpe7f+Hu85PmdwJK3X2Luz/n6gAt55QIpCYr3X1TYsLMWpvZHVHVyVpCVcTeydUjKT5JjLj7hmi0pI7rfg34PGkewIeZAq5ljJ8kjW9IiulryfuOTsSrMr0X4df/CDPbAxgBzHf3pVEcB0XVHp9EcfxfQumgJjvFACxN+XxHmtnsqOprDTChlvtN7HtpyrylQOek6UzHpsaY3T05aSbv91uEJLnUzJ41s6Oi+b8E3gX+ambvm9k1tfsYkk1KBFKT1F9nVwIHA0e6+17sqIrIVN2TDcuBfcysddK8A6pZvz4xLk/ed/Se7TOt7O5vEU54Q9m5WghCFdNi4MAojp/sTgyE6q1kvyeUiA5w97bA7Un7renX9MeEKrNkXYGPahFXTfs9IKV+f/t+3f0Vdz+dUG00g1DSwN3XufuV7t6dUCq5wsxOqGcsUkdKBFJXbQh17quj+ubr437D6Bd2JTDRzFpEvyZPq2aT+sT4R+BUMzs6ati9gZr/T34PXEpIOH9IiWMtsN7MegAX1DKGh4CxZtYzSkSp8bchlJA2mdkRhASUsJJQldU9w75nAgeZ2XfMrJmZnQn0JFTj1MdLhLaLH5lZczMbQvgbTY/+ZqPNrK27byEck60AZnaqmX0jagtKzN+a/i0kLkoEUlc3Aa2Az4AXgady9L6jCQ2uq4D/Ah4k3O+Qzm7H6O4LgYsIJ/flwBeExszqPAAMAZ5x98+S5v+QcJJeB9wZxVybGJ6MPsMzhGqTZ1JWuRC4wczWAdcR/bqOtt1AaBP5e3QlzoCUfa8CTiWUmlYBPwJOTYm7ztx9MzCMUDL6DLgVGOPui6NVzgaWRFVkE4DvRvMPBGYB64F/ALe6+5z6xCJ1Z2qXkYbIzB4EFrt77CUSkcZOJQJpEMzscDP7upk1iS6vPJ1Q1ywi9aQ7i6Wh2B/4E6Hhdhlwgbu/mt+QRBoHVQ2JiBQ5VQ2JiBS5Blc11KFDBy8rK8t3GCIiDcq8efM+c/eO6ZY1uERQVlZGZWVlvsMQEWlQzCz1jvLtVDUkIlLklAhERIqcEoGISJFrcG0EIpJ7W7ZsYdmyZWzatKnmlSWvWrZsSZcuXWjevHmtt1EiEJEaLVu2jDZt2lBWVkbm5wpJvrk7q1atYtmyZXTr1q3W2xVF1dC0aVBWBk2ahNdpeoy3SJ1s2rSJ9u3bKwkUODOjffv2dS65NfoSwbRpMH48bIgeabJ0aZgGGD06f3GJNDRKAg3D7vydGn2J4NprdySBhA0bwnwRESmCRPDBB3WbLyKFZ9WqVZSXl1NeXs7+++9P586dt09v3ry52m0rKyu59NJLa3yPgQMHZiXWOXPmcOqpp2ZlX7nS6BNB19SH/NUwX0TqL9vtcu3bt2fBggUsWLCACRMmcPnll2+fbtGiBVVVVRm3raioYPLkyTW+xwsvvFC/IBuwRp8IJk2C1q13nte6dZgvItmXaJdbuhTcd7TLZfsijbFjx3LFFVdw3HHHcfXVV/Pyyy8zcOBA+vXrx8CBA3n77beBnX+hT5w4kXHjxjFkyBC6d+++U4IoKSnZvv6QIUMYOXIkPXr0YPTo0SR6aZ45cyY9evTg6KOP5tJLL63xl//nn3/O8OHD6dOnDwMGDOD1118H4Nlnn91eounXrx/r1q1j+fLlDB48mPLycg499FCee+657B6wajT6xuJEg/C114bqoK5dQxJQQ7FIPKprl8v2/90///lPZs2aRdOmTVm7di1z586lWbNmzJo1i5/85Cc8/PDDu2yzePFiZs+ezbp16zj44IO54IILdrnm/tVXX2XhwoV87WtfY9CgQfz973+noqKC888/n7lz59KtWzdGjRpVY3zXX389/fr1Y8aMGTzzzDOMGTOGBQsWcOONNzJlyhQGDRrE+vXradmyJVOnTuXf/u3fuPbaa9m6dSsbUg9ijBp9IoDw5dOJXyQ3ctku9+1vf5umTZsCsGbNGs455xzeeecdzIwtW7ak3eaUU05hjz32YI899mDfffdlxYoVdOnSZad1jjjiiO3zysvLWbJkCSUlJXTv3n379fmjRo1i6tSp1cb3/PPPb09Gxx9/PKtWrWLNmjUMGjSIK664gtGjRzNixAi6dOnC4Ycfzrhx49iyZQvDhw+nvLy8XsemLhp91ZCI5FYu2+X23HPP7eM//elPOe6443jzzTd57LHHMl5Lv8cee2wfb9q0adr2hXTr7M5DvNJtY2Zcc8013HXXXWzcuJEBAwawePFiBg8ezNy5c+ncuTNnn302v/nNb+r8frtLiUBEsipf7XJr1qyhc+fOANx3331Z33+PHj14//33WbJkCQAPPvhgjdsMHjyYaVHjyJw5c+jQoQN77bUX7733Hr179+bqq6+moqKCxYsXs3TpUvbdd1/OO+88vve97zF//vysf4ZMlAhEJKtGj4apU6G0FMzC69Sp8VfP/uhHP+LHP/4xgwYNYuvWrVnff6tWrbj11ls56aSTOProo9lvv/1o27ZttdtMnDiRyspK+vTpwzXXXMP9998PwE033cShhx5K3759adWqFUOHDmXOnDnbG48ffvhhfvCDH2T9M2TS4J5ZXFFR4XowjUhuLVq0iEMOOSTfYeTd+vXrKSkpwd256KKLOPDAA7n88svzHdYu0v29zGyeu1ekW18lAhGRWrrzzjspLy+nV69erFmzhvPPPz/fIWVFUVw1JCKSDZdffnlBlgDqK7YSgZkdYGazzWyRmS00s10qvMxsiJmtMbMF0XBdXPGIiEh6cZYIqoAr3X2+mbUB5pnZ39z9rZT1nnP3htUxh4hIIxJbicDdl7v7/Gh8HbAI6BzX+4mIyO7JSWOxmZUB/YCX0iw+ysxeM7MnzaxXLuIREZEdYk8EZlYCPAxc5u5rUxbPB0rdvS9wMzAjwz7Gm1mlmVWuXLky3oBFpOAMGTKEv/zlLzvNu+mmm7jwwgur3SZxqfnJJ5/M6tWrd1ln4sSJ3HjjjdW+94wZM3jrrR012tdddx2zZs2qS/hpFVJ31bEmAjNrTkgC09z9T6nL3X2tu6+PxmcCzc2sQ5r1prp7hbtXdOzYMc6QRaQAjRo1iunTp+80b/r06bXq+A1Cr6F77733br13aiK44YYbOPHEE3drX4UqzquGDLgbWOTuv86wzv7RepjZEVE8q+KKSUQappEjR/L444/z1VdfAbBkyRI+/vhjjj76aC644AIqKiro1asX119/fdrty8rK+OyzzwCYNGkSBx98MCeeeOL2rqoh3CNw+OGH07dvX771rW+xYcMGXnjhBR599FGuuuoqysvLee+99xg7dix//OMfAXj66afp168fvXv3Zty4cdvjKysr4/rrr6d///707t2bxYsXV/v58t1ddZxXDQ0CzgbeMLMF0byfAF0B3P12YCRwgZlVARuBs7yh3eosUmQuuwwWLKh5vbooL4ebbsq8vH379hxxxBE89dRTnH766UyfPp0zzzwTM2PSpEnss88+bN26lRNOOIHXX3+dPn36pN3PvHnzmD59Oq+++ipVVVX079+fww47DIARI0Zw3nnnAfAf//Ef3H333VxyySUMGzaMU089lZEjR+60r02bNjF27FiefvppDjroIMaMGcNtt93GZZddBkCHDh2YP38+t956KzfeeCN33XVXxs+X7+6q47xq6Hl3N3fv4+7l0TDT3W+PkgDufou793L3vu4+wN2L9xFBIlKt5Oqh5Gqhhx56iP79+9OvXz8WLly4UzVOqueee44zzjiD1q1bs9deezFs2LDty958802OOeYYevfuzbRp01i4cGG18bz99tt069aNgw46CIBzzjmHuXPnbl8+YsQIAA477LDtHdVl8vzzz3P22WcD6burnjx5MqtXr6ZZs2Ycfvjh3HvvvUycOJE33niDNm3aVLvv2tCdxSJSJ9X9co/T8OHDueKKK5g/fz4bN26kf//+/Otf/+LGG2/klVdeoV27dowdOzZj99MJUW30LsaOHcuMGTPo27cv9913H3PmzKl2PzVVXiS6ss7U1XVN+0p0V33KKacwc+ZMBgwYwKxZs7Z3V/3EE09w9tlnc9VVVzFmzJhq918T9TUkIg1CSUkJQ4YMYdy4cdtLA2vXrmXPPfekbdu2rFixgieffLLafQwePJhHHnmEjRs3sm7dOh577LHty9atW0enTp3YsmXL9q6jAdq0acO6det22VePHj1YsmQJ7777LgC//e1vOfbYY3frs+W7u2qVCESkwRg1ahQjRozYXkXUt29f+vXrR69evejevTuDBg2qdvv+/ftz5plnUl5eTmlpKcccc8z2ZT/72c848sgjKS0tpXfv3ttP/meddRbnnXcekydP3t5IDNCyZUvuvfdevv3tb1NVVcXhhx/OhAkTdutzTZw4kXPPPZc+ffrQunXrnbqrnj17Nk2bNqVnz54MHTqU6dOn88tf/pLmzZtTUlKSlQfYqBtqEamRuqFuWNQNtYiI1IkSgYhIkVMiEJFaaWjVyMVqd/5OSgQiUqOWLVuyatUqJYMC5+6sWrWKli1b1mk7XTUkIjXq0qULy5YtQ50+Fr6WLVvSpUuXOm2jRCAiNWrevDndunXLdxgSE1UNiYgUOSUCEZEip0QgIlLklAhERIqcEoGISJFTIhARKXJKBCIiRU6JQESkyCkRiIgUOSUCEZEip0QgIlLklAhERIqcEoGISJFTIhARKXJKBCIiRU6JQESkyCkRiIgUOSUCEZEip0QgIlLklAhERIpcbInAzA4ws9lmtsjMFprZD9KsY2Y22czeNbPXzax/XPGIiEh6zWLcdxVwpbvPN7M2wDwz+5u7v5W0zlDgwGg4ErgtehURkRyJrUTg7svdfX40vg5YBHROWe104DcevAjsbWad4opJRER2lZM2AjMrA/oBL6Us6gx8mDS9jF2TBWY23swqzaxy5cqVcYUpIlKUYk8EZlYCPAxc5u5rUxen2cR3meE+1d0r3L2iY8eOcYQpIlK0Yk0EZtackASmufuf0qyyDDggaboL8HGcMYmIyM7ivGrIgLuBRe7+6wyrPQqMia4eGgCscfflccUkIiK7ivOqoUHA2cAbZrYgmvcToCuAu98OzAROBt4FNgDnxhiPiIikEVsicPfnSd8GkLyOAxfFFYOIiNRMdxaLiBQ5JQIRkSKnRCAiUuSUCEREipwSgYhIkSuaRPDii3DaafDll/mORESksBRNIqiqgscfh2nT8h2JiEhhKZpEMGgQlJfDzTeD79KbkYhI8SqaRGAGF18Mb74Jc+fmOxoRkcJRNIkAYNQoaNcObrkl35GIiBSOokoErVvD978PjzwCH35Y8/oiIsWgqBIBwAUXwLZtcMcd+Y5ERKQwFF0i6NYtXEY6dSp89VW+oxERyb+iSwQQGo1XroSHHsp3JCIi+VeUieCEE+Dgg9VoLCICRZoImjQJpYKXXw6DiEgxK8pEADBmDJSUqFQgIlK0iWCvvWDsWHjwQfj003xHIyKSP0WbCAAuugg2b4Y778x3JCIi+VPUiaBHDzjxRLjtttApnYhIMSrqRABwySXw0Ufw5z/nOxIRkfwo+kRwyilQVhZ6JRURKUZFnwiaNoULL4Rnn4U33sh3NCIiuVf0iQBg3Dho2VKXkopIcVIiANq3h+98B373O/jii3xHIyKSW0oEkYsvhg0b4L778h2JiEhuKRFE+vULj7OcMiV0Uy0iUiyUCJJccgm89x489VS+IxERyR0lgiRnnAGdOulSUhEpLrElAjO7x8w+NbM3MywfYmZrzGxBNFwXVyy11aIFnH9+KBG8806+oxERyY04SwT3ASfVsM5z7l4eDTfEGEutnX8+NG8Ot96a70hERHIjtkTg7nOBz+Paf1z23x9GjoR77oH16/MdjYhI/GqVCMxsTzNrEo0fZGbDzKx5Ft7/KDN7zcyeNLNe1bz/eDOrNLPKlStXZuFtq3fJJbB2bbivQESksattiWAu0NLMOgNPA+cSqn7qYz5Q6u59gZuBGZlWdPep7l7h7hUdO3as59vWbMAA6N8/3GnsHvvbiYjkVW0Tgbn7BmAEcLO7nwH0rM8bu/tad18fjc8EmptZh/rsM1vMQqlg4UKYMyff0YiIxKvWicDMjgJGA09E85rV543NbH8zs2j8iCiWVfXZZzadeWboeuKWW2DatNBDaZMm4XXatHxHJyKSPbU9mV8G/Bh4xN0Xmll3YHZ1G5jZA8AQoIOZLQOuB5oDuPvtwEjgAjOrAjYCZ7kXTkVMq1bw/e/DL34BM2fCpk1h/tKlMH58GB89On/xiYhki9X13Bs1Gpe4+9p4QqpeRUWFV1ZW5uS9li4NJYB0SkthyZKchCEiUm9mNs/dK9Itq+1VQ783s73MbE/gLeBtM7sqm0EWotLSzMs++CB3cYiIxKm2bQQ9oxLAcGAm0BU4O7aoCsi++6af37VrbuMQEYlLbRNB8+i+geHAn919C1Aw9flx+tWvwlVEyVq3hkmT8hOPiEi21TYR3AEsAfYE5ppZKZCXNoJc++53YcyYHdOlpTB1qhqKRaTxqHNj8fYNzZq5e1WW46lRLhuLE9avh86d4bTTdLexiDRM2Wgsbmtmv05082BmvyKUDopCSQmMHQsPPRRuMhMRaUxqWzV0D7AO+PdoWAvcG1dQhejKK6FDBxgyBBYsyHc0IiLZU9tE8HV3v97d34+G/wS6xxlYoenaFebODQ3Fxx0HL7+c74hERLKjtolgo5kdnZgws0GEu4GLyje+EZLBPvvAiSfCc8/lOyIRkfqrbSKYAEwxsyVmtgS4BTg/tqgKWGlpSAadO8NJJ8GsWfmOSESkfmqVCNz9tai76D5AH3fvBxwfa2QFrHNnePbZUEI49VR44omatxERKVR1ekJZ1HV04v6BK2KIp8HYd1+YPRt694bhw+GPf8x3RCIiu6c+j6q0mldp3PbZJ1QNHXlk6LZa9xiISENUn0RQFF1M1KRtW3jqqXBZ6ZgxcNdd+Y5IRKRuqn0egZmtI/0J34BWsUTUAJWUwOOPw7e+BeedBxs3hieciYg0BNUmAndvk6tAGrpWreCRR2DUKLj00pAMfvSjfEclIlKz+lQNSYo99oAHHwzJ4OqrYeJEKJxnromIpFev5w7Lrpo3h9/+Flq2hP/8T9iwAX7+8127shYRKRRKBDFo2jQ0GrdqBb/8ZUgGkydDE5W/RKQAKRHEpEkTuOWWkAx+9SvYtAnuuCMkCRGRQqJEECOzUCLYc0+44YZQMpgyBdq1y3dkIiI7qLIiZmahreC//xseeAC6dIEJE/RcAxEpHEoEOXLNNeE5BqNGwf33w6GHhh5MH30Utm7Nd3QiUsyUCHKob9/QiPzhh6GE8PbbcPrpcOCB8Otfw+rV+Y5QRIqREkEedOgQSgj/+hf84Q+huujKK0OvphdeCG+9le8IRaSYKBHkwLRpUFYWriQqKwvTAM2awciR4fkG8+eHjuvuuQd69YJvfhMee0zVRiISPyWCmE2bBuPHw9Kl4S7jpUvDdCIZJPTrF5LAhx/CpEmwaBEMGwYHHQT/8z+qNhKR+Jg3sD4QKioqvLKyMt9h1FpZWTj5pyothSVLMm+3ZUvou2jyZPj738MlqOecAxddBD17xhWtSHGrqoIvvoBVq8Lw+ec7xr/4IlTflpdDnz7hf7IhMbN57l6RdpkSQbyaNEnf35AZbNtWu33Mmwc33xwuP928GY47LiSE008P1UsiUr116+Dpp0OJO91JPjG+Zk3mfZjt+F82C6X18vIw9OsXXvfbLzvxbtwI778P77678zByJJy/mw8Jri4R6DQSs65d05cIunat/T4OOwzuuy/cnHb33XDbbeEL0blzuCfhvPOy9wUUaSw++SRcnj1jRkgCmzfvWLb33uHBUu3bh4s3Dj54x3RiSJ0uKYFly+DVV8Ol4AsWwEsvhY4mE/bff+fEUF4eHmmbrnuZtWvhvffCCT7xmhg++mjndffZJ/N+siG2EoGZ3QOcCnzq7oemWW7A/wInAxuAse4+v6b9NrQSQaKNYMOGHfNat4apU2H06N3b59at4fkHU6bA3/4WOrobOTKUEgYOVAd3UrwWL4Y//zmc/F96KfyC7949PE522LBQrdquXXZL0l98Aa+9tiM5LFgQbhitqgrL99wzXDrep084DyRO9p9+uvN+9t8/nOy//vXwmhi+/vXs9EaQl6ohMxsMrAd+kyERnAxcQkgERwL/6+5H1rTfhpYIICSDa6+FDz4IJYFJk3Y/CaT65z/h1ltDiWHNmvAL5KKL4DvfCQlHpDHbti2c8GfMCAng7bfD/IqKUHU6fHi4Ci/XP46++ipcBp6cHF5/Hdq02fUkn3gtKYk3pry1EZhZGfB4hkRwBzDH3R+Ipt8Ghrj78ur22RATQS6sXx8SzpQp8MYboeh77rnhvoRvfCPf0Ylkz6ZN8Mwz4eT/6KOwYkX4hX/cceHkP2wYHHBAvqMsPIXaRtAZ+DBpelk0b5dEYGbjgfEAXetSuV5ESkpCI9L48fD88yEh3HxzuPT0pJNCKWHoUPV+Kg3Lpk2hXv7DD0Pj6VNPwZNPwpdfhu/8ySeHX/1Dh4YfP7J78pkI0hXW0hZP3H0qMBVCiSDOoBo6MzjmmDAsXw533gm33w6nnRYuWT3lFDjhBBgyJDRAieRLVVX4jn744Y7hgw92nk6tR+/UCb773XDyP+648FRAqb98JoJlQHIBrgvwcZ5iaZQ6dYLrroMf/zgUo++7L3R4d+utIWGUl4ekcPzxIXHEXUcphcc9tC2tWLHr8OmnO0+vXBm+Ny1ahBNw6mu6ecnLmjUL+0yc5D/+eNc759u0CdU6XbtC//5hPDF07Rrq0vWAp+zLZxvBKcDF7GgsnuzuR9S0T7UR1M+WLfDyy6GO9Zln4IUXwmV1zZrBkUeGpHDCCTBggH5tZcvGjfDii6FRs6oqXOXVokX615qWVVWFv9fuDJs2hWvlU0/2yZdVJjRpAh07wr77hkuT99svTJuFhtDNm3d+TTcvddnmzeFSza5ddz3BJ8bbts3936dY5OuqoQeAIUAHYAVwPdAcwN1vjy4fvQU4iXD56LnuXuMZXokguzZsCMng6adDYqisDFditGoFRx8dEsPxx4d7GdS+UDsbNoQT/5w5YXjppfQn21wyC4m9Q4edT+6JIXVe+/b6ezc2urNYam316tAJ3jPPhOTw5pthftu2MHhwuBa6Z0845JBwE44uUQ0Nl//4RzjpP/tsOPFv2RJ+VffvH9pjjj02JNaSkpAUtmyp22tiPFFCyDQkqmNSB53URYlAdtuKFTB7dkgMzz0H77yzo17XLDRAH3LIjiGRJBrz4zjXrw+lqGefDSf/V14JJ+mmTUPJ6dhjw8l/0CBVdUjhUCKQrNm8OSSDRYt2HhYvDnXQCfvtt3OCOOSQ0DdL8+YhkaQO27bVPM8d9torXCbYrl04ycbR19LGjaFhdOVK+OyzHeMffxw6AHzllVBX37RpuHEp8Yt/0KAQn0ghKtT7CKQBatEi3KnZq9fO87duDX0qJRLDW2+F19//vvqOvOqrTZsdiaFdux3jqa/t2oV116zZcWJPPdEnpr/8Mv17NW8eTvw//GE4+Q8cGPYp0tApETQAcXZRkS1Nm4Y+Xbp3D/cqJLiHzr8WLQoda23bFtZt0iS8JobaTJuFjrpWrw79u6R7ff/9HdPr11cfc6tW4UqYxNCjx47xDh12XtahQ0gqunRRGsK/IDsAAAt1SURBVCMlggKX2mld4sE2UHjJIB2zcD9Dp07h6qNcqqoKSSGRGNauDdVJiZO7GrpFArURFLjdfbCNiEiy6toIVNAtcB98ULf5IiJ1pURQ4DL1sae+90QkW5QICtykSbvWZbduHeaLiGSDEkGBGz06PM2stHTHDVz1ebqZiEgqXTXUAIwerRO/iMRHJQIRkSKnRCAiUuSUCEREipwSgYhIkVMiEBEpckoEIiJFTomgCEybFvosatIkvE6blu+IRKSQ6D6CRq6h914qIvFTiaCRu/baHUkgYcOGMF9EBJQIGj31XioiNVEiaOTUe6mI1ESJoJFT76UiUhMlgkZOvZeKSE101VARUO+lIlIdlQhERIqcEoGISJFTIpBa0d3JIo2X2gikRro7WaRxi7VEYGYnmdnbZvaumV2TZvkQM1tjZgui4bo445Hdo7uTRRq32EoEZtYUmAJ8E1gGvGJmj7r7WymrPufup8YVh9Sf7k4WadziLBEcAbzr7u+7+2ZgOnB6jO8nMdHdySKNW5yJoDPwYdL0smheqqPM7DUze9LMeqXbkZmNN7NKM6tcuXJlHLFKNXR3skjjFmcisDTzPGV6PlDq7n2Bm4EZ6Xbk7lPdvcLdKzp27JjlMKUmujtZpHGL86qhZcABSdNdgI+TV3D3tUnjM83sVjPr4O6fxRiX7AbdnSzSeMVZIngFONDMuplZC+As4NHkFcxsfzOzaPyIKJ5VMcYkeaL7EEQKV2wlAnevMrOLgb8ATYF73H2hmU2Ilt8OjAQuMLMqYCNwlrunVh9JA6f7EEQKmzW0825FRYVXVlbmOwypg7KycPJPVVoKS5bkOhqR4mRm89y9It0ydTEhsdN9CCKFTYlAYqf7EEQKmxKBxE73IYgUNiUCiV027kPQVUci8VHvo5IT9bkPQVcdicRLJQIpeOr9VCReSgRS8HTVkUi8lAik4GXjqiO1MYhkpkQgBa++Vx0l2hiWLgX3HW0MSgYigRKBFLz6XnWkNgaR6qmLCWn0mjQJJYFUZrBtW+7jEckHdTEhRU1tDCLVUyKQRk9tDCLVUyKQRk9tDCLVUyKQojB6dOjyetu28FqXO5KzcR+DqpakkCkRiNSgvm0MqlqSQqdEIFKD+rYxZKNqSSUKiZMSgUgN6tvGUN+qpWyUKJRIpDpKBCK1UJ82hvpWLdW3RKFEIjVRIhCJWX2rlupbolAikZooEYjErL5VS/UtUSiRKBHVyN0b1HDYYYe5SDH53e/cW7d2D6fRMLRuHebXRmnpztsmhtLS2m1vln57s9y8f30/f323T+yjtDR85tLSum1bCNu7uwOVnuG8mvcTe10HJQIpRvU5ERR7ImnoiSgbicxdiUCk6BVzImnoiai+2ydUlwjURiBSBOpz1VN92zjq21he3zaSfLex5Hv72lAiEJEaNeRE0tATUTZ6z61RpqJCoQ6qGhIpPvlsbM13Hb/aCJQIRKQA5Puqn7ivGtITykREikDenlBmZieZ2dtm9q6ZXZNmuZnZ5Gj562bWP854RERkV7ElAjNrCkwBhgI9gVFm1jNltaHAgdEwHrgtrnhERCS9OEsERwDvuvv77r4ZmA6cnrLO6cBvoiqsF4G9zaxTjDGJiEiKOBNBZ+DDpOll0by6riMiIjGKMxFYmnmpLdO1WQczG29mlWZWuXLlyqwEJyIiQbMY970MOCBpugvw8W6sg7tPBaYCmNlKM1ua3VCzpgPwWb6DqEahxweFH6Piqx/FVz/1ia8004I4E8ErwIFm1g34CDgL+E7KOo8CF5vZdOBIYI27L69up+7eMY5gs8HMKjNdnlUICj0+KPwYFV/9KL76iSu+2BKBu1eZ2cXAX4CmwD3uvtDMJkTLbwdmAicD7wIbgHPjikdERNKLs0SAu88knOyT592eNO7ARXHGICIi1VOnc9k1Nd8B1KDQ44PCj1Hx1Y/iq59Y4mtwXUyIiEh2qUQgIlLklAhERIqcEkEdmdkBZjbbzBaZ2UIz+0GadYaY2RozWxAN1+U4xiVm9kb03rt01ZrPzv7M7OCk47LAzNaa2WUp6+T8+JnZPWb2qZm9mTRvHzP7m5m9E722y7BttZ0rxhjfL81scfQ3fMTM9s6wbbXfhxjjm2hmHyX9HU/OsG2+jt+DSbEtMbMFGbaN9fhlOqfk9PuXqX9qDRke4ACdgP7ReBvgn0DPlHWGAI/nMcYlQIdqlp8MPEm4s3sA8FKe4mwKfAKU5vv4AYOB/sCbSfN+AVwTjV8D/DzDZ3gP6A60AF5L/T7EGN//AZpF4z9PF19tvg8xxjcR+GEtvgN5OX4py38FXJeP45fpnJLL759KBHXk7svdfX40vg5YRMPrH6lQOvs7AXjP3fN+p7i7zwU+T5l9OnB/NH4/MDzNprXpXDGW+Nz9r+5eFU2+SLgzPy8yHL/ayNvxSzAzA/4deCDb71sb1ZxTcvb9UyKoBzMrA/oBL6VZfJSZvWZmT5pZr5wGFvpr+quZzTOz8WmWF0pnf2eR+Z8vn8cvYT+P7nSPXvdNs06hHMtxhFJeOjV9H+J0cVR1dU+Gqo1COH7HACvc/Z0My3N2/FLOKTn7/ikR7CYzKwEeBi5z97Upi+cTqjv6AjcDM3Ic3iB370943sNFZjY4ZXmtOvuLk5m1AIYBf0izON/Hry4K4VheC1QB0zKsUtP3IS63AV8HyoHlhOqXVHk/fsAoqi8N5OT41XBOybhZmnl1Pn5KBLvBzJoT/mDT3P1Pqcvdfa27r4/GZwLNzaxDruJz94+j10+BRwjFx2S16uwvZkOB+e6+InVBvo9fkhWJKrPo9dM06+T1WJrZOcCpwGiPKo1T1eL7EAt3X+HuW919G3BnhvfN9/FrBowAHsy0Ti6OX4ZzSs6+f0oEdRTVJ94NLHL3X2dYZ/9oPczsCMJxXpWj+PY0szaJcUKD4pspqz0KjImuHhpALTr7i0HGX2H5PH4pHgXOicbPAf6cZp3tnStGpZyzou1iZ2YnAVcDw9x9Q4Z1avN9iCu+5HanMzK8b96OX+REYLG7L0u3MBfHr5pzSu6+f3G1hDfWATiaUPR6HVgQDScDE4AJ0ToXAwsJLfgvAgNzGF/36H1fi2K4NpqfHJ8RHiP6HvAGUJHjY9iacGJvmzQvr8ePkJSWA1sIv7K+B7QHngbeiV73idb9GjAzaduTCVd6vJc43jmK711C/XDie3h7anyZvg85iu+30ffrdcLJqVMhHb9o/n2J713Sujk9ftWcU3L2/VMXEyIiRU5VQyIiRU6JQESkyCkRiIgUOSUCEZEip0QgIlLklAhEIma21XbuGTVrPWGaWVlyz5cihSTWZxaLNDAb3b0830GI5JpKBCI1iPqj/7mZvRwN34jml5rZ01Gnak+bWddo/n4Wng/wWjQMjHbV1MzujPqc/6uZtYrWv9TM3or2Mz1PH1OKmBKByA6tUqqGzkxattbdjwBuAW6K5t1C6M67D6HDt8nR/MnAsx46zetPuCMV4EBgirv3AlYD34rmXwP0i/YzIa4PJ5KJ7iwWiZjZencvSTN/CXC8u78fdQ72ibu3N7PPCN0mbInmL3f3Dma2Euji7l8l7aMM+Ju7HxhNXw00d/f/MrOngPWEXlZneNThnkiuqEQgUjueYTzTOul8lTS+lR1tdKcQ+n46DJgX9YgpkjNKBCK1c2bS6z+i8RcIvT0CjAaej8afBi4AMLOmZrZXpp2aWRPgAHefDfwI2BvYpVQiEif98hDZoZXt/ADzp9w9cQnpHmb2EuHH06ho3qXAPWZ2FbASODea/wNgqpl9j/DL/wJCz5fpNAV+Z2ZtCb3C/o+7r87aJxKpBbURiNQgaiOocPfP8h2LSBxUNSQiUuRUIhARKXIqEYiIFDklAhGRIqdEICJS5JQIRESKnBKBiEiR+/8CHjzLPjM5MQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plotting the training and validation loss\n",
    "loss = history.history[\"loss\"]\n",
    "val_loss = history.history[\"val_loss\"]\n",
    "epochs = range(1, len(loss) + 1)\n",
    "plt.plot(epochs, loss, \"bo\", label=\"Training loss\")\n",
    "plt.plot(epochs, val_loss, \"b\", label=\"Validation loss\")\n",
    "plt.title(\"Training and validation loss\")\n",
    "plt.xlabel(\"Epochs\")\n",
    "plt.ylabel(\"Loss\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU1f3/8dcnQYGIggZEBCSoKIISlkgroqLYCi4obkCplVKLoLj161ZxoVp/X6u2da39Yt2lDVIVlwoqiHtVwiqgCGhYBRFkXyTk/P44d5LJMJNMlplJMu/n4zGPmblzl8/cTM7nnnPuPdecc4iISPrKSHUAIiKSWkoEIiJpTolARCTNKRGIiKQ5JQIRkTSnRCAikuaUCGQvZjbZzC6t6XlTycwKzez0BKzXmdmRweu/m9lt8cxbhe0MNbO3qhqnSHlM1xHUD2a2NextFrAL2BO8v9w5Nz75UdUeZlYIXOacm1rD63VAB+fckpqa18xygG+AfZxzRTURp0h5GqQ6AKkZzrkmodflFXpm1kCFi9QW+j3WDmoaqufMrI+ZrTSzm8xsDfCUmR1oZq+b2Toz+yF43SZsmXfN7LLg9TAz+9DM7g/m/cbM+ldx3vZm9r6ZbTGzqWb2qJk9HyPueGK8y8w+Ctb3lpk1D/v8EjNbZmbrzWxMOfvnp2a2xswyw6YNNLN5weueZvZfM9toZt+a2SNmtm+MdT1tZn8Me39DsMxqMxseMe9ZZjbbzDab2QozGxv28fvB80Yz22pmJ4T2bdjyvcxshpltCp57xbtvKrmfDzKzp4Lv8IOZTQr77FwzmxN8h6Vm1i+YXqYZzszGhv7OZpYTNJH9xsyWA+8E0ycGf4dNwW+kc9jyjc3sz8Hfc1PwG2tsZv8xs6sivs88Mzsv2neV2JQI0sMhwEFAO2AE/u/+VPD+MGAH8Eg5y/8EWAQ0B+4FnjAzq8K8/wQ+A7KBscAl5Wwznhh/AfwaOBjYF7gewMw6AY8F6z802F4bonDOfQJsA06LWO8/g9d7gOuC73MC0Be4opy4CWLoF8TzM6ADENk/sQ34FdAMOAsYFVaAnRw8N3PONXHO/Tdi3QcB/wEeCr7bX4D/mFl2xHfYa99EUdF+fg7f1Ng5WNdfgxh6As8CNwTf4WSgMNb+iOIU4BjgjOD9ZPx+OhiYBYQ3Zd4P9AB64X/HNwLFwDPAL0MzmVku0Bp4oxJxCIBzTo969sD/Q54evO4D/Ag0Kmf+rsAPYe/fxTctAQwDloR9lgU44JDKzIsvZIqArLDPnweej/M7RYvx1rD3VwBTgte3A/lhn+0X7IPTY6z7j8CTwev98YV0uxjzXgu8HPbeAUcGr58G/hi8fhK4J2y+o8LnjbLeB4C/Bq9zgnkbhH0+DPgweH0J8FnE8v8FhlW0byqzn4FW+AL3wCjz/V8o3vJ+f8H7saG/c9h3O7ycGJoF8zTFJ6odQG6U+RoCG/D9LuATxt+S/f9WHx6qEaSHdc65naE3ZpZlZv8XVLU345simoU3j0RYE3rhnNsevGxSyXkPBTaETQNYESvgOGNcE/Z6e1hMh4av2zm3DVgfa1v4o//zzawhcD4wyzm3LIjjqKC5ZE0Qx//D1w4qUiYGYFnE9/uJmU0PmmQ2ASPjXG9o3csipi3DHw2HxNo3ZVSwn9vi/2Y/RFm0LbA0znijKdk3ZpZpZvcEzUubKa1ZNA8ejaJtyzm3C3gB+KWZZQBD8DUYqSQlgvQQeWrY/wBHAz9xzh1AaVNErOaemvAtcJCZZYVNa1vO/NWJ8dvwdQfbzI41s3NuIb4g7U/ZZiHwTUxf4o86DwBuqUoM+BpRuH8CrwJtnXNNgb+HrbeiU/lW45tywh0GrIojrkjl7ecV+L9ZsyjLrQCOiLHObfjaYMghUeYJ/46/AM7FN581xdcaQjF8D+wsZ1vPAEPxTXbbXUQzmsRHiSA97Y+vbm8M2pvvSPQGgyPsAmCsme1rZicA5yQoxn8DZ5tZ76Bj904q/q3/E7gaXxBOjIhjM7DVzDoCo+KM4QVgmJl1ChJRZPz744+2dwbt7b8I+2wdvknm8BjrfgM4ysx+YWYNzGwQ0Al4Pc7YIuOIup+dc9/i2+7/FnQq72NmoUTxBPBrM+trZhlm1jrYPwBzgMHB/HnAhXHEsAtfa8vC17pCMRTjm9n+YmaHBrWHE4LaG0HBXwz8GdUGqkyJID09ADTGH219AkxJ0naH4jtc1+Pb5SfgC4Boqhyjc24BcCW+cP8W+AFYWcFi/8L3p7zjnPs+bPr1+EJ6C/B4EHM8MUwOvsM7wJLgOdwVwJ1mtgXfp/FC2LLbgbuBj8yfrfTTiHWvB87GH82vx3eenh0Rd7wq2s+XALvxtaLv8H0kOOc+w3dG/xXYBLxHaS3lNvwR/A/AHyhbw4rmWXyNbBWwMIgj3PXA58AMfJ/Anyhbdj0LHIfvc5Iq0AVlkjJmNgH40jmX8BqJ1F9m9itghHOud6pjqatUI5CkMbPjzeyIoCmhH75deFJFy4nEEjS7XQGMS3UsdZkSgSTTIfhTG7fiz4Ef5ZybndKIpM4yszPw/Slrqbj5ScqhpiERkTSnGoGISJqrc4PONW/e3OXk5KQ6DBGROmXmzJnfO+daRPusziWCnJwcCgoKUh2GiEidYmaRV6OXUNOQiEiaUyIQEUlzSgQiImlOiUBEJM0pEYiIpDklAhGRCowfDzk5kJHhn8ePr2iJurV9JQIRqfVSWRCPHw8jRsCyZeCcfx4xonIxVCf+mth+hVJ9i7TKPnr06OFEJLmef965du2cM/PPzz+fvOWff965rCznfDHoH1lZlV9HVbffrl3ZbYce7dolJ/7qbj8EKHAxytWUF+yVfSgRSDqqywVxqgvC6m7fLPr2zZITf3W3H6JEIFKH1fWCONUFYbrHH1JeIlAfgUgtN2YMbN9edtr27X56MpZfvrxy02t6+cMi7/ZcwfSa3v7dd0NWVtlpWVl+ejyqG391tx8PJQKRJKhOZ2FdL4hTXRBWd/tDh8K4cdCuHZj553Hj/PR4VDf+6m4/LrGqCrX1oaYhqWvqetNMqpumQutIZWdzdVW3j6cmoD4Ckeqpy2edpLogronlqyvV268NlAhEqiHVZ52EYqjLBbGkXnmJoM7dqjIvL8/pfgSSTDk5/iKeSO3aQWFh4pcXqQlmNtM5lxftM3UWS1pIZWdtMs76EKkOJQKp96p7iX6qzzoRSTQ1DUm9V92mmVAiCT8XPytLhbnULWoakrRW3aYdHdFLfadEIHVCddr4q9u0A77QLyyE4mL/rCQg9YkSgdR61W3jV2etSPmUCKTWq+5YOWraESmfOoul1svI8DWBSGa+qUZEKqbOYqnTaqKNX0RiUyKQpKhOZ6/a+EUSS4lAEq66nb1q4xdJLPURSMJprB2R1FMfgaRUdS/oEpHEUiKQhFNnr0jtpkQgCafOXpHaTYlAEk6dvSK1W0ITgZn1M7NFZrbEzG6O8vmBZvaymc0zs8/M7NhExiOpo7F6RGqvhCUCM8sEHgX6A52AIWbWKWK2W4A5zrkuwK+ABxMVj4iIRJfIGkFPYIlz7mvn3I9APnBuxDydgGkAzrkvgRwza5nAmEREJEIiE0FrYEXY+5XBtHBzgfMBzKwn0A5oE7kiMxthZgVmVrBu3boEhSsikp4SmQgsyrTIq9fuAQ40sznAVcBsoGivhZwb55zLc87ltWjRouYjFRFJY4lMBCuBtmHv2wCrw2dwzm12zv3aOdcV30fQAvgmgTFJFVVnrCARqd0aJHDdM4AOZtYeWAUMBn4RPoOZNQO2B30IlwHvO+c2JzAmqYLIe/aGxgoCnf0jUh8krEbgnCsCRgNvAl8ALzjnFpjZSDMbGcx2DLDAzL7En110TaLikaqr7o1hRKR206BzUiHdGEak7tOgc1ItGitIpH5TIpAKaawgkfpNiUAqpLGCROq3RJ41JPXI0KEq+EXqK9UIRETSnBKBiEiaUyIQEUlzSgQiImlOiUBEJM0pEYiIpDklgjSgkUNFpDy6jqCe08ihIlIR1QjqOY0cKiIVUSKo55Yvr9x0EUk/SgT1nEYOFZGKKBHUcxo5VEQqokRQz2nkUBGpiM4aSgMaOVREyqMagYhImlMiEBFJc0oEIiJpTolARCTNKRGIiKQ5JQIRkTSnRCAikuaUCERE0pwSgYhImlMiEBFJc0oEdYDuMCbpqrgY1q+HL76ATz6BH39MdUT1k8YaquV0h7HqW7wYJkyAt9+GQw6BY4/1j86d4YgjIDMz1RGmD+dgyxZYtw6++670Efk+NG3dOtizp3T5E06AV1+F5s1T9x3qI3POpTqGSsnLy3MFBQWpDiNpcnJ84R+pXTsoLEx2NHVHYSG88IJPALNm+Wndu8MPP8A335TO16gRHHOMTwqh5HDssf5+DRmqL9eIXbvgqafg4Ydh6VL/PpoDDoCDD4YWLfxz5OOHH+B3v4O2bWHKFDj88OR+j7rOzGY65/KifqZEULtlZPijqEhmvtospVatgokTfeH/ySd+Ws+eMGgQXHSRL0AAtm71TQ0LFsD8+f6xYAGsXFm6riZNoFOnsrWH446DVq2S/73qqm3b4PHH4b77YPVq/7c45ZTSgj28wG/Rwiflinz8MZxzDjRoAK+/Dscfn/jvUV8oEdRh9aFGUFgI06fDzJnQsiUceaR/dOgAzZpVb93ffQf//rcv/D/4wCfNrl194X/xxZU7aty40SeEyATx3Xel8xx1FPTv7x8nnwyNG1cv/vpo82Z49FH4y1/g++/h1FP9PbJPO80fwFTXokV+/69d6//uZ59d/XWmAyWCOiyyjwD8HcZq881lVq3yBf/06fDOO6UJq0kTfzQeLjvbJ4Tw5BB6fdBB0de/YQO89BLk5/ttFBf7o/fBg30COOqomv0+69b5hDB7Nrz1Frz7Luzc6ZNAnz6lieHII2t2u+CPpGfO9I8FC/beX0ccEd+RdDKsXw8PPuibgDZu9PtkzBg48cSa39batT4BzJrlk87IkTW/jfpGiaCOGz/e/0MtX+7bru++u3Ylge++84VjqOD/6is//cADfVPAaaf5o8LOnX0BunQpLFniH4sXlz6vWFF2vQcdVDZBZGfD5Mm+07eoyE8bNMg/jj02ed93xw7/fadM8fEsXuynH3FEaVLo02fvW4SWxzmfQEOF/qxZ/nnNGv95Roav3WzY4B8hZtCmTfRkesQRyamxrFkDf/4zPPaYbw46/3y45Rbo0SOx2922zf/t//Mf+P3v/f9FTdQ4aopzvl8jvOO7ZUv/f1DdmnBVKBFIjfrhB3jvPV/oT5/um1DAH/GffHJpwZ+bW7kzcnbs8B25oeQQniiWL/f/WO3alRb+3brVjn/8pUtLk8I77/jv0bChT4KhxHDUUaWxOueTXqiwDz1CTVAZGb4Du0eP0kfXrrDffv7zDRv8NsOTaGh/ff992djatNm7ptWhg08SlUlU0SxfDvfeC//4B+zeDUOG+AK5c+fqrbcyiorgyitLa8hPPgn77pu47W3btvfZTbHOfFq3zscXTevWZfufjj3W12pDf+NESFkiMLN+wINAJvAP59w9EZ83BZ4HDsOfynq/c+6p8tapRJBczvlCZ+ZM+OwzfyQ8e7af3rgx9O7tC/3TTvMFVoMEnZC8a5dvDmjbtnYU/rHs3On7KiZP9o8vv/TTc3L8flq92ieAdev89MxMXwD06OHPaurRwyfQqhYIGzfunURDr0PbDGndOnqT3JFHlr/9JUvgnnvgmWf83+JXv4Kbb05M01g8nIP//d/SfoiXXoKmTWtu/atW+drOE0+U1tAiNWkSvRM8/JGd7f/+4f1PCxeWPYvq8MNLE0MoSXTs6A8sqislicDMMoGvgJ8BK4EZwBDn3MKweW4BmjrnbjKzFsAi4BDnXMzLRpQIEqe4uLTQD2+i2LTJf77vvv487lDB37NnzfxA67PCwtLawocf+iP08CP9Ll2qf2Qer02bytYkwpPF2rVl523Vau8E0by5P/rPz/e/hcsugxtu8M2VtcFzz8Hw4b7gfOON0rPEqsI5+Ogj39/x4ov+f+Pss/2BT2SB36JF1f+Ge/b4v0noBIXQ86JFpbWJzEz/d+jc2de6LrigattKVSI4ARjrnDsjeP97AOfc/4bN83ugLXAlkAO8DRzlnIt5YqQSQc0oLvaFQHjzxKxZ/owP8P/oXbqULbSOPTax1W5JnS1bSpNDZG3i229L59tvP7jiCn8+/yGHpC7eWKZN830U++/vk0GXLpVbfscO+Ne/fAKYM8e35V92mf/O7dsnJuZofvzR/w3Caw/z5/tEd/PNVVtnqhLBhUA/59xlwftLgJ8450aHzbM/8CrQEdgfGOSc+09561UiqJrdu32V+dNPfaE/e7b/5wd/VJ+bW7bQ79RJhb54W7f6o9bly6FXL9/EUZvNmwdnnul/3y+9BH37VrzM8uXwt7/5Gs/69f6akauu8v0OyaqxxcO5qjeNlpcIEjnERLRwI7POGcAc4DTgCOBtM/vAObe5zIrMRgAjAA6rLfXQOmTWLH8kMXeuP9Wwa1e45JKyhf4++6Q6SqmtmjTxBwq5uamOJD5duvgLCvv3h379fAfyJZfsPZ9z/qSHhx6CV17x0847D66+2p/0UBv7ohIVUyITwUp8s09IG2B1xDy/Bu5xvlqyxMy+wdcOPgufyTk3DhgHvkaQsIjrmZ074Q9/8Fd2Hnywb+scMCBxHboitUWbNr5PZuBA35m9YoU/o8nMn/kzfrxv/pk/39dwbrwRRo2qPf0dyZbIImEG0MHM2gOrgMHALyLmWQ70BT4ws5bA0cDXCYwpbXz0EfzmN77Tafhwf553Ks5dFkmVpk19R/3w4f6Moq+/9v8DTzzhz67q1s3XFgYP1hXiCUsEzrkiMxsNvIk/ffRJ59wCMxsZfP534C7gaTP7HN+UdJNz7vuYK5UKbd3qf/QPP+yPbt58E37+81RHJZIa++7rzyY67DB/immDBv6sm6uu8v0dtbH5JxV0QVk9MnUq/Pa3/pTF0aP9D79Jk1RHJVI7fPihP/OndetUR5IaqeosliTZtAmuv96f8XDUUf6Cpt69Ux2VSO2i/4nYKhxx3czONjONzF5LvfaaP+vnySd9h9ecOfrBi0jlxFPADwYWm9m9ZnZMogOS+Hz/vT/HecAAf9bDp5/Cn/6kTi8RqbwKE4Fz7pdAN2Ap8JSZ/dfMRgQXg0mSOefHYO/Uyd+EZexYKCiAvKgtfyIiFYurySe4wOtFIB9oBQwEZpnZVQmMTSKsXu3Pix482A9iNmsW3HGHrgAWkeqJp4/gHDN7GXgH2Afo6ZzrD+QC1yc4PsHXAp56yg869eabfujfjz9O7hj8IlJ/xVMjuAj4q3Oui3PuPufcdwDOue3A8IRGV0+MH++P4DMy/PP48fEvW1gIZ5zhL4o57jg/TMQNN+jqYBGpOfEUJ3cAJeMPmlljoKVzrtA5Ny1hkdUTkbeaXLbMv4fy7zJWXOzHQL/pJn/RS+h2fBk6f0tEalg8xcpEIHxY6D3BNInDmDFl7zcM/v2YMbGX+eorf6vD0aP9/V7nz/fD4CoJiEgixFO0NAi/UUzwWt2TcVq+PP7pRUW+/T83Fz7/3PcLTJnib88oIpIo8SSCdWY2IPTGzM4FNB5QnGKNZhg5/fPP/d2/brrJD527cCEMG6axUEQk8eJJBCOBW8xsuZmtAG4CLk9sWPXH3XfvfWOLrCw/HfydiMaO9fcFWLbMXyPw0kv+VoEiIslQYWexc24p8FMza4IfpG5L4sOqP0IdwmPG+Oagww7zSWDoUJgxww8V/fnn/v0DD/j7woqIJFNcJyGa2VlAZ6CRBW0Vzrk7ExhXvTJ0aNkzhHbs8OMC/fnP/sj/tdf8jbFFRFKhwkRgZn8HsoBTgX8AFxJxBzGJ3wcf+FrA4sV+yOj77vM30BARSZV4+gh6Oed+BfzgnPsDcAJlb0Epcdi5058OevLJ/uygadNg3DglARFJvXgSwc7gebuZHQrsBtonLqT6xzm4/HJ/Udg11/g+gdNOS3VUIiJePH0Er5lZM+A+YBbggMcTGlU988gj8Oyz/uygO+5IdTQiImWVmwiCG9JMc85tBF40s9eBRs65TUmJrh547z247jp/34Dbbkt1NCIieyu3acg5Vwz8Oez9LiWB+C1fDhddBEce6W+grSEiRKQ2iqdoesvMLjDTNa6VsWMHnH++7ySeNAkOOCDVEYmIRBdPH8HvgP2AIjPbCRjgnHMq2mJwzo8UOnMmvPIKdOyY6ohERGKL58pi3ZKyksI7hwcMqHB2EZGUiueCspOjTXfOvV/z4dR96hwWkbomnqahG8JeNwJ6AjMBnQkfQZ3DIlIXxdM0dE74ezNrC9ybsIjqKHUOi0hdVZU7364EdNv0MOocFpG6LJ4+gofxVxODP920KzA3kUHVNeocFpG6LJ4aQUHY6yLgX865jxIUT52jzmERqeviSQT/BnY65/YAmFmmmWU557ZXsFy9p85hEakP4im6pgGNw943BqYmJpy6Q53DIlJfxFMjaOSc2xp645zbamZZ5S1Q36lzWETqk3hqBNvMrHvojZn1AHYkLqTaL9Q5fMcd6hwWkbovnhrBtcBEM1sdvG8FDEpcSLVbeOfw7benOhoRkeqL54KyGWbWETgaP+Dcl8653QmPrBZS57CI1EcVFmVmdiWwn3NuvnPuc6CJmV2R+NBql5071TksIvVTPMe0vw3uUAaAc+4H4LeJC6l2euYZ3zn8zDPqHBaR+iWeRJARflMaM8sE9o1n5WbWz8wWmdkSM7s5yuc3mNmc4DHfzPaY2UHxh588//oXHH00nHdeqiMREalZ8SSCN4EXzKyvmZ0G/AuYXNFCQcJ4FOgPdAKGmFmn8Hmcc/c557o657oCvwfec85tqOyXSLSHH/adxIsWQfv2MH58qiMSEak58Zw1dBMwAhiF7yyejT9zqCI9gSXOua8BzCwfOBdYGGP+IfgkU6uMHw//8z+l75ctgxEj/OuhQ1MTk4hITaqwRhDcwP4T4GsgD+gLfBHHulsDK8Lerwym7SW4QK0f8GKMz0eYWYGZFaxbty6OTdecMWNgd8Q5Utu3++kiIvVBzBqBmR0FDMYfqa8HJgA4506Nc93RbnbvokwDOAf4KFazkHNuHDAOIC8vL9Y6EmLZsujTly9PZhQiIolTXo3gS/zR/znOud7OuYeBPZVY90qgbdj7NsDqGPMOphY2CwE0axZ9+mGHJTcOEZFEKS8RXACsAaab2eNm1pfoR/mxzAA6mFl7M9sXX9i/GjmTmTUFTgFeqcS6k2b//fe+cCwrC+6+OzXxiIjUtJiJwDn3snNuENAReBe4DmhpZo+Z2c8rWrFzrggYjT/r6AvgBefcAjMbaWYjw2YdCLzlnNtWje+REAsWwIoV8MtfQrt2YOafx41TR7GI1B/mXPxN7sE5/hcBg5xzKbl5fV5enisoKKh4xhpw++3+yH/VKjjkkKRsUkQkIcxspnMuL9pnlRotxzm3wTn3f6lKAsnknL+IrE8fJQERqd80bFoMs2bBkiUwZEiqIxERSSwlghjy82GfffxAcyIi9ZkSQRTFxT4RnHEGHFQrRz4SEak5SgRRfPwxrFwJgwenOhIRkcRTIogiPx8aNdJtKEUkPSgRRCgqgokT4Zxz/MVkIiL1nRJBhOnT4bvv1CwkIulDiSBCfr6vCfTvn+pIRESSQ4kgzK5d8OKLMHAgNG6c6mhERJJDiSDMm2/Cpk1qFhKR9KJEECY/H7Kz4fTTUx2JiEjyKBEEtm2DV16BCy/0VxSLiKQLJYLA66/7W1CqWUhE0o0SQSA/H1q1gpNOSnUkIiLJpUQAbNwIb7wBgwZBZmaqoxERSS4lAmDSJPjxRzULiUh6UiLANwu1bw89e6Y6EhGR5Ev7RLBuHUyd6msDZqmORkQk+dI+Efz737Bnj5qFRCR9pX0iyM+HTp3guONSHYmISGqkdSJYuRI++EDNQiKS3tI6EbzwAjinZiERSW9pnQjy86FHD+jQIdWRiIikTtomgiVLYMYM1QZERNI2EUyY4J8vvji1cYiIpFraJoL8fOjdGw47LNWRiIikVlomgvnz/UPNQiIiaZoI8vMhI8Pfe0BEJN2lXSJwzieCvn2hZctURyMiknpplwgKCmDpUjULiYiEpF0iyM/3t6IcODDVkYiI1A5plQiKi/1po/37w4EHpjoaEZHaIa0SwYcfwqpVahYSEQmXVokgPx+ysmDAgFRHIiJSe6RNIigqgokT4ZxzYL/9Uh2NiEjtkdBEYGb9zGyRmS0xs5tjzNPHzOaY2QIzey9RsUybBt9/r2YhEZFIDRK1YjPLBB4FfgasBGaY2avOuYVh8zQD/gb0c84tN7ODExVPq1Zw+eW+o1hEREolskbQE1jinPvaOfcjkA+cGzHPL4CXnHPLAZxz3yUqmC5d4O9/h4YNE7UFEZG6KZGJoDWwIuz9ymBauKOAA83sXTObaWa/SmA8IiISRcKahoBoN390UbbfA+gLNAb+a2afOOe+KrMisxHACIDDNFyoiEiNSmSNYCXQNux9G2B1lHmmOOe2Oee+B94HciNX5Jwb55zLc87ltWjRImEBi4iko0QmghlABzNrb2b7AoOBVyPmeQU4ycwamFkW8BPgiwTGJCIiERLWNOScKzKz0cCbQCbwpHNugZmNDD7/u3PuCzObAswDioF/OOfmJyomERHZmzkX2Wxfu+Xl5bmCgoJUhyEiUqeY2UznXF60z9LmymIREYkukWcNiUiC7d69m5UrV7Jz585UhyK1RKNGjWjTpg377LNP3MsoEYjUYStXrmT//fcnJycHs2hnbEs6cc6xfv16Vq5cSfv27eNeTk1DInXYzp07yc7OVhIQAMyM7OzsStcQlQhE6jglAQlXld+DEoGISJpTIhBJI+PHQ04OZGT45/Hjq7e+9evX07VrV7p27cohhxxC69atS97/+OOP5S5bUFDA1VdfXeE2evXqVb0gpULqLBZJE+PHw4gRsH27f79smX8PMHRo1daZnZ3NnDlzABg7dixNmjTh+uuvL/m8qHMnjTYAABD0SURBVKiIBg2iFzN5eXnk5UU9rb2Mjz/+uGrBpdCePXvIzMxMdRhxU41AJE2MGVOaBEK2b/fTa9KwYcP43e9+x6mnnspNN93EZ599Rq9evejWrRu9evVi0aJFALz77rucffbZgE8iw4cPp0+fPhx++OE89NBDJetr0qRJyfx9+vThwgsvpGPHjgwdOpTQBbFvvPEGHTt2pHfv3lx99dUl6w1XWFjISSedRPfu3enevXuZBHPvvfdy3HHHkZuby803+3toLVmyhNNPP53c3Fy6d+/O0qVLy8QMMHr0aJ5++mkAcnJyuPPOO+nduzcTJ07k8ccf5/jjjyc3N5cLLriA7cHOX7t2LQMHDiQ3N5fc3Fw+/vhjbrvtNh588MGS9Y4ZM6bMPkg01QhE0sTy5ZWbXh1fffUVU6dOJTMzk82bN/P+++/ToEEDpk6dyi233MKLL7641zJffvkl06dPZ8uWLRx99NGMGjVqr3PhZ8+ezYIFCzj00EM58cQT+eijj8jLy+Pyyy/n/fffp3379gwZMiRqTAcffDBvv/02jRo1YvHixQwZMoSCggImT57MpEmT+PTTT8nKymLDhg0ADB06lJtvvpmBAweyc+dOiouLWbFiRdR1hzRq1IgPP/wQ8M1mv/3tbwG49dZbeeKJJ7jqqqu4+uqrOeWUU3j55ZfZs2cPW7du5dBDD+X888/nmmuuobi4mPz8fD777LNK7/eqUiIQSROHHeabg6JNr2kXXXRRSdPIpk2buPTSS1m8eDFmxu7du6Muc9ZZZ9GwYUMaNmzIwQcfzNq1a2nTpk2ZeXr27FkyrWvXrhQWFtKkSRMOP/zwkvPmhwwZwrhx4/Za/+7duxk9ejRz5swhMzOTr77yo91PnTqVX//612RlZQFw0EEHsWXLFlatWsXAgQMBX8DHY9CgQSWv58+fz6233srGjRvZunUrZ5xxBgDvvPMOzz77LACZmZk0bdqUpk2bkp2dzezZs1m7di3dunUjOzs7rm3WBCUCkTRx991l+wgAsrL89Jq23377lby+7bbbOPXUU3n55ZcpLCykT58+UZdpGHb7wMzMTIqKiuKaJ97x0v7617/SsmVL5s6dS3FxcUnh7pzb65TLWOts0KABxcXFJe8jz9cP/97Dhg1j0qRJ5Obm8vTTT/Puu++WG99ll13G008/zZo1axg+fHhc36mmqI9AJE0MHQrjxkG7dmDmn8eNq3pHcbw2bdpE69b+5oSh9vSa1LFjR77++msKCwsBmDBhQsw4WrVqRUZGBs899xx79uwB4Oc//zlPPvlkSRv+hg0bOOCAA2jTpg2TJk0CYNeuXWzfvp127dqxcOFCdu3axaZNm5g2bVrMuLZs2UKrVq3YvXs348NOz+rbty+PPfYY4DuVN2/eDMDAgQOZMmUKM2bMKKk9JIsSgUgaGToUCguhuNg/JzoJANx44438/ve/58QTTywpfGtS48aN+dvf/ka/fv3o3bs3LVu2pGnTpnvNd8UVV/DMM8/w05/+lK+++qrk6L1fv34MGDCAvLw8unbtyv333w/Ac889x0MPPUSXLl3o1asXa9asoW3btlx88cV06dKFoUOH0q1bt5hx3XXXXfzkJz/hZz/7GR07diyZ/uCDDzJ9+nSOO+44evTowYIFCwDYd999OfXUU7n44ouTfsaRhqEWqcO++OILjjnmmFSHkXJbt26lSZMmOOe48sor6dChA9ddd12qw6qU4uJiunfvzsSJE+nQoUO11hXtd6FhqEWkXnv88cfp2rUrnTt3ZtOmTVx++eWpDqlSFi5cyJFHHknfvn2rnQSqQp3FIlLnXXfddXWuBhCuU6dOfP311ynbvmoEIiJpTolARCTNKRGIiKQ5JQIRkTSnRCAiVdanTx/efPPNMtMeeOABrrjiinKXCZ0CfuaZZ7Jx48a95hk7dmzJ+fyxTJo0iYULF5a8v/3225k6dWplwpeAEoGIVNmQIUPIz88vMy0/Pz/mwG+R3njjDZo1a1albUcmgjvvvJPTTz+9SutKlURcYFcVOn1UpJ649loIbg1QY7p2hQceiP35hRdeyK233squXbto2LAhhYWFrF69mt69ezNq1ChmzJjBjh07uPDCC/nDH/6w1/I5OTkUFBTQvHlz7r77bp599lnatm1LixYt6NGjB+CvERg3bhw//vgjRx55JM899xxz5szh1Vdf5b333uOPf/wjL774InfddRdnn302F154IdOmTeP666+nqKiI448/nscee4yGDRuSk5PDpZdeymuvvcbu3buZOHFimat+wQ9Xfckll7Bt2zYAHnnkkZKb49x7770899xzZGRk0L9/f+655x6WLFnCyJEjWbduHZmZmUycOJEVK1Zw//338/rrrwN+uOq8vDyGDRtGTk4Ow4cP56233mL06NFs2bJlr++XlZXF2rVrGTlyZMlppY899hiTJ0+mefPmXHPNNYAfrrply5Zx3eCnPKoRiEiVZWdn07NnT6ZMmQL42sCgQYMwM+6++24KCgqYN28e7733HvPmzYu5npkzZ5Kfn8/s2bN56aWXmDFjRsln559/PjNmzGDu3Lkcc8wxPPHEE/Tq1YsBAwZw3333MWfOHI444oiS+Xfu3MmwYcOYMGECn3/+OUVFRSVj+wA0b96cWbNmMWrUqKjNT6HhqmfNmsWECRNKCtnw4arnzp3LjTfeCPjhqq+88krmzp3Lxx9/TKtWrSrcb6HhqgcPHhz1+wElw1XPnTuXWbNm0blzZ37zm9/wzDPPAJQMVz20BsYJUY1ApJ4o78g9kULNQ+eeey75+fk8+eSTALzwwguMGzeOoqIivv32WxYuXEiXLl2iruODDz5g4MCBJUNBDxgwoOSzWMM5x7Jo0SLat2/PUUcdBcCll17Ko48+yrXXXgv4xALQo0cPXnrppb2WT8fhqtOiRlDT92kVkVLnnXce06ZNY9asWezYsYPu3bvzzTffcP/99zNt2jTmzZvHWWedtdeQzZEih4IOGTZsGI888giff/45d9xxR4XrqWj8tNBQ1rGGug4frrqgoKDk3suJHK66Mt8vNFz1U089VWPDVdf7RBC6T+uyZeBc6X1alQxEakaTJk3o06cPw4cPL+kk3rx5M/vttx9NmzZl7dq1TJ48udx1nHzyybz88svs2LGDLVu28Nprr5V8Fms45/33358tW7bsta6OHTtSWFjIkiVLAD+K6CmnnBL390nH4arrfSJI1n1aRdLZkCFDmDt3LoMHDwYgNzeXbt260blzZ4YPH86JJ55Y7vLdu3dn0KBBdO3alQsuuICTTjqp5LNYwzkPHjyY++67j27durF06dKS6Y0aNeKpp57ioosu4rjjjiMjI4ORI0fG/V3Scbjqej8MdUaGrwlEMvNjsovUZRqGOv3EM1y1hqGOEOt+rIm4T6uISCIlarjqen/WUDLv0yoikkiJGq663tcIUnWfVpFkqWvNu5JYVfk91PsaAfhCXwW/1EeNGjVi/fr1ZGdnxzz9UtKHc47169fHfT1DSFokApH6qk2bNqxcuZJ169alOhSpJRo1akSbNm0qtYwSgUgdts8++9C+fftUhyF1XL3vIxARkfIpEYiIpDklAhGRNFfnriw2s3XAslTHEUNz4PtUB1GO2h4f1P4YFV/1KL7qqU587ZxzLaJ9UOcSQW1mZgWxLuGuDWp7fFD7Y1R81aP4qidR8alpSEQkzSkRiIikOSWCmjUu1QFUoLbHB7U/RsVXPYqvehISn/oIRETSnGoEIiJpTolARCTNKRFUkpm1NbPpZvaFmS0ws2uizNPHzDaZ2ZzgcXuSYyw0s8+Dbe91OzfzHjKzJWY2z8y6JzG2o8P2yxwz22xm10bMk/T9Z2ZPmtl3ZjY/bNpBZva2mS0Ong+MsWw/M1sU7M+bkxjffWb2ZfA3fNnMmsVYttzfQwLjG2tmq8L+jmfGWDZV+29CWGyFZjYnxrIJ3X+xypSk/v6cc3pU4gG0AroHr/cHvgI6RczTB3g9hTEWAs3L+fxMYDJgwE+BT1MUZyawBn+hS0r3H3Ay0B2YHzbtXuDm4PXNwJ9ifIelwOHAvsDcyN9DAuP7OdAgeP2naPHF83tIYHxjgevj+A2kZP9FfP5n4PZU7L9YZUoyf3+qEVSSc+5b59ys4PUW4AugdWqjqrRzgWed9wnQzMxapSCOvsBS51zKrxR3zr0PbIiYfC7wTPD6GeC8KIv2BJY45752zv0I5AfLJTw+59xbzrmi4O0nQOXGHq5BMfZfPFK2/0LM38jhYuBfNb3deJRTpiTt96dEUA1mlgN0Az6N8vEJZjbXzCabWeekBgYOeMvMZprZiCiftwZWhL1fSWqS2WBi//Olcv+FtHTOfQv+nxU4OMo8tWVfDsfX8qKp6PeQSKODpqsnYzRt1Ib9dxKw1jm3OMbnSdt/EWVK0n5/SgRVZGZNgBeBa51zmyM+noVv7sgFHgYmJTm8E51z3YH+wJVmdnLE59FuZZXU84jNbF9gADAxysep3n+VURv25RigCBgfY5aKfg+J8hhwBNAV+Bbf/BIp5fsPGEL5tYGk7L8KypSYi0WZVun9p0RQBWa2D/4PNt4591Lk5865zc65rcHrN4B9zKx5suJzzq0Onr8DXsZXH8OtBNqGvW8DrE5OdCX6A7Occ2sjP0j1/guzNtRkFjx/F2WelO5LM7sUOBsY6oJG40hx/B4Swjm31jm3xzlXDDweY7up3n8NgPOBCbHmScb+i1GmJO33p0RQSUF74hPAF865v8SY55BgPsysJ34/r09SfPuZ2f6h1/gOxfkRs70K/Co4e+inwKZQFTSJYh6FpXL/RXgVuDR4fSnwSpR5ZgAdzKx9UMsZHCyXcGbWD7gJGOCc2x5jnnh+D4mKL7zfaWCM7aZs/wVOB750zq2M9mEy9l85ZUryfn+J6gmvrw+gN77qNQ+YEzzOBEYCI4N5RgML8D34nwC9khjf4cF25wYxjAmmh8dnwKP4sw0+B/KSvA+z8AV707BpKd1/+KT0LbAbf5T1GyAbmAYsDp4PCuY9FHgjbNkz8Wd6LA3t7yTFtwTfPhz6Hf49Mr5Yv4ckxfdc8Puahy+cWtWm/RdMfzr0uwubN6n7r5wyJWm/Pw0xISKS5tQ0JCKS5pQIRETSnBKBiEiaUyIQEUlzSgQiImlOiUAkYGZ7rOzIqDU2EqaZ5YSPfClSmzRIdQAitcgO51zXVAchkmyqEYhUIBiP/k9m9lnwODKY3s7MpgWDqk0zs8OC6S3N3x9gbvDoFawq08weD8acf8vMGgfzX21mC4P15Kfoa0oaUyIQKdU4omloUNhnm51zPYFHgAeCaY/gh/Pugh/w7aFg+kPAe84Pmtcdf0UqQAfgUedcZ2AjcEEw/WagW7CekYn6ciKx6MpikYCZbXXONYkyvRA4zTn3dTA42BrnXLaZfY8fNmF3MP1b51xzM1sHtHHO7QpbRw7wtnOuQ/D+JmAf59wfzWwKsBU/yuokFwy4J5IsqhGIxMfFeB1rnmh2hb3eQ2kf3Vn4sZ96ADODETFFkkaJQCQ+g8Ke/xu8/hg/2iPAUODD4PU0YBSAmWWa2QGxVmpmGUBb59x04EagGbBXrUQkkXTkIVKqsZW9gfkU51zoFNKGZvYp/uBpSDDtauBJM7sBWAf8Oph+DTDOzH6DP/IfhR/5MppM4Hkza4ofFfavzrmNNfaNROKgPgKRCgR9BHnOue9THYtIIqhpSEQkzalGICKS5lQjEBFJc0oEIiJpTolARCTNKRGIiKQ5JQIRkTT3/wHW3ImqripvQQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plotting the training and validation accuracy\n",
    "plt.clf()\n",
    "acc = history.history[\"accuracy\"]\n",
    "val_acc = history.history[\"val_accuracy\"]\n",
    "plt.plot(epochs, acc, \"bo\", label=\"Training accuracy\")\n",
    "plt.plot(epochs, val_acc, \"b\", label=\"Validation accuracy\")\n",
    "plt.title(\"Training and validation accuracy\")\n",
    "plt.xlabel(\"Epochs\")\n",
    "plt.ylabel(\"Accuracy\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "由图可见，网络在训练9轮后开始过拟合，因此我们从头开始训练一个新网络"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/9\n",
      "18/18 [==============================] - 1s 15ms/step - loss: 3.1356 - accuracy: 0.4042\n",
      "Epoch 2/9\n",
      "18/18 [==============================] - 0s 19ms/step - loss: 1.5120 - accuracy: 0.6950\n",
      "Epoch 3/9\n",
      "18/18 [==============================] - 0s 19ms/step - loss: 1.0402 - accuracy: 0.7798\n",
      "Epoch 4/9\n",
      "18/18 [==============================] - 0s 20ms/step - loss: 0.8158 - accuracy: 0.8282\n",
      "Epoch 5/9\n",
      "18/18 [==============================] - 0s 16ms/step - loss: 0.6275 - accuracy: 0.8693\n",
      "Epoch 6/9\n",
      "18/18 [==============================] - 0s 19ms/step - loss: 0.5104 - accuracy: 0.8912\n",
      "Epoch 7/9\n",
      "18/18 [==============================] - 0s 15ms/step - loss: 0.4131 - accuracy: 0.9131\n",
      "Epoch 8/9\n",
      "18/18 [==============================] - 0s 15ms/step - loss: 0.3441 - accuracy: 0.9287\n",
      "Epoch 9/9\n",
      "18/18 [==============================] - 0s 15ms/step - loss: 0.2649 - accuracy: 0.9448\n",
      "71/71 [==============================] - 0s 1ms/step - loss: 0.9505 - accuracy: 0.7930\n"
     ]
    }
   ],
   "source": [
    "# Retraining a model from scratch\n",
    "model = keras.Sequential([\n",
    "  layers.Dense(64, activation=\"relu\"),\n",
    "  layers.Dense(64, activation=\"relu\"),\n",
    "  layers.Dense(46, activation=\"softmax\")\n",
    "])\n",
    "model.compile(optimizer=\"rmsprop\",\n",
    "              loss=\"categorical_crossentropy\",\n",
    "              metrics=[\"accuracy\"])\n",
    "model.fit(x_train,\n",
    "          y_train,\n",
    "          epochs=9,\n",
    "          batch_size=512)\n",
    "results = model.evaluate(x_test, y_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.1829919857524488"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 计算完全随机的精度，大约为18%\n",
    "import copy\n",
    "test_labels_copy = copy.copy(test_labels)\n",
    "np.random.shuffle(test_labels_copy)\n",
    "hits_array = np.array(test_labels) == np.array(test_labels_copy)\n",
    "hits_array.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.5.5 Generating predictions on new data 在新数据上生成预测结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = model.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(46,)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#每个元素都是长度为46的向量\n",
    "predictions[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0000001"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#这个向量的所有元素总和为1\n",
    "np.sum(predictions[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#最大的元素就是预测类别，即概率最大的类别\n",
    "np.argmax(predictions[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.5.6 A different way to handle the labels and the loss 处理标签和损失的另一种方法\n",
    "前面提到了另一种编码标签的方法，就是将其转换为整数张量，如下所示。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = np.array(train_labels)\n",
    "y_test = np.array(test_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "对于这种编码方法，唯一需要改变的是损失函数的选择。  \n",
    "原本的损失函数 categorical_crossentropy，标签应该遵循分类编码。  \n",
    "对于整数标签，你应该使用sparse_categorical_crossentropy。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer='rmsprop',\n",
    "        loss='sparse_categorical_crossentropy',\n",
    "        metrics=['acc'])\n",
    "#这个新的损失函数在数学上与categorical_crossentropy完全相同，二者只是接口不同。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.5.7 The importance of having sufficiently large intermediate layers\n",
    "中间层维度足够大的重要性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "63/63 [==============================] - 1s 14ms/step - loss: 3.2789 - accuracy: 0.1416 - val_loss: 2.0881 - val_accuracy: 0.5820\n",
      "Epoch 2/20\n",
      "63/63 [==============================] - 1s 10ms/step - loss: 1.7881 - accuracy: 0.6146 - val_loss: 1.5757 - val_accuracy: 0.6010\n",
      "Epoch 3/20\n",
      "63/63 [==============================] - 1s 10ms/step - loss: 1.4347 - accuracy: 0.6340 - val_loss: 1.4783 - val_accuracy: 0.6240\n",
      "Epoch 4/20\n",
      "63/63 [==============================] - 1s 9ms/step - loss: 1.2615 - accuracy: 0.6527 - val_loss: 1.4279 - val_accuracy: 0.6220\n",
      "Epoch 5/20\n",
      "63/63 [==============================] - 1s 9ms/step - loss: 1.1276 - accuracy: 0.6780 - val_loss: 1.4198 - val_accuracy: 0.6340\n",
      "Epoch 6/20\n",
      "63/63 [==============================] - 1s 9ms/step - loss: 1.0698 - accuracy: 0.6970 - val_loss: 1.4303 - val_accuracy: 0.6420\n",
      "Epoch 7/20\n",
      "63/63 [==============================] - 1s 9ms/step - loss: 0.9986 - accuracy: 0.7199 - val_loss: 1.4244 - val_accuracy: 0.6670\n",
      "Epoch 8/20\n",
      "63/63 [==============================] - 1s 9ms/step - loss: 0.9506 - accuracy: 0.7442 - val_loss: 1.4660 - val_accuracy: 0.6610\n",
      "Epoch 9/20\n",
      "63/63 [==============================] - 1s 10ms/step - loss: 0.8976 - accuracy: 0.7536 - val_loss: 1.4464 - val_accuracy: 0.6630\n",
      "Epoch 10/20\n",
      "63/63 [==============================] - 1s 10ms/step - loss: 0.8376 - accuracy: 0.7760 - val_loss: 1.5083 - val_accuracy: 0.6690\n",
      "Epoch 11/20\n",
      "63/63 [==============================] - 1s 10ms/step - loss: 0.7741 - accuracy: 0.7871 - val_loss: 1.4993 - val_accuracy: 0.6690\n",
      "Epoch 12/20\n",
      "63/63 [==============================] - 1s 10ms/step - loss: 0.6959 - accuracy: 0.8032 - val_loss: 1.5343 - val_accuracy: 0.6750\n",
      "Epoch 13/20\n",
      "63/63 [==============================] - 1s 10ms/step - loss: 0.6701 - accuracy: 0.8066 - val_loss: 1.6229 - val_accuracy: 0.6760\n",
      "Epoch 14/20\n",
      "63/63 [==============================] - 1s 10ms/step - loss: 0.6129 - accuracy: 0.8230 - val_loss: 1.6305 - val_accuracy: 0.6880\n",
      "Epoch 15/20\n",
      "63/63 [==============================] - 1s 10ms/step - loss: 0.5717 - accuracy: 0.8370 - val_loss: 1.6627 - val_accuracy: 0.6830\n",
      "Epoch 16/20\n",
      "63/63 [==============================] - 1s 10ms/step - loss: 0.5481 - accuracy: 0.8432 - val_loss: 1.7123 - val_accuracy: 0.6810\n",
      "Epoch 17/20\n",
      "63/63 [==============================] - 1s 10ms/step - loss: 0.5388 - accuracy: 0.8450 - val_loss: 1.7624 - val_accuracy: 0.6860\n",
      "Epoch 18/20\n",
      "63/63 [==============================] - 1s 11ms/step - loss: 0.4747 - accuracy: 0.8645 - val_loss: 1.8452 - val_accuracy: 0.6890\n",
      "Epoch 19/20\n",
      "63/63 [==============================] - 1s 11ms/step - loss: 0.4641 - accuracy: 0.8630 - val_loss: 1.8689 - val_accuracy: 0.6860\n",
      "Epoch 20/20\n",
      "63/63 [==============================] - 1s 10ms/step - loss: 0.4388 - accuracy: 0.8730 - val_loss: 1.8915 - val_accuracy: 0.6820\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fed3d71e790>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = models.Sequential()\n",
    "model.add(layers.Dense(64, activation='relu', input_shape=(10000,)))\n",
    "model.add(layers.Dense(4, activation='relu'))\n",
    "model.add(layers.Dense(46, activation='softmax'))\n",
    "model.compile(optimizer='rmsprop',\n",
    "        loss='categorical_crossentropy',\n",
    "        metrics=['accuracy'])\n",
    "model.fit(partial_x_train,\n",
    "        partial_y_train,\n",
    "        epochs=20,\n",
    "        batch_size=128,\n",
    "        validation_data=(x_val, y_val))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "现在网络的验证精度最大约为 71%，比前面下降了 8%。导致这一下降的主要原因在于，你试图将大量信息（这些信息足够恢复 46个类别的分割超平面）压缩到维度很小的中间空间。网络能够将大部分必要信息塞入这个四维表示中，但并不是全部信息。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.5.8　Further experiments 进一步的实验"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/9\n",
      "18/18 [==============================] - 1s 18ms/step - loss: 3.2489 - accuracy: 0.3385\n",
      "Epoch 2/9\n",
      "18/18 [==============================] - 0s 17ms/step - loss: 1.5930 - accuracy: 0.6240\n",
      "Epoch 3/9\n",
      "18/18 [==============================] - 0s 16ms/step - loss: 1.1623 - accuracy: 0.7311\n",
      "Epoch 4/9\n",
      "18/18 [==============================] - 0s 17ms/step - loss: 0.9234 - accuracy: 0.7885\n",
      "Epoch 5/9\n",
      "18/18 [==============================] - 0s 16ms/step - loss: 0.7483 - accuracy: 0.8245\n",
      "Epoch 6/9\n",
      "18/18 [==============================] - 0s 17ms/step - loss: 0.5889 - accuracy: 0.8655\n",
      "Epoch 7/9\n",
      "18/18 [==============================] - 0s 16ms/step - loss: 0.4830 - accuracy: 0.8901\n",
      "Epoch 8/9\n",
      "18/18 [==============================] - 0s 17ms/step - loss: 0.4125 - accuracy: 0.9094\n",
      "Epoch 9/9\n",
      "18/18 [==============================] - 0s 16ms/step - loss: 0.3334 - accuracy: 0.9281\n",
      "71/71 [==============================] - 1s 1ms/step - loss: 1.1635 - accuracy: 0.7609\n"
     ]
    }
   ],
   "source": [
    "model = keras.Sequential([\n",
    "  layers.Dense(64, activation=\"relu\"),\n",
    "  layers.Dense(64, activation=\"relu\"),\n",
    "  layers.Dense(64, activation=\"relu\"),\n",
    "  layers.Dense(64, activation=\"relu\"),\n",
    "  layers.Dense(46, activation=\"softmax\")\n",
    "])\n",
    "model.compile(optimizer=\"rmsprop\",\n",
    "              loss=\"categorical_crossentropy\",\n",
    "              metrics=[\"accuracy\"])\n",
    "model.fit(x_train,\n",
    "          y_train,\n",
    "          epochs=9,\n",
    "          batch_size=512)\n",
    "results = model.evaluate(x_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/9\n",
      "18/18 [==============================] - 1s 44ms/step - loss: 2.7390 - accuracy: 0.3555\n",
      "Epoch 2/9\n",
      "18/18 [==============================] - 1s 42ms/step - loss: 1.2133 - accuracy: 0.7145\n",
      "Epoch 3/9\n",
      "18/18 [==============================] - 1s 42ms/step - loss: 0.8463 - accuracy: 0.7972\n",
      "Epoch 4/9\n",
      "18/18 [==============================] - 1s 47ms/step - loss: 0.6595 - accuracy: 0.8375\n",
      "Epoch 5/9\n",
      "18/18 [==============================] - 1s 44ms/step - loss: 0.3738 - accuracy: 0.9103\n",
      "Epoch 6/9\n",
      "18/18 [==============================] - 1s 43ms/step - loss: 0.2809 - accuracy: 0.9336\n",
      "Epoch 7/9\n",
      "18/18 [==============================] - 1s 43ms/step - loss: 0.1866 - accuracy: 0.9512\n",
      "Epoch 8/9\n",
      "18/18 [==============================] - 1s 42ms/step - loss: 0.1925 - accuracy: 0.9497\n",
      "Epoch 9/9\n",
      "18/18 [==============================] - 1s 42ms/step - loss: 0.1521 - accuracy: 0.9512\n",
      "71/71 [==============================] - 0s 3ms/step - loss: 1.1956 - accuracy: 0.7916\n"
     ]
    }
   ],
   "source": [
    "model = keras.Sequential([\n",
    "  layers.Dense(256, activation=\"relu\"),\n",
    "  layers.Dense(256, activation=\"relu\"),\n",
    "  layers.Dense(256, activation=\"relu\"),\n",
    "  layers.Dense(256, activation=\"relu\"),\n",
    "  layers.Dense(46, activation=\"softmax\")\n",
    "])\n",
    "model.compile(optimizer=\"rmsprop\",\n",
    "              loss=\"categorical_crossentropy\",\n",
    "              metrics=[\"accuracy\"])\n",
    "model.fit(x_train,\n",
    "          y_train,\n",
    "          epochs=9,\n",
    "          batch_size=512)\n",
    "results = model.evaluate(x_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}