data_aug_cifar10.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.14.0\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras\n",
    "from keras.datasets import cifar10\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "import matplotlib.pyplot as plt\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 32\n",
    "num_classes = 10\n",
    "epochs = 40\n",
    "data_augmentation = True\n",
    "num_predictions = 20\n",
    "save_dir = os.path.join(os.getcwd(), 'saved_models')\n",
    "model_name = 'keras_cifar10_trained_model.h5'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (50000, 32, 32, 3)\n",
      "50000 train samples\n",
      "10000 test samples\n",
      "uint8\n"
     ]
    }
   ],
   "source": [
    "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
    "print('x_train shape:', x_train.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')\n",
    "print(x_train.dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[6]\n",
      " [9]\n",
      " [9]\n",
      " [4]\n",
      " [1]\n",
      " [1]\n",
      " [2]\n",
      " [7]\n",
      " [8]\n",
      " [3]]\n",
      "(50000, 1)\n",
      "__this is after keras.utils.to_categorical__\n",
      "[[0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 1.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 1.]\n",
      " [0. 0. 0. 0. 1. 0. 0. 0. 0. 0.]\n",
      " [0. 1. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 1. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 1. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 1. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 1. 0.]\n",
      " [0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]]\n",
      "(50000, 10)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print(y_train[:10])\n",
    "print(y_train.shape)\n",
    "\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)\n",
    "\n",
    "print(\"__this is after keras.utils.to_categorical__\")\n",
    "print(y_train[:10])\n",
    "print(y_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def no_batch_norm():\n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(32, (3, 3), padding='same',\n",
    "                     input_shape=x_train.shape[1:]))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Conv2D(32, (3, 3)))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Dropout(0.25))\n",
    "\n",
    "    model.add(Conv2D(64, (3, 3), padding='same'))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Conv2D(64, (3, 3)))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Dropout(0.25))\n",
    "\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(512))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(num_classes))\n",
    "    model.add(Activation('softmax'))\n",
    "    \n",
    "    print(type(model))\n",
    "    model.summary()\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'keras.engine.sequential.Sequential'>\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_5 (Conv2D)            (None, 32, 32, 32)        896       \n",
      "_________________________________________________________________\n",
      "activation_7 (Activation)    (None, 32, 32, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_6 (Conv2D)            (None, 30, 30, 32)        9248      \n",
      "_________________________________________________________________\n",
      "activation_8 (Activation)    (None, 30, 30, 32)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2 (None, 15, 15, 32)        0         \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 15, 15, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_7 (Conv2D)            (None, 15, 15, 64)        18496     \n",
      "_________________________________________________________________\n",
      "activation_9 (Activation)    (None, 15, 15, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_8 (Conv2D)            (None, 13, 13, 64)        36928     \n",
      "_________________________________________________________________\n",
      "activation_10 (Activation)   (None, 13, 13, 64)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_4 (MaxPooling2 (None, 6, 6, 64)          0         \n",
      "_________________________________________________________________\n",
      "dropout_5 (Dropout)          (None, 6, 6, 64)          0         \n",
      "_________________________________________________________________\n",
      "flatten_2 (Flatten)          (None, 2304)              0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 512)               1180160   \n",
      "_________________________________________________________________\n",
      "activation_11 (Activation)   (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dropout_6 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 10)                5130      \n",
      "_________________________________________________________________\n",
      "activation_12 (Activation)   (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 1,250,858\n",
      "Trainable params: 1,250,858\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "myModel = no_batch_norm()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0729 05:31:17.147496 140582718240512 deprecation_wrapper.py:119] From /usr/local/lib/python3.5/dist-packages/keras/optimizers.py:790: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n",
      "W0729 05:31:17.154800 140582718240512 deprecation_wrapper.py:119] From /usr/local/lib/python3.5/dist-packages/keras/backend/tensorflow_backend.py:3295: The name tf.log is deprecated. Please use tf.math.log instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "opt = keras.optimizers.rmsprop(lr=0.0001, decay=1e-6)\n",
    "\n",
    "myModel.compile(loss='categorical_crossentropy',\n",
    "              optimizer=opt,\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "float32\n",
      "[[[0.23137255 0.24313726 0.24705882]\n",
      "  [0.16862746 0.18039216 0.1764706 ]\n",
      "  [0.19607843 0.1882353  0.16862746]\n",
      "  [0.26666668 0.21176471 0.16470589]\n",
      "  [0.38431373 0.28627452 0.20392157]]\n",
      "\n",
      " [[0.0627451  0.07843138 0.07843138]\n",
      "  [0.         0.         0.        ]\n",
      "  [0.07058824 0.03137255 0.        ]\n",
      "  [0.2        0.10588235 0.03137255]\n",
      "  [0.34509805 0.2        0.08235294]]\n",
      "\n",
      " [[0.09803922 0.09411765 0.08235294]\n",
      "  [0.0627451  0.02745098 0.        ]\n",
      "  [0.19215687 0.10588235 0.03137255]\n",
      "  [0.3254902  0.19607843 0.09019608]\n",
      "  [0.43137255 0.28235295 0.16078432]]\n",
      "\n",
      " [[0.12941177 0.09803922 0.06666667]\n",
      "  [0.14901961 0.07843138 0.01568628]\n",
      "  [0.34117648 0.21176471 0.09803922]\n",
      "  [0.41568628 0.24705882 0.10980392]\n",
      "  [0.4509804  0.27450982 0.12941177]]\n",
      "\n",
      " [[0.19607843 0.1254902  0.08235294]\n",
      "  [0.23137255 0.1254902  0.04313726]\n",
      "  [0.4        0.25490198 0.13333334]\n",
      "  [0.49803922 0.30980393 0.15294118]\n",
      "  [0.4862745  0.3019608  0.14117648]]]\n"
     ]
    }
   ],
   "source": [
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "print(x_train.dtype)\n",
    "print(x_train[0,:5,:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "datagen = ImageDataGenerator(\n",
    "        featurewise_center=False,  # set input mean to 0 over the dataset\n",
    "        samplewise_center=False,  # set each sample mean to 0\n",
    "        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n",
    "        samplewise_std_normalization=False,  # divide each input by its std\n",
    "        zca_whitening=False,  # apply ZCA whitening\n",
    "        zca_epsilon=1e-06,  # epsilon for ZCA whitening\n",
    "        rotation_range=20,  # randomly rotate images in the range (degrees, 0 to 180)\n",
    "        # randomly shift images horizontally (fraction of total width)\n",
    "        width_shift_range=0.1,\n",
    "        # randomly shift images vertically (fraction of total height)\n",
    "        height_shift_range=0.1,\n",
    "        shear_range=3,  # set range for random shear\n",
    "        zoom_range=0.,  # set range for random zoom\n",
    "        channel_shift_range=0.,  # set range for random channel shifts\n",
    "        # set mode for filling points outside the input boundaries\n",
    "        fill_mode='nearest',\n",
    "        cval=0.,  # value used for fill_mode = \"constant\"\n",
    "        horizontal_flip=True,  # randomly flip images\n",
    "        vertical_flip=False,  # randomly flip images\n",
    "        # set rescaling factor (applied before any other transformation)\n",
    "        rescale=None,\n",
    "        # set function that will be applied on each input\n",
    "        preprocessing_function=None,\n",
    "        # image data format, either \"channels_first\" or \"channels_last\"\n",
    "        data_format=None,\n",
    "        # fraction of images reserved for validation (strictly between 0 and 1)\n",
    "        validation_split=0.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "datagen.fit(x_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0729 05:31:34.208864 140582718240512 deprecation.py:323] From /usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/math_grad.py:1250: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "520/520 [==============================] - 36s 70ms/step - loss: 1.9371 - acc: 0.2890 - val_loss: 1.6826 - val_acc: 0.3937\n",
      "Epoch 2/30\n",
      "520/520 [==============================] - 30s 57ms/step - loss: 1.7102 - acc: 0.3748 - val_loss: 1.5487 - val_acc: 0.4316\n",
      "Epoch 3/30\n",
      "520/520 [==============================] - 29s 57ms/step - loss: 1.6205 - acc: 0.4088 - val_loss: 1.4685 - val_acc: 0.4645\n",
      "Epoch 4/30\n",
      "520/520 [==============================] - 30s 57ms/step - loss: 1.5492 - acc: 0.4378 - val_loss: 1.4061 - val_acc: 0.4968\n",
      "Epoch 5/30\n",
      "520/520 [==============================] - 29s 56ms/step - loss: 1.4925 - acc: 0.4590 - val_loss: 1.3385 - val_acc: 0.5197\n",
      "Epoch 6/30\n",
      "520/520 [==============================] - 29s 56ms/step - loss: 1.4512 - acc: 0.4755 - val_loss: 1.3203 - val_acc: 0.5270\n",
      "Epoch 7/30\n",
      "520/520 [==============================] - 29s 56ms/step - loss: 1.4078 - acc: 0.4946 - val_loss: 1.3440 - val_acc: 0.5275\n",
      "Epoch 8/30\n",
      "520/520 [==============================] - 28s 55ms/step - loss: 1.3777 - acc: 0.5054 - val_loss: 1.2334 - val_acc: 0.5608\n",
      "Epoch 9/30\n",
      "520/520 [==============================] - 28s 55ms/step - loss: 1.3489 - acc: 0.5189 - val_loss: 1.2554 - val_acc: 0.5526\n",
      "Epoch 10/30\n",
      "520/520 [==============================] - 29s 55ms/step - loss: 1.3173 - acc: 0.5280 - val_loss: 1.2075 - val_acc: 0.5721\n",
      "Epoch 11/30\n",
      "520/520 [==============================] - 28s 55ms/step - loss: 1.2922 - acc: 0.5373 - val_loss: 1.2240 - val_acc: 0.5662\n",
      "Epoch 12/30\n",
      "520/520 [==============================] - 28s 54ms/step - loss: 1.2715 - acc: 0.5461 - val_loss: 1.2106 - val_acc: 0.5714\n",
      "Epoch 13/30\n",
      "520/520 [==============================] - 28s 54ms/step - loss: 1.2439 - acc: 0.5573 - val_loss: 1.1431 - val_acc: 0.5946\n",
      "Epoch 14/30\n",
      "520/520 [==============================] - 28s 54ms/step - loss: 1.2200 - acc: 0.5660 - val_loss: 1.1083 - val_acc: 0.6075\n",
      "Epoch 15/30\n",
      "520/520 [==============================] - 29s 55ms/step - loss: 1.1941 - acc: 0.5748 - val_loss: 1.0505 - val_acc: 0.6276\n",
      "Epoch 16/30\n",
      "520/520 [==============================] - 28s 54ms/step - loss: 1.1808 - acc: 0.5804 - val_loss: 1.0828 - val_acc: 0.6152\n",
      "Epoch 17/30\n",
      "520/520 [==============================] - 28s 54ms/step - loss: 1.1661 - acc: 0.5886 - val_loss: 1.0229 - val_acc: 0.6389\n",
      "Epoch 18/30\n",
      "520/520 [==============================] - 28s 54ms/step - loss: 1.1498 - acc: 0.5927 - val_loss: 1.1393 - val_acc: 0.6047\n",
      "Epoch 19/30\n",
      "520/520 [==============================] - 28s 54ms/step - loss: 1.1319 - acc: 0.6003 - val_loss: 0.9992 - val_acc: 0.6425\n",
      "Epoch 20/30\n",
      "520/520 [==============================] - 28s 54ms/step - loss: 1.1157 - acc: 0.6039 - val_loss: 1.0185 - val_acc: 0.6362\n",
      "Epoch 21/30\n",
      "520/520 [==============================] - 28s 54ms/step - loss: 1.1056 - acc: 0.6063 - val_loss: 1.0542 - val_acc: 0.6300\n",
      "Epoch 22/30\n",
      "520/520 [==============================] - 28s 55ms/step - loss: 1.0845 - acc: 0.6164 - val_loss: 0.9752 - val_acc: 0.6555\n",
      "Epoch 23/30\n",
      "520/520 [==============================] - 28s 54ms/step - loss: 1.0732 - acc: 0.6190 - val_loss: 1.0509 - val_acc: 0.6332\n",
      "Epoch 24/30\n",
      "520/520 [==============================] - 28s 54ms/step - loss: 1.0693 - acc: 0.6209 - val_loss: 1.0530 - val_acc: 0.6342\n",
      "Epoch 25/30\n",
      "520/520 [==============================] - 28s 54ms/step - loss: 1.0502 - acc: 0.6292 - val_loss: 0.9688 - val_acc: 0.6596\n",
      "Epoch 26/30\n",
      "520/520 [==============================] - 28s 54ms/step - loss: 1.0426 - acc: 0.6311 - val_loss: 0.9259 - val_acc: 0.6751\n",
      "Epoch 27/30\n",
      "520/520 [==============================] - 28s 54ms/step - loss: 1.0365 - acc: 0.6353 - val_loss: 0.9101 - val_acc: 0.6822\n",
      "Epoch 28/30\n",
      "520/520 [==============================] - 28s 54ms/step - loss: 1.0040 - acc: 0.6458 - val_loss: 0.8978 - val_acc: 0.6804\n"
     ]
    }
   ],
   "source": [
    "## Using a data generator <May limit the GPU performance because GPU is too fast>\n",
    "## This version has data augmentation\n",
    "history = myModel.fit_generator(datagen.flow(x_train, y_train,\n",
    "                                 batch_size=batch_size*3),\n",
    "                                steps_per_epoch= x_train.shape[0] //(batch_size*3),\n",
    "                    epochs=30,\n",
    "                    validation_data=(x_test, y_test),\n",
    "                    workers=8)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.isdir(\"./models\"):\n",
    "    os.makedirs(\"./models\")\n",
    "myModel_save_path = \"./models/model_generator_aug.hdf5\"\n",
    "myModel.save(myModel_save_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 1s 70us/step\n",
      "Test loss: 0.8977753735542298\n",
      "Test accuracy: 0.6804\n"
     ]
    }
   ],
   "source": [
    "scores = myModel.evaluate(x_test, y_test, verbose=1)\n",
    "print('Test loss:', scores[0])\n",
    "print('Test accuracy:', scores[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xd8VGX2+PHPSe8dSCBAQu8gBFCKAotS7AoodkAR17quu7rf3f3puuuuuuuuBRsqgopYwIIFsaJSlCa9CEICgUAaSUgv8/z+uEOMmJBJMpPJZM779cqLyZ1bzt1x5+Q+5TxijEEppZQC8HF3AEoppVoOTQpKKaWqaVJQSilVTZOCUkqpapoUlFJKVdOkoJRSqpomBaUcICJJImJExM+BfW8QkVVNPY9S7qBJQbU6IpIqIuUiEnfK9h/sX8hJ7olMqZZPk4JqrQ4A00/+IiL9gRD3haOUZ9CkoFqrV4Hravx+PfBKzR1EJFJEXhGRLBFJE5G/iIiP/T1fEfmPiGSLyH7g/FqOfUlEMkTksIj8Q0R8GxqkiLQXkWUikisi+0TkphrvDRORDSJSICLHROS/9u1BIvKaiOSISJ6IrBeRdg29tlK10aSgWqvvgAgR6W3/sr4SeO2UfZ4CIoEuwDlYSWSG/b2bgAuAM4AUYMopxy4AKoFu9n3OA25sRJxvAOlAe/s1/iki4+zvPQE8YYyJALoCb9m3X2+PuyMQC8wBShpxbaV+RZOCas1OPi2cC+wCDp98o0ai+JMx5oQxJhV4DLjWvss04HFjzCFjTC7wrxrHtgMmA3cZY4qMMZnA/+znc5iIdARGAvcaY0qNMZuBF/n5CacC6CYiccaYQmPMdzW2xwLdjDFVxpiNxpiChlxbqbpoUlCt2avAVcANnNJ0BMQB/kBajW1pQAf76/bAoVPeO6mz/dgMe/NNHvA80LaB8bUHco0xJ+qIYRbQA9htbyK6oMZ9rQDeEJEjIvKoiPg38NpK1UqTgmq1jDFpWB3Ok4F3Tnk7G+sv7s41tnXi56eJDKzmmZrvnXQIKAPijDFR9p8IY0zfBoZ4BIgRkfDaYjDG7DXGTMdKNo8AS0Qk1BhTYYz5mzGmDzACq5nrOpRyAk0KqrWbBYwzxhTV3GiMqcJqo39IRMJFpDNwNz/3O7wF3CEiiSISDdxX49gM4FPgMRGJEBEfEekqIuc0JDBjzCFgDfAve+fxAHu8rwGIyDUi0sYYYwPy7IfZRGSsiPS3N4EVYCU3W0OurVRdNCmoVs0Y85MxZkMdb98OFAH7gVXA68B8+3svYDXRbAE28esnjeuAAGAncBxYAiQ0IsTpQBLWU8O7wP3GmM/t700EdohIIVan85XGmBIg3n69Aqy+kq+xmpSUajLRRXaUUkqdpE8KSimlqmlSUEopVU2TglJKqWqaFJRSSlXzuPK9cXFxJikpyd1hKKWUR9m4cWO2MaZNfft5XFJISkpiw4a6RhgqpZSqjYik1b+XNh8ppZSqQZOCUkqpapoUlFJKVfO4PoXaVFRUkJ6eTmlpqbtDaTZBQUEkJibi76/FMZVSztMqkkJ6ejrh4eEkJSUhIu4Ox+WMMeTk5JCenk5ycrK7w1FKtSKtovmotLSU2NhYr0gIACJCbGysVz0ZKaWaR6tICoDXJISTvO1+lVLNo9UkhfqUVlRxJK8Em02rwiqlVF28JimUV9rILiyjuLzK6efOyclh0KBBDBo0iPj4eDp06FD9e3l5uUPnmDFjBnv27HF6bEop1RCtoqPZESGBvgAUlVcSFuTc246NjWXz5s0APPDAA4SFhXHPPff8Yh9jDMYYfHxqz8Mvv/yyU2NSSqnG8JonBT8fH4L9fSkqq2y2a+7bt48+ffpw9dVX07dvXzIyMpg9ezYpKSn07duXBx98sHrfUaNGsXnzZiorK4mKiuK+++5j4MCBnHXWWWRmZjZbzEop79bqnhT+9sEOdh4pqPW98kobFTYboQENu+0+7SO4/8KGrslu2b17N6+88gopKSkAPPzww8TExFBZWcnYsWOZMmUKffr0+cUx+fn5nHPOOTz88MPcfffdzJ8/n/vuu6+20yullFN5zZMCgK+PgAFbMy5B2rVr1+qEALB48WIGDx7M4MGD2bVrFzt37vzVMcHBwUyaNAmAIUOGkJqa2lzhKqW8XKt7UjjdX/SVVTZ2ZhQQHxlE2/CgZoknNDS0+vXevXt54oknWLduHVFRUVxzzTW1zjUICAiofu3r60tlZfM1eSmlvJtXPSn4+foQ6OdLUZnzRyA5oqCggPDwcCIiIsjIyGDFihVuiUMpperS6p4U6hMW6EtecQXGmGafADZ48GD69OlDr1696Ny5MyNHjmzW6yulVH3ENGP7ujOkpKSYUxfZ2bVrF71793bo+Lzicg7mFtO9bRjBDexwbmkact9KKe8mIhuNMSn17edVzUcAIfZEUOimJiSllGrJvC4pBPj5EODn06zzFZRSylN4XVIACA3wo7i8Ek9rOlNKKVfzzqQQ6EelzVBWaXN3KEop1aJ4aVKw10HSJiSllPoFr0wKAb4++Ptqv4JSSp3KK5OCiBAa6EdReZVT+hWcUTobYP78+Rw9erTJ8SilVGN59kD9JggN8CWvuJzyShuB/r5NOpcjpbMdMX/+fAYPHkx8fHyT4lFKqcby3qQQaN16UXlVk5PC6SxcuJCnn36a8vJyRowYwdy5c7HZbMyYMYPNmzdjjGH27Nm0a9eOzZs3c8UVVxAcHMy6det+UQNJKaWaQ+tLCsvvg6Pb6t0tEEPX8iqrcqpfPUkhvj9MerjBoWzfvp13332XNWvW4Ofnx+zZs3njjTfo2rUr2dnZbNtmxZmXl0dUVBRPPfUUc+fOZdCgQQ2+llJKOUPrSwoOEgRfH3Hpms2ff/4569evry6dXVJSQseOHZkwYQJ79uzhjjvu4Pzzz+e8885zWQxKKdUQrS8pNOAv+sLCMo7kldArPoIAP+f3uRtjmDlzJn//+99/9d7WrVtZvnw5Tz/9NEuXLmXevHlOv75SSjWUV44+Oik04Od1m11h/PjxvPXWW2RnZwPWKKWDBw+SlZWFMYapU6fy4IMPsmnTJgDCw8M5ceKES2JRSilHtL4nhQYI8vfF10coKqskOsT5nbr9+/fn/vvvZ/z48dhsNvz9/Xnuuefw9fVl1qxZ1eW7H3nkEQBmzJjBjTfeqB3NSim3cVnpbBGZD1wAZBpj+tXyfiTwGtAJKzn9xxjzcn3nbWrp7FOlZhdRVmmjZ3x4o453Jy2drZRyVEsonb0AmHia928FdhpjBgJjgMdEpNn/NA4N9KWssoqKKq2DpJRSLksKxphvgNzT7QKEi7X8WZh932avOxFqX19BS14opZR7O5rnAr2BI8A24E5jTK1/rovIbBHZICIbsrKyaj1ZY5vBggJ88RGhqNyzFt3Rst9KKVdwZ1KYAGwG2gODgLkiElHbjsaYecaYFGNMSps2bX71flBQEDk5OY36ovQRISTA16OeFIwx5OTkEBQU5O5QlFKtjDtHH80AHjbWN/k+ETkA9ALWNfREiYmJpKenU9dTRH0KSis4UVJJRXYQPj7SqHM0t6CgIBITE90dhlKqlXFnUjgI/Ab4VkTaAT2B/Y05kb+/P8nJyY0O5Pv9Odz45ne8cF0K5/Zu1+jzKKWUp3NZ85GILAbWAj1FJF1EZonIHBGZY9/l78AIEdkGfAHca4zJdlU8pzOwYxQBfj6sO5DjjssrpVSL4bInBWPM9HrePwK0iKI/Qf6+DOoYxboDpxsspZRSrZ9Xl7moaXhyDNuPFFDoQR3OSinlbJoU7IYlx1BlM2xMO+7uUJRSym00KdgN7hSNr49ov4JSyqtpUrALDfSjf4dI7VdQSnk1TQo1DE+OYcuhfEorPGt2s1JKOYsmhRqGJcdQXmXjh4N57g5FKaXcQpNCDSlJMYigTUhKKa+lSaGGyGB/esdHsC5VO5uVUt5Jk8IphiXHsDHtOOWVur6CUsr7aFI4xfDkGEorbGw7nO/uUJRSqtlpUjjF0OQYQPsVlFLeSZPCKeLCAukVH86SjYe0CUkp5XU0KdTijxN78lNWES+ualQlb6WU8liaFGoxrlc7JvaN58kv9nIot9jd4SilVLPRpFCH/3dhH3xEuH/ZDl0PWSnlNbwrKWTvdXjX9lHB3H1uD77cncmKHcdcGJRSSrUc3pMUflgEc1Mgc5fDh9wwIole8eH87YMdFOk6C0opL+A9SaHHBPANgA3zHT7Ez9eHhy7tT0Z+KY9//qMLg1NKqZbBe5JCaBz0vRQ2L4ayQocPG9I5munDOjJ/dSq7MgpcGKBSSrmf9yQFgJRZUH4Ctr3doMPundiLyGB//vzuNmw27XRWSrVe3pUUOg6Ddv1h/YvQgBFFUSEB/N/k3mw6mMebGw65MECllHIv70oKIjB0FhzbDofWNejQywd3YHhyDA8v301OYZmLAlRKKffyrqQA0H8qBITDhpcadJiI8I9L+lFUVsk/P97touCUUsq9vC8pBIbBoOmw410oym7Qod3bhTP77C4s3ZTOd/t1zQWlVOvjfUkBIGUmVJXDD681+NDbx3UnMTqYv7y3XQvmKaVaHe9MCm17Q+dR1pwFW1WDDg0O8OXBi/uyL7OQF77VgnlKqdbFO5MCwNCZkJcG+75o8KHjerVjQt92PPWlFsxTSrUu3psUel0IoW0b3OF80v0X9tWCeUqpVsd7k4JfAAy+Dn5cAcfTGnx4zYJ5b6zXuQtKqdbBZUlBROaLSKaIbD/NPmNEZLOI7BCRr10VS52G3GDNXdi4oFGHzxiZzOjucdy/bAfb0nVNZ6WU53Plk8ICYGJdb4pIFPAMcJExpi8w1YWx1C6qI/SYBJtegcqGT0jz9RGeuPIM4kIDuGXRRvKKy10QpFJKNR+XJQVjzDdA7ml2uQp4xxhz0L5/pqtiOa2hM6E4G3Z90KjDY0IDePrqwRwrKOV3b27W2khKKY/mzj6FHkC0iKwUkY0icl1dO4rIbBHZICIbsrKynBtFl3EQnWzVQ2qkMzpF8/8u6MNXe7J4ZuU+JwanlFLNy51JwQ8YApwPTAD+KiI9atvRGDPPGJNijElp06aNc6Pw8bEmsx1cC8d2NPo015zZmUsGteexz37k271OTlxKKdVM3JkU0oEVxpgiY0w28A0w0C2RnHEN+AbC+sYNTwWrNtI/L+tP97Zh3PnGZo7klTgxQKWUah7uTArvA6NExE9EQoDhgONrZTpTSAz0uwy2vgllJxp/mgA/nr1mCOWVNn67aJOWwVBKeRxXDkldDKwFeopIuojMEpE5IjIHwBizC/gE2AqsA140xtQ5fNXlht4I5YVWYmiCrm3CeHTKADYfyuOhj3Y6KTillGoefq46sTFmugP7/Bv4t6tiaJAOQyBhoNWElDLLmr/QSJP7J3DjqGReXHWAwZ2juXhQBycGqpRSruO9M5pPJWIlg8ydVqdzE907qRdDk6K5b+k2fjzW+CYppZRqTpoUauo/BQIjm9ThfJK/rw9zrxpMaKAfc17bSGFZpRMCVEop19KkUFNAqLUAz873obDpc+naRQQx96ozSMsp5t4lW7VwnlKqxdOkcKqUWWCrgI0LnXK6M7vE8ocJPfloWwbPf6PrLyilWjZNCqdq0wN6TIRV/4Wcn5xyypvP7sL5AxJ4ePluXv2u4RVZlVKquWhSqM35/wVff3h3ToNXZquNiPC/aYMY37stf31vO2+uP+iEIJVSyvk0KdQmsgNMfgzS18HqJ5xyygA/H56+ejBn92jDfe9s451N6U45r1JKOZMmhbr0nwJ9LoGv/glHtznllIF+vsy7dghndYnlnre38MGWI045r1JKOYsmhbqIWM1IITHwzs2NWm+hNkH+vrx4fQopnWO4683NfLI9wynnVUopZ9CkcDqhsXDRU5C5w3picJKQAD/mzxjKwMRIbl/8A5/vPOa0cyulVFNoUqhPjwkw+HqrbyGt6TOdTwoL9GPBzGH0SYjgt4s2sXKPe9YYUkqpmjQpOGLCQxDVCd6b06QqqqeKCPLnlZnD6dY2jJtf3cjqfdlOO7dSSjWGJgVHBIbDpc/B8TT49C9OPXVkiD+v3TicpNhQbly4ge/35zj1/Eop1RCaFBzVeQSMuB02LoAfP3XqqWNCA1h003DaRwUxc8F6NqYdd+r5lVLKUZoUGmLcX6BtH1h2GxTnOvXUcWGBLL7pTNpGBHH9/HV8uuOoU8+vlFKO0KTQEH6BcOnzVkL48Hfg5AJ3bSOCeP2m4STHhTL71Y08+MFOXb1NKdWsNCk0VMIAGPsn2PkebF/q/NNHBrPklrO4YUQS81cfYOpzaziUW+z06yilVG00KTTGiDshcRh8dDcUOH9WcqCfLw9c1JfnrhnM/uwiJj/5LZ9s1+YkpZTraVJoDF8/azRSVQW8fyvYXNPEM7FfAh/dPprkuFDmvLaRB5btoKyy6QX6lFKqLpoUGiu2K5z3D/jpS/jiby67TKfYEN6ecxYzRiaxYE0qU55dy8EcbU5SSrmGJoWmSJlp/ax+HNa94LLLBPr5cv+FfXn+2iGk5RRx/pPfsnyb1kxSSjmfQ0lBRLqKSKD99RgRuUNEolwbmgcQgcn/gZ6T4eM/wK4PXXq5CX3j+eiO0XRpG8YtizZx//vbtTlJKeVUjj4pLAWqRKQbMA/oCLzusqg8iY8vXP4SdBgCS2fBoXUuvVzHmBDevvksZo1KZuHaNKY+t5bDeSUuvaZSyns4mhRsxphK4FLgKWPMH4AE14XlYQJC4Ko3IaI9vH4FZO9z7eX8fPjrBX14/toh7M8q4sKnVmndJKWUUziaFCpEZDpwPXCyjcTfNSF5qNA4uHoJiA+8dhkUur7q6YS+8bx/20hiQwO49qXvee7rnzBOnlCnlPIujiaFGcBZwEPGmAMikgy86rqwPFRsV7jqLSshvD4NygpdfsmubcJ479aRTOqXwMPLd/PbRZsoLKt0+XWVUq2TQ0nBGLPTGHOHMWaxiEQD4caYR1wcm2dKHAJTX4aMLbBkBlS5/gs6NNCPuVedwZ8n9+bTnce4eO4q9mW6PiEppVofR0cfrRSRCBGJATYBL4jIf10bmgfrOQnOfwz2fmrNem6GJh0R4aazu/DqrGHkFVdwydOrdalPpVSDOdp8FGmMKQAuA14xxgwHxrsurFYgZSaM/j1sWgjf/KfZLjuiaxwf3D6Krm3DmPPaJh75ZDdVNu1nUEo5xtGk4CciCcA0fu5oPi0RmS8imSKyvZ79hopIpYhMcTAWzzHurzBwOnz1D/hhUbNdtn1UMG/dfCZXDe/Esyt/4vr568gtKm+26yulPJejSeFBYAXwkzFmvYh0AfbWc8wCYOLpdhARX+ARwLmr1rQUInDhk9BlDHxwB+z7otkuHejnyz8v7c+jlw9gXWouFzz5LR9uPaKjk5RSp+VoR/PbxpgBxphb7L/vN8ZcXs8x3wD1rURzO9bEuNa7ar1fAEx7Fdr0grdnQHZ9udS5pg3tyJI5ZxER7M9tr//AJU+vZu1PuuSnUqp2jnY0J4rIu/bmoEwRWSoiiU25sIh0wJoM96wD+84WkQ0isiErK6spl3WPoAiYvtiqrrr4SijJa9bLD0iM4qM7RvOfqQPJPFHG9Be+Y+aC9fx47ESzxqGUavkcbT56GVgGtLf/fGDf1hSPA/caY+qtO22MmWeMSTHGpLRp06aJl3WTqE7WE8PxVFh6I9iat2aRr48wZUgiX90zhvsm9WJ9ai4TH/+GPy7ZwtH80maNRSnVcjmaFNoYY142xlTafxYATf12TgHeEJFUYArwjIhc0sRztmxJI2HSo7DvM5eW2z6dIH9f5pzTlW/+MJYZI5N574cjjPnPVzz6yW4KSivcEpNSquVwNCnkiMg1IuJr/7kGaFLDtDEm2RiTZIxJApYAvzXGvNeUc3qEobPs5bafgK1vuS2M6NAA/npBH774/TlM6BvPMyt/4pxHv2L+qgO6LrRSXszRpDATazjqUSAD6y/7G053gIgsBtYCPUUkXURmicgcEZnThHhbh4mPQOeRsOx2OLzJraF0jAnhiSvP4IPbRtE7IYIHP9zJuMdWsnRjus5vUMoLSWOHKIrIXcaYx50cT71SUlLMhg0bmvuyzleUDfPGWH0Ls7+C8Hh3R4Qxhm/2ZvPvFbvZfriAbm3DuOe8HkzoG4+IuDs8pVQTiMhGY0xKffs1ZeW1u5twrAqNgytfh9I8ePNaqCxzd0SICOf0aMOyW0fxzNWDMcYw57VNXDR3Nd/8mKVzHJTyAk1JCvqnY1MlDIBLnoH0dfBh89RIcoSPjzC5fwIr7jqbf08ZQG5ROdfNX8eV875jQ2p9U0+UUp6sKUmhZXyDebq+l8LZf4DNr8H3z7s7ml/w8/VhakpHvrznHP52UV9+yipiynNrmblgPTuO5Ls7PKWUC5y2T0FETlD7l78AwcYYP1cFVpdW06dQk80Gb14DP34C175jlcVogYrLK1mwJpXnVv5EQWklFwxI4N6JvegYE+Lu0JRS9XC0T6HRHc3u0iqTAkDZCXjxXDiRYXU8x3RxznkrSmDb29YopwkPQUBok0+ZX1LBC9/s58VV+7EZmDUqmVvHdiMssNn/RlBKOUiTgifK3Q8vjIPQNjD2z9D93MZ/iecdgg0vwcYFUHLc2nbu32HkHU4LNyO/hH9/sod3fjhMXFggf5jQgylDOuLro91NSrU0mhQ81YFvYMlMKMoC/xArMfS5GLpPgMCw0x9rDBxcC98/B7s+BAz0Oh+Gz7HWdDi6De7a6pSnhZo2H8rjwQ92sOlgHn3bR/DXC/pwZpdYp15DKdU0mhQ8ma0K0lbDzvdh5zIoygS/IOg2HvpcAj0mWEX2Tqoohe1LrGRwdBsERcGQ62HojVbNJYCD38H8CXDeP2DE7U4P2RjDB1szePjjXRzJL2VSv3j+NKk3nWK1v0GplkCTQmthq7K+0He+D7uWWX0OvoHQ7TfQ+0LI2Wc1ERXnQNs+MPxm6D8NAmr5Mn7lYji2A+7c4vSnhZNKK6p44Zv9PLPyJ6pshpmjkrl1bFfCg/xdcj2llGM0KbRGNps1p2HHe1aSOHEEEHsT0c2QNNpa2KcuaWvh5Ykue1qo6Wh+Kf9esYelm9KJCwvgtrHdmDa0IyEB2hmtlDtoUmjtbDY4ugVCYn9uInJE9dPC1tqfJpxsy6E8HvpoF+tSc4kK8efaMztz3VlJtAkPdPm1lVI/06Sgalf9tPAQjLit2S67ITWX57/Zz+e7juHv68PlgxO5aXQyXdrU03mulHIKTQqqbgsvgsydzfa0UNNPWYW8+O1+lm46TEWVjXN7t+Pmc7owpHNMs8ahlLdpjoJ4ylONuc8a8rphfrNfumubMP512QBW3zuO28Z24/sDuVz+7Fouf3YNK3YcxablupVyK31S8FYLL4LMXfaRSO4bNlpcXslb6w/x4qoDpB8vISk2hGlDO3LZGYnERwa5LS6lWhttPlKnl7YGXp4EE/4JZ93q7miorLKxfPtRXlmbyvrU4/gIjO7ehilDEjm3TzuC/H3dHaJSHk2Tgqrfwgshc7fbnxZOdSC7iHc2pbN0YzpH8kuJCPLjwoHtmZrSkYGJkbrgj1KNoElB1S91NSyY3GKeFk5lsxnW7s/h7Q2H+GTHUUorbHRrG8aUIYlcekYH2kVo85JSjtKkoByz8ELI2mM9LfgHuzuaOhWUVvDx1gyWbExnQ5rVvDSxXzx3/KY7veIj6j+BUl5Ok4JyTPXTwr/grN+6OxqH7M8q5K0N6bz2XRqFZZVMsieH3gmaHJSqiyYF5bgFF0D2jy3+aeFUecXlzF91gJdXp3KirJKJfa3k0Ke9JgelTqXzFJTjxtwHhcdgw8uO7Z+5C5bfay0KlLHVtbGdRlRIAHef15NV947jjt90Z/W+bCY/+S1zXt3IziMFbotLKU+mTwrKUt/TQkWJVYhv48tw6HvwDYCAMKgqh2kLrbLebpZfXMFLqw/w8qoDnCirZELfdtzxm+70bR/p7tCUcjttPlINk7oKFpwPEx+GM2/5efuxnVZp7q1vQGk+xHaDITfAwKugqgwWTbNKZlzwX2t7C5BfXMH81QeYb08O5/Vpx42juzA0KVqHsyqvpUlBNdzJp4Xffgc/fmIlg5NPBb0vsr70k0b9sjx32Ql4+wbY9zmMuhvG/RV8Wkar5MnksGBNKvklFfRtH8GMkclcODCBQD+dDKe8iyYF1XAHvoWFF4CPP9gqILa7/algOoSeZnnNqkr4+PdWEuk3BS55BvxaTmns4vJK3v3hMAtWp7I3s5C4sACuGtaJa87sTFud66C8hCYF1Tgf3g1lBVYy6Dzy9Iv21GQMrH4cPn8AOo2AKxdBSMuqfGqMYfW+HF5efYAv92Ti5yOc3z+BG0YmM6hjlLvDU8qlNCko99i+FN6dA1Gd4eq3ISa5/mOKc2HfF/DjcmtkU8Ig6DzC+onp4nhiaoDU7CIWrk3l7Q3pFJZVckanKGaMTGZSv3j8fVtG85dSzqRJQblP2lp4YzqIL1z1JiSe8t+hMZC910oCP66w1qA2VRDaBhIGwpHNUJxt7Rue8HOC6DwS4no6tc/iRGkFSzams3BNKqk5xXSICubG0clcoUuHqlbG7UlBROYDFwCZxph+tbx/NXAvIMAJ4BZjzJb6zqtJwUNk74VFU+DEUbj8Reg+AQ6usZLAnuVw/IC1X7v+0HMi9JgI7QdbX/jGWB3eaautGddpq+FEhrV/cMzPCaL/VAhr45RwbTbDV3syef7r/axLzSU6xJ8bRiRz/YjORIUEOOUaSrlTS0gKZwOFwCt1JIURwC5jzHERmQQ8YIwZXt95NSl4kKJsWHwlpG+AwHCrr8I3ELqcAz0mWIkgMrH+8xgDx1Otct9p9iRxPBXiesCNn0OQc+chbEjN5dmVP/HF7kxCAny5algnZo1OJiHSc2Z7K3UqtycFexBJwIe1JYVT9osGthtjOtR3Tk0KHqaiBL54EMoLrSTQZQwEhDb9vPtXwmuXQ9dxMP0N8HH+ENPdRwt4/uv9LNtyBB+BS8/owM3ndKWrriutPJCnJYVE0vi0AAAZe0lEQVR7gF7GmBvreH82MBugU6dOQ9LS0pwcqfJI61+Cj+6GkXfBuX9z2WUO5Rbz4rf7eWP9IcqrbEzoE89NZydzRsdofHx0MpzyDB6TFERkLPAMMMoYk1PfOfVJQf3Ch7+z1pq+7AUYMM2ll8ouLGPhmlQWrkmloLSSiCA/hibFMDQ5hmHJMfTvEKkjl1SL5WhScOvwChEZALwITHIkISj1KxMfgawfYdntENsVOgxx2aXiwgL5/Xk9ufmcrny64yjrU3P5/kAuX+zOBCDY35fBnaMYmmQliTM6RhMcoDOnlWdx25OCiHQCvgSuM8ascfSc+qSgfqUoG+aNtWZhz14J4fHNevmsE2WsT81l3QHrZ9fRAowBf19hQGIUZ3aJYVS3NgzuHKXlNZTbuL35SEQWA2OAOOAYcD/gD2CMeU5EXgQuB052EFQ6ErAmBVWro9vhpfOgbW+44SPwd1/5ivySCjalHef7A7l8fyCHren5VNkMQf4+DEuOZVS3WEZ1a0Ov+HDtk1DNxu1JwVU0Kag67VwGb11rVXC95BmXzIRujBOlFXy/P5dV+7JZtS+bfZmFAMSGBjCiWxyjusUyslscidEhbo5UtWYe0aeglFP1uQjG/AlW/gvi+8FZt7o7IgDCg/wZ36cd4/u0A+Bofimr92Wz2p4kPthyBIDkuFDG9mzLuF5tGZYcQ4Cfdlqr5qdPCqp1sdng7eth94dW7aUWsPjP6Rhj2JtZyKq92Xz9YxZr9+dQXmkjLNCP0d3jGNerLWN7tSUurOVUnVWeSZuPlPcqK4T5EyD/ENz4JcR1c3dEDisur2T1vhy+3J3Jl7uPcaygDBEYmBjFuF7WU0Tf9hG6WJBqME0KyrsdT4MXxlq1km76wumlMJqDMYYdRwr4cncmX+zOZGt6HsZAfEQQ4/u0ZVK/BIYnx+CncyOUAzQpKJW6Gl65yCqtMXUhBHp2eYqsE2Ws3JPJF7sy+frHLEoqqogO8ee8PvFM6h/PiK5x2g+h6qRJQSmwVoP74E6I7ATnPwY9znN3RE234s+UdTiTr0hh+fajfLErk8Iya4b1+D7tmNQvgdHd4wjy1zkR6meaFJQ6KW0tfHgXZO2Gvpdas6DD27k7qsZJWwMvT7KWSr1tPYhQWlHF6n3ZfLztKJ/tPEpBaSWhAb6M692OSf3iGd09jvAgf3dHrtxMk4JSNVWWw+on4JtHwT8Yxv8NBl/v1AV7msVrU2DfZ9brGcuttSVqKK+0sXZ/Dp9sz2DFjmPkFpXj5yMMTYphbK82jOvVlq5twrSj2gtpUlCqNtn7rKeG1G+h01lw4RPQpqe7o3JMxhZ4/mwYfQ+smwc9J8Nlz9e5e2WVjY1px/lqTxYr92Sy++gJABKjgxnbsy1je7XhrC5xWp/JS2hSUKouxsDmRfDpX6zhq6PvhlF3u7U0hkPevgH2fg6/2w5f/A02vw6/3w3B0Q4dfiSvhJV7svhydyar92VTUlFFoJ8PZ3WNZWzPtgxNiqFb2zDtrG6lNCkoVZ/CLFjxf7DtLauN/sLHIWmUu6OqXfY+mJsCo+6C8Q9Y61jPOwcm/RuGz27w6coqq1h3IJevdmfx1Z5MDmQXAVYRv25tw+mdEE6fhAj6JETQOyGC6FBdktTTaVJQylH7PocP74a8NKtpZtxfWkzdpGrv3wbb3oa7tkFYW2vb82eDrQrmrGpyvGk5RWxJz2dXRgE7jxSwK6OAzBNl1e/HRwRZiaJ9BAMSozinRxsd3eRhtPaRUo7qNh5++x0s/yN8+x/wC4Rz/ujuqH6Wfxi2vAFDbvg5IYDVUf7R3XB4EyQ2bR2JzrGhdI4N5aKB7au3ZReWsSujwP5zgl0ZBXy7N5tKmyEy2J+LB7VnWkpHnWHdymhSUAogIAQufNL6y/urh8A/BEbc5u6oLGufBmODEbf/cnv/qVa/yKaFTU4KtYkLC2R09zaM7t6meltZZRXrDxzn7Y2HeGP9IV5Zm0av+HCmpXTkkjM6EKPNTB5Pm4+UqqmqEpbOgp3vwQX/g5SZ7o2nKAce7we9L6p9pNF7t1qx/n5Ps8/Yzi+uYNnWI7y94RBb0/Px9xXG927HtJSOjO4ep+U3WhhtPlKqMXz9rPWeK0qsfgb/EBh4ZePPZ0zT2vvXPQ8VxTDqd7W/P+R62PwabF9qvW5GkSH+XHtmZ649szO7jxbw9oZ03v3hMMu3H6VdRCCXDU5kfO+29EmI1GGvHkSfFJSqTUUpvD4VUlfB1AXQ5+KGHW8MbFtiDR1NPhsumtvwiXJlJ+B/fSFpNFy5qO7rPHMmBITCTV827PwuUF5p48vdmby94RBf7cnEZsBHoEe7cPp3iGRAxygGdIikV0K4Lk3azPRJQamm8A+CKxfDa5fBkllwZbDjdZMytsDHf4RD31k1lzYvsr60Jz3asKeGDS9Dab41h6IuIlaH84o/WUuSxv9qOfRmFeDnw8R+8UzsF092YRk/HMxjW3oeW9Lz+WJ3Jm9vTAesoa8948Pp3yGKgYmR9EqIIDY0gMgQf8ID/bTj2o30SUGp0ynJsyqtZu2xFu1JPrvufYuy4cu/w8aFEBIDv7kfzrgGPvt/sHYujPk/GHOvY9etKIUnBlqzra9fdvp9i3PhsZ7W6KTJ/3b41pqbMYbDeSVsS89n6+F8tqbnsTU9nxOllb/Yz9dHiAr2JzLEn+iQgOrXUcEBRIf4c0anaM7qGouvrm/dIPqkoJQzBEfBNe/CgvPh9Svhuveg47Bf7lNVCRteskYtlRXCmbfAOfdaxwKc9w8oOQ4r/2kli2E31X/dLa9D4dHTlrGoFhJjdURvfRPOfdCq7dQCiQiJ0SEkRocwqX8CYCWKtJxi9mYWkldcTl5xBXklJ/+tIK+4nKMFpew+eoK84nKKyqsAa2TU+f3juWhQewZ3itYnCyfSJwWlHHHiqFWdtCgHbvgAEgZa2/d/Dcvvhaxd1roNEx+Btr1+fXxVJbx1LexZDpe/CP2n1H2tqkqYOwRCYuHGLxxrcjrwDSy8EC59vmkd4y1cSXkVK/dksmzLEb7YnUl5pY0OUcFcOLA9Fw1sT++EcE0QddAZzUo5W94hKzFUFMNl86xmol3LIKoTTPgX9Dr/9F/gFSVWldND38H0N6F7HetHb1tiDYu9YhH0vsCx2IyBpwZDWDzMXN7we3NU+karmOCI28HHyR3FNhtUllr/O1UU1/5vVZk12TAokhOlFXy64xgfbD3Ct3uzqbIZurYJ5aKBHbhoUHuS40KdG5+H06SglCvk/GQlhsJj1nDVUXdbk9wcbbIpzYcFF0DOPrju/V83RRkDz44EUwW3rG3YiKVV/4PPH4Bb10ObHo4f54jyIvjyIfjuGcDAuL/C2fc459ybF8PHf4DyE47tnzgMZn7yi6SUW1TOx9syWLblCOtTczEGerYLp1vbMDrFhtA5JsT6NzaU+Iggr+yP0KSglKtk7bEqlA67CSITG358YRbMnwDFOdaaCO36/Pzenk9g8RWNawYqzIT/9rb6NM77R8Pjqsv+lbDsDqs2VMosKMqCPR9bTVvtBzXt3Md2Wmtpt+trPQH4B1vJ9hf/1nh98HtY/geY+LB1n7XIyC/ho60ZfLs3m4O5xaQfL6ai6ufvuQBfHxKjg2ski1ASo4NJiAwiITKY2NAAfFph0tCkoFRLdjzNSgzGwKwVEJ1kvX7pPKv/4o5N4NuI1dLevMZaae7uXeDXxJITJcetMho/vAYxXeGipyBppDXa6dmREBgON3/d+I7tihKYNxaKs2HOasdWwzMGXp9mzR+5ZQ3EJNd7SJXNcCSvhIO5xaTlFJOWW8TBHOv1wdxiCst+OfopwNeHdpGBJEQEkxAVRHxkEO0jg4mPDKJzbAg92oZ7ZNLQpKBUS5e5C+ZPtEYPzVwB2T9ao5wm/8exEUq12fs5LLrcmnDX99LGx7brA/jo99Yw2xG3w5j7fvnl/9NX8OolMOxmmPxo467x4e9gw3y4Zqn1lOCo/MPWhL2EgXD9B02aMW6MIbeonIz8Uo7klXC0oJQjeaUczS/hSH4pR+0/5VW26mMig/0ZlhzD8OQYzuwSS++ECI9ojtIhqUq1dG17W3MfXrnYmiQXGAmhbay5DY3VdSxEdrQ6wRuTFE4cs5pndr4P8f3hqrdqbyLqOhaG3wLfPws9JkC33zTsOjvftxLCiNsblhAAIjvAeX+HD+6EjQsgZUbDjq9BRIgNCyQ2LJB+HSJr3cdmM+QWl5ORV8q+rBN8vz+X7/bn8NnOYwBEBPkxLDmWM7t4VpKoiz4pKOVu+z635kDYKqwJb6NPM4PZESsfseZE3LnFapZyhDGwZTF88ierWWfMvTDijtM3YVWUwLwx1gS/3661nngckXcInhtpNUnNXNG4Zi5jrEmFh3+AW79rXN9OE2Xkl1QniO/255CaUwxAeJAfw5Nj6BUfQWJ0MB2ig0mMDqF9VJBbS3to85FSnmTn+9Zf91NfhqDa/2J1WH46PN7fGhn1m7+eft/SfNj1odVvcHANdDzT6jtwdPRSxhZ44TfQazJMXVh/U05VpdVEdmwHzPkGYro4dp3a5B6AZ0dYq+Vd9ZbbF0aqmSTWHcglLbeYKtvP368i0DY80D6BL9hKGFEhJMWG0C8xkoigRvQhNYDbk4KIzAcuADKNMb8qyCLWDJMngMlAMXCDMWZTfefVpKCUAxZNhaPb4K7tVuXXmipK4MdPrPkQez+zxv5HdbaaclJmNbxw37f/tQr/OTJi6suH4JtHrUq0A6Y17Dq1+e45+OTeFjlpr7LKxtGCUtKPl5B+vITDx0tIP15s/Z5XzJG80uqkIQJd24QxqGNU9U+v+HCnlh9vCUnhbKAQeKWOpDAZuB0rKQwHnjDGDK/vvJoUlHLArg/hzauton69JkNVhTW0dNsS2P2RNScgtC30u8xarKfDkMb/pW2rsuZeHNsOt6y2JvPV5sC31qzrgdPh0mcbfWu/vLbNmjeStRtuXefYCKYWorLKxrETZfyUWcjmQ3nVP7lF5QAE+fvQv0OkPUlEM6hTFO0jgxo9Y9vtScEeRBLwYR1J4XlgpTFmsf33PcAYY0zG6c6pSUEpB1RVWGW3o5OtOQA737PmRQRGQp+LrDIbSaOdNyv5eJo1TDVhgDUi6NTzFuVY/QgBoTD7a+cuCJS917p2j/Pgitecd143MMaQfryEHw7lsflgHpsPHWf7kQLKK63RTzeOSuYvF/Sp5yy184TRRx2AQzV+T7dv+1VSEJHZwGyATp3q+CtEKfUzX39rFNO3j1nt/j0nWYmg23hrDWpni+5sDU197xZY8xSMuuvn94yB92+1ktJVbzp/hbi47jD2T9Zs7h3vQd9LnHv+ZiQidIwJoWNMSPV62eWVNnYfLWDzoTx6tgt3eQweMSTVGDMPmAfWk4Kbw1HKM4y+x+o47jyieZbqHDjdKvj35T+g6zjrqQFg3Tz4cbk1C/lkIUFnO+t2KyF8fI/1BBQa65rruEGAnw8DEqMYkBjVLNdz5yKqh4GONX5PtG9TSjlDQIjVpNJcazeLwAWPW0NT35ltrQlxdJs1K7r7BBg+x3XX9vWDS56xhsd+cp/rruMF3JkUlgHXieVMIL++/gSlVAsXGgsXP2OVEl/xJ1gyE4JjrC9sVw8ZbdfXKtK37S2rhpRqFJc1H4nIYmAMECci6cD9gD+AMeY54GOskUf7sIakNn5aolKq5eg+HobeBOtfAMRaOS40rnmuPepu2LkMPrwLOn3380JHymEuSwrGmOn1vG+AW111faWUG537oFXLqdv40y9h6mx+AXDxXHjxN/Dpn+Hip5vv2q2ER3Q0K6U8TEBI/WtLu0qHwTDyTmt9CbAKDLbQJUpbIk0KSqnWZ9xfwcffmj2dsRWmveJQmW3l3o5mpZRyDR9fGPdnqyZSXhrMOwd+/NTdUXkETQpKqdarxwRrBnVUJ3h9Knz1T6ssh7tUVVoFCw9+D9uXwuonYd0L1vYWQpuPlFKtW0wyzPrMWjTo60cgfQNc/qLjpb4bI22N1WxVkG4tCpSfDgWHrVX1TC1J6cDXcPlLrplt3kCaFJRSrZ9/sDUSKXEoLP8jPH8OTFtodUo7U85P1uS5vfamKt9Aa1GgiA7WKKyIDvbfE61/IxPhh0XWnI5FU+HKRdYyp26k6ykopbzL4Y3w1vVQeMwamTTk+qafs6wQvvk3rH0a/IKsRYoGXGnNz3Bk0t7mxVZ9qISBcPUSl5TpcLQgnvYpKKW8S4chVj9D0ij44A7ry7issHHnMga2vg1zU2D141YZ8ts3WGtThLVxfBb3oOlWhddjO6xS4Pnuq/ijTwpKKe9kq4KVD1vDVn0DIWkkdDsXup8HsV3r/0LP2AIf/xEOfQcJg2Dyv6HjsKbFlLoKFk+3Vt+79l2rAqyTtIj1FFxBk4JSyqnSN8COd61V6LL3WNuik6zk0O1c64kiIOTn/Ytz4cu/w8YFVl2n8ffDoGsavmJdXTK2wKuXWa+vWQrtBznltJoUlFKqoY6nwb7PYO/n1oigimKrjyBplJUgRGDlv6C0AIbdBGP+5Jr6Stn74NVLrKqv0xdD8ugmn1KTglJKNUVFKRxcYz1B7P0McvZa25NGw6RHoV3jVkBzWP5heO0yyD0AUxdYy6o2gSYFpZRyptwDUJRlDWt1dRnwk4pzYdEUOLLZGlI76LR1Rk9LRx8ppZQzxSRbHcnNlRDAmmB33TKr+ei9OfD98y6/pCYFpZRqyQLDrBpO/aZATBeXX05nNCulVEvnFwhTXmqWS+mTglJKqWqaFJRSSlXTpKCUUqqaJgWllFLVNCkopZSqpklBKaVUNU0KSimlqmlSUEopVc3jah+JSBaQ1sjD44BsJ4bTErS2e2pt9wOt755a2/1A67un2u6nszGmTX0HelxSaAoR2eBIQShP0truqbXdD7S+e2pt9wOt756acj/afKSUUqqaJgWllFLVvC0pzHN3AC7Q2u6ptd0PtL57am33A63vnhp9P17Vp6CUUur0vO1JQSml1GloUlBKKVXNa5KCiEwUkT0isk9E7nN3PM4gIqkisk1ENouIxy1cLSLzRSRTRLbX2BYjIp+JyF77v9HujLGh6rinB0TksP1z2iwiTVuBvRmJSEcR+UpEdorIDhG5077dIz+n09yPJ39GQSKyTkS22O/pb/btySLyvf07700RCXDofN7QpyAivsCPwLlAOrAemG6M2enWwJpIRFKBFGOMR066EZGzgULgFWNMP/u2R4FcY8zD9uQdbYy5151xNkQd9/QAUGiM+Y87Y2sMEUkAEowxm0QkHNgIXALcgAd+Tqe5n2l47mckQKgxplBE/IFVwJ3A3cA7xpg3ROQ5YIsx5tn6zuctTwrDgH3GmP3GmHLgDeBiN8fk9Ywx3wC5p2y+GFhof70Q6/+wHqOOe/JYxpgMY8wm++sTwC6gAx76OZ3mfjyWsRTaf/W3/xhgHLDEvt3hz8hbkkIH4FCN39Px8P8Q7AzwqYhsFJHZ7g7GSdoZYzLsr48C7dwZjBPdJiJb7c1LHtHUcioRSQLOAL6nFXxOp9wPePBnJCK+IrIZyAQ+A34C8owxlfZdHP7O85ak0FqNMsYMBiYBt9qbLloNY7Vttob2zWeBrsAgIAN4zL3hNJyIhAFLgbuMMQU13/PEz6mW+/Hoz8gYU2WMGQQkYrWM9GrsubwlKRwGOtb4PdG+zaMZYw7b/80E3sX6j8HTHbO3+55s/810czxNZow5Zv8/rQ14AQ/7nOzt1EuBRcaYd+ybPfZzqu1+PP0zOskYkwd8BZwFRImIn/0th7/zvCUprAe623vjA4ArgWVujqlJRCTU3lGGiIQC5wHbT3+UR1gGXG9/fT3wvhtjcYqTX552l+JBn5O9E/MlYJcx5r813vLIz6mu+/Hwz6iNiETZXwdjDajZhZUcpth3c/gz8orRRwD2IWaPA77AfGPMQ24OqUlEpAvW0wGAH/C6p92TiCwGxmCV+T0G3A+8B7wFdMIqkT7NGOMxHbd13NMYrGYJA6QCN9doj2/RRGQU8C2wDbDZN/8fVju8x31Op7mf6XjuZzQAqyPZF+sP/beMMQ/avyPeAGKAH4BrjDFl9Z7PW5KCUkqp+nlL85FSSikHaFJQSilVTZOCUkqpapoUlFJKVdOkoJRSqpomBaVOISJVNaplbnZmVV0RSapZQVWplsav/l2U8jol9pIBSnkdfVJQykH29Sseta9hsU5Eutm3J4nIl/Zial+ISCf79nYi8q69zv0WERlhP5WviLxgr33/qX0WqlItgiYFpX4t+JTmoytqvJdvjOkPzMWaIQ/wFLDQGDMAWAQ8ad/+JPC1MWYgMBjYYd/eHXjaGNMXyAMud/H9KOUwndGs1ClEpNAYE1bL9lRgnDFmv72o2lFjTKyIZGMt3FJh355hjIkTkSwgsWZpAXu55s+MMd3tv98L+Btj/uH6O1OqfvqkoFTDmDpeN0TN+jNVaN+eakE0KSjVMFfU+Het/fUarMq7AFdjFVwD+AK4BaoXQYlsriCVaiz9C0WpXwu2r2J10ifGmJPDUqNFZCvWX/vT7dtuB14WkT8AWcAM+/Y7gXkiMgvrieAWrAVclGqxtE9BKQfZ+xRSjDHZ7o5FKVfR5iOllFLV9ElBKaVUNX1SUEopVU2TglJKqWqaFJRSSlXTpKCUUqqaJgWllFLV/j9x1Mexy5SjoAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot training & validation loss values\n",
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "plt.title('Model loss')\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Train', 'Test'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'keras.engine.sequential.Sequential'>\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_9 (Conv2D)            (None, 32, 32, 32)        896       \n",
      "_________________________________________________________________\n",
      "activation_13 (Activation)   (None, 32, 32, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_10 (Conv2D)           (None, 30, 30, 32)        9248      \n",
      "_________________________________________________________________\n",
      "activation_14 (Activation)   (None, 30, 30, 32)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_5 (MaxPooling2 (None, 15, 15, 32)        0         \n",
      "_________________________________________________________________\n",
      "dropout_7 (Dropout)          (None, 15, 15, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_11 (Conv2D)           (None, 15, 15, 64)        18496     \n",
      "_________________________________________________________________\n",
      "activation_15 (Activation)   (None, 15, 15, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_12 (Conv2D)           (None, 13, 13, 64)        36928     \n",
      "_________________________________________________________________\n",
      "activation_16 (Activation)   (None, 13, 13, 64)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_6 (MaxPooling2 (None, 6, 6, 64)          0         \n",
      "_________________________________________________________________\n",
      "dropout_8 (Dropout)          (None, 6, 6, 64)          0         \n",
      "_________________________________________________________________\n",
      "flatten_3 (Flatten)          (None, 2304)              0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 512)               1180160   \n",
      "_________________________________________________________________\n",
      "activation_17 (Activation)   (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dropout_9 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 10)                5130      \n",
      "_________________________________________________________________\n",
      "activation_18 (Activation)   (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 1,250,858\n",
      "Trainable params: 1,250,858\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "#REDEFINE and re-compile the model here, and run for the same 30 epochs.\n",
    "myModel = no_batch_norm()\n",
    "opt = keras.optimizers.rmsprop(lr=0.0001, decay=1e-6)\n",
    "\n",
    "myModel.compile(loss='categorical_crossentropy',\n",
    "              optimizer=opt,\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 50000 samples, validate on 10000 samples\n",
      "Epoch 1/30\n",
      "50000/50000 [==============================] - 5s 110us/step - loss: 1.9218 - acc: 0.2976 - val_loss: 1.7824 - val_acc: 0.3638\n",
      "Epoch 2/30\n",
      "50000/50000 [==============================] - 5s 98us/step - loss: 1.6560 - acc: 0.3975 - val_loss: 1.5358 - val_acc: 0.4484\n",
      "Epoch 3/30\n",
      "50000/50000 [==============================] - 5s 98us/step - loss: 1.5502 - acc: 0.4396 - val_loss: 1.4280 - val_acc: 0.4889\n",
      "Epoch 4/30\n",
      "50000/50000 [==============================] - 5s 98us/step - loss: 1.4647 - acc: 0.4713 - val_loss: 1.3764 - val_acc: 0.5034\n",
      "Epoch 5/30\n",
      "50000/50000 [==============================] - 5s 97us/step - loss: 1.3953 - acc: 0.4973 - val_loss: 1.3056 - val_acc: 0.5400\n",
      "Epoch 6/30\n",
      "50000/50000 [==============================] - 5s 98us/step - loss: 1.3389 - acc: 0.5188 - val_loss: 1.2516 - val_acc: 0.5559\n",
      "Epoch 7/30\n",
      "50000/50000 [==============================] - 5s 98us/step - loss: 1.2897 - acc: 0.5406 - val_loss: 1.2307 - val_acc: 0.5718\n",
      "Epoch 8/30\n",
      "50000/50000 [==============================] - 5s 97us/step - loss: 1.2474 - acc: 0.5560 - val_loss: 1.1590 - val_acc: 0.5886\n",
      "Epoch 9/30\n",
      "50000/50000 [==============================] - 5s 98us/step - loss: 1.2093 - acc: 0.5696 - val_loss: 1.1201 - val_acc: 0.6023\n",
      "Epoch 10/30\n",
      "50000/50000 [==============================] - 5s 98us/step - loss: 1.1741 - acc: 0.5831 - val_loss: 1.1262 - val_acc: 0.6022\n",
      "Epoch 11/30\n",
      "50000/50000 [==============================] - 5s 98us/step - loss: 1.1380 - acc: 0.5981 - val_loss: 1.0678 - val_acc: 0.6196\n",
      "Epoch 12/30\n",
      "50000/50000 [==============================] - 5s 98us/step - loss: 1.1049 - acc: 0.6094 - val_loss: 1.1070 - val_acc: 0.6080\n",
      "Epoch 13/30\n",
      "50000/50000 [==============================] - 5s 98us/step - loss: 1.0773 - acc: 0.6207 - val_loss: 1.0060 - val_acc: 0.6449\n",
      "Epoch 14/30\n",
      "50000/50000 [==============================] - 5s 97us/step - loss: 1.0488 - acc: 0.6328 - val_loss: 0.9986 - val_acc: 0.6496\n",
      "Epoch 15/30\n",
      "50000/50000 [==============================] - 5s 97us/step - loss: 1.0228 - acc: 0.6395 - val_loss: 0.9566 - val_acc: 0.6649\n",
      "Epoch 16/30\n",
      "50000/50000 [==============================] - 5s 97us/step - loss: 1.0022 - acc: 0.6482 - val_loss: 0.9430 - val_acc: 0.6738\n",
      "Epoch 17/30\n",
      "50000/50000 [==============================] - 5s 97us/step - loss: 0.9788 - acc: 0.6570 - val_loss: 0.9720 - val_acc: 0.6601\n",
      "Epoch 18/30\n",
      "50000/50000 [==============================] - 5s 97us/step - loss: 0.9593 - acc: 0.6642 - val_loss: 0.9163 - val_acc: 0.6802\n",
      "Epoch 19/30\n",
      "50000/50000 [==============================] - 5s 98us/step - loss: 0.9321 - acc: 0.6725 - val_loss: 0.8852 - val_acc: 0.6923\n",
      "Epoch 20/30\n",
      "50000/50000 [==============================] - 5s 98us/step - loss: 0.9210 - acc: 0.6774 - val_loss: 0.8916 - val_acc: 0.6861\n",
      "Epoch 21/30\n",
      "50000/50000 [==============================] - 5s 98us/step - loss: 0.9026 - acc: 0.6839 - val_loss: 0.8676 - val_acc: 0.6980\n",
      "Epoch 22/30\n",
      "50000/50000 [==============================] - 5s 96us/step - loss: 0.8883 - acc: 0.6887 - val_loss: 0.8637 - val_acc: 0.6988\n",
      "Epoch 23/30\n",
      "50000/50000 [==============================] - 5s 98us/step - loss: 0.8699 - acc: 0.6956 - val_loss: 0.8391 - val_acc: 0.7077\n",
      "Epoch 24/30\n",
      "50000/50000 [==============================] - 5s 98us/step - loss: 0.8548 - acc: 0.7001 - val_loss: 0.8254 - val_acc: 0.7116\n",
      "Epoch 25/30\n",
      "50000/50000 [==============================] - 5s 98us/step - loss: 0.8396 - acc: 0.7056 - val_loss: 0.8105 - val_acc: 0.7193\n",
      "Epoch 26/30\n",
      "50000/50000 [==============================] - 5s 98us/step - loss: 0.8253 - acc: 0.7117 - val_loss: 0.7993 - val_acc: 0.7231\n",
      "Epoch 27/30\n",
      "50000/50000 [==============================] - 5s 101us/step - loss: 0.8112 - acc: 0.7183 - val_loss: 0.8059 - val_acc: 0.7201\n",
      "Epoch 28/30\n",
      "50000/50000 [==============================] - 5s 102us/step - loss: 0.7994 - acc: 0.7219 - val_loss: 0.8012 - val_acc: 0.7244\n",
      "Epoch 29/30\n",
      "50000/50000 [==============================] - 5s 97us/step - loss: 0.7856 - acc: 0.7276 - val_loss: 0.8068 - val_acc: 0.7219\n",
      "Epoch 30/30\n",
      "50000/50000 [==============================] - 5s 97us/step - loss: 0.7758 - acc: 0.7306 - val_loss: 0.7695 - val_acc: 0.7348\n"
     ]
    }
   ],
   "source": [
    "## Direcly using \n",
    "history_no_generator = myModel.fit(x_train, y_train,\n",
    "                                 batch_size=batch_size*3,\n",
    "                    epochs=30,\n",
    "                    validation_data=(x_test, y_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.isdir(\"./models\"):\n",
    "    os.makedirs(\"./models\")\n",
    "myModel_save_path = \"./models/model_without_generator.hdf5\"\n",
    "myModel.save(myModel_save_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 1s 66us/step\n",
      "Test loss: 0.769478184890747\n",
      "Test accuracy: 0.7348\n"
     ]
    }
   ],
   "source": [
    "scores = myModel.evaluate(x_test, y_test, verbose=1)\n",
    "print('Test loss:', scores[0])\n",
    "print('Test accuracy:', scores[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xd8VGX2+PHPSe8dSCBAQu8gBFCKAotS7AoodkAR17quu7rf3f3puuuuuuuuBRsqgopYwIIFsaJSlCa9CEICgUAaSUgv8/z+uEOMmJBJMpPJZM779cqLyZ1bzt1x5+Q+5TxijEEppZQC8HF3AEoppVoOTQpKKaWqaVJQSilVTZOCUkqpapoUlFJKVdOkoJRSqpomBaUcICJJImJExM+BfW8QkVVNPY9S7qBJQbU6IpIqIuUiEnfK9h/sX8hJ7olMqZZPk4JqrQ4A00/+IiL9gRD3haOUZ9CkoFqrV4Hravx+PfBKzR1EJFJEXhGRLBFJE5G/iIiP/T1fEfmPiGSLyH7g/FqOfUlEMkTksIj8Q0R8GxqkiLQXkWUikisi+0TkphrvDRORDSJSICLHROS/9u1BIvKaiOSISJ6IrBeRdg29tlK10aSgWqvvgAgR6W3/sr4SeO2UfZ4CIoEuwDlYSWSG/b2bgAuAM4AUYMopxy4AKoFu9n3OA25sRJxvAOlAe/s1/iki4+zvPQE8YYyJALoCb9m3X2+PuyMQC8wBShpxbaV+RZOCas1OPi2cC+wCDp98o0ai+JMx5oQxJhV4DLjWvss04HFjzCFjTC7wrxrHtgMmA3cZY4qMMZnA/+znc5iIdARGAvcaY0qNMZuBF/n5CacC6CYiccaYQmPMdzW2xwLdjDFVxpiNxpiChlxbqbpoUlCt2avAVcANnNJ0BMQB/kBajW1pQAf76/bAoVPeO6mz/dgMe/NNHvA80LaB8bUHco0xJ+qIYRbQA9htbyK6oMZ9rQDeEJEjIvKoiPg38NpK1UqTgmq1jDFpWB3Ok4F3Tnk7G+sv7s41tnXi56eJDKzmmZrvnXQIKAPijDFR9p8IY0zfBoZ4BIgRkfDaYjDG7DXGTMdKNo8AS0Qk1BhTYYz5mzGmDzACq5nrOpRyAk0KqrWbBYwzxhTV3GiMqcJqo39IRMJFpDNwNz/3O7wF3CEiiSISDdxX49gM4FPgMRGJEBEfEekqIuc0JDBjzCFgDfAve+fxAHu8rwGIyDUi0sYYYwPy7IfZRGSsiPS3N4EVYCU3W0OurVRdNCmoVs0Y85MxZkMdb98OFAH7gVXA68B8+3svYDXRbAE28esnjeuAAGAncBxYAiQ0IsTpQBLWU8O7wP3GmM/t700EdohIIVan85XGmBIg3n69Aqy+kq+xmpSUajLRRXaUUkqdpE8KSimlqmlSUEopVU2TglJKqWqaFJRSSlXzuPK9cXFxJikpyd1hKKWUR9m4cWO2MaZNfft5XFJISkpiw4a6RhgqpZSqjYik1b+XNh8ppZSqQZOCUkqpapoUlFJKVfO4PoXaVFRUkJ6eTmlpqbtDaTZBQUEkJibi76/FMZVSztMqkkJ6ejrh4eEkJSUhIu4Ox+WMMeTk5JCenk5ycrK7w1FKtSKtovmotLSU2NhYr0gIACJCbGysVz0ZKaWaR6tICoDXJISTvO1+lVLNo9UkhfqUVlRxJK8Em02rwiqlVF28JimUV9rILiyjuLzK6efOyclh0KBBDBo0iPj4eDp06FD9e3l5uUPnmDFjBnv27HF6bEop1RCtoqPZESGBvgAUlVcSFuTc246NjWXz5s0APPDAA4SFhXHPPff8Yh9jDMYYfHxqz8Mvv/yyU2NSSqnG8JonBT8fH4L9fSkqq2y2a+7bt48+ffpw9dVX07dvXzIyMpg9ezYpKSn07duXBx98sHrfUaNGsXnzZiorK4mKiuK+++5j4MCBnHXWWWRmZjZbzEop79bqnhT+9sEOdh4pqPW98kobFTYboQENu+0+7SO4/8KGrslu2b17N6+88gopKSkAPPzww8TExFBZWcnYsWOZMmUKffr0+cUx+fn5nHPOOTz88MPcfffdzJ8/n/vuu6+20yullFN5zZMCgK+PgAFbMy5B2rVr1+qEALB48WIGDx7M4MGD2bVrFzt37vzVMcHBwUyaNAmAIUOGkJqa2lzhKqW8XKt7UjjdX/SVVTZ2ZhQQHxlE2/CgZoknNDS0+vXevXt54oknWLduHVFRUVxzzTW1zjUICAiofu3r60tlZfM1eSmlvJtXPSn4+foQ6OdLUZnzRyA5oqCggPDwcCIiIsjIyGDFihVuiUMpperS6p4U6hMW6EtecQXGmGafADZ48GD69OlDr1696Ny5MyNHjmzW6yulVH3ENGP7ujOkpKSYUxfZ2bVrF71793bo+Lzicg7mFtO9bRjBDexwbmkact9KKe8mIhuNMSn17edVzUcAIfZEUOimJiSllGrJvC4pBPj5EODn06zzFZRSylN4XVIACA3wo7i8Ek9rOlNKKVfzzqQQ6EelzVBWaXN3KEop1aJ4aVKw10HSJiSllPoFr0wKAb4++Ptqv4JSSp3KK5OCiBAa6EdReZVT+hWcUTobYP78+Rw9erTJ8SilVGN59kD9JggN8CWvuJzyShuB/r5NOpcjpbMdMX/+fAYPHkx8fHyT4lFKqcby3qQQaN16UXlVk5PC6SxcuJCnn36a8vJyRowYwdy5c7HZbMyYMYPNmzdjjGH27Nm0a9eOzZs3c8UVVxAcHMy6det+UQNJKaWaQ+tLCsvvg6Pb6t0tEEPX8iqrcqpfPUkhvj9MerjBoWzfvp13332XNWvW4Ofnx+zZs3njjTfo2rUr2dnZbNtmxZmXl0dUVBRPPfUUc+fOZdCgQQ2+llJKOUPrSwoOEgRfH3Hpms2ff/4569evry6dXVJSQseOHZkwYQJ79uzhjjvu4Pzzz+e8885zWQxKKdUQrS8pNOAv+sLCMo7kldArPoIAP+f3uRtjmDlzJn//+99/9d7WrVtZvnw5Tz/9NEuXLmXevHlOv75SSjWUV44+Oik04Od1m11h/PjxvPXWW2RnZwPWKKWDBw+SlZWFMYapU6fy4IMPsmnTJgDCw8M5ceKES2JRSilHtL4nhQYI8vfF10coKqskOsT5nbr9+/fn/vvvZ/z48dhsNvz9/Xnuuefw9fVl1qxZ1eW7H3nkEQBmzJjBjTfeqB3NSim3cVnpbBGZD1wAZBpj+tXyfiTwGtAJKzn9xxjzcn3nbWrp7FOlZhdRVmmjZ3x4o453Jy2drZRyVEsonb0AmHia928FdhpjBgJjgMdEpNn/NA4N9KWssoqKKq2DpJRSLksKxphvgNzT7QKEi7X8WZh932avOxFqX19BS14opZR7O5rnAr2BI8A24E5jTK1/rovIbBHZICIbsrKyaj1ZY5vBggJ88RGhqNyzFt3Rst9KKVdwZ1KYAGwG2gODgLkiElHbjsaYecaYFGNMSps2bX71flBQEDk5OY36ovQRISTA16OeFIwx5OTkEBQU5O5QlFKtjDtHH80AHjbWN/k+ETkA9ALWNfREiYmJpKenU9dTRH0KSis4UVJJRXYQPj7SqHM0t6CgIBITE90dhlKqlXFnUjgI/Ab4VkTaAT2B/Y05kb+/P8nJyY0O5Pv9Odz45ne8cF0K5/Zu1+jzKKWUp3NZ85GILAbWAj1FJF1EZonIHBGZY9/l78AIEdkGfAHca4zJdlU8pzOwYxQBfj6sO5DjjssrpVSL4bInBWPM9HrePwK0iKI/Qf6+DOoYxboDpxsspZRSrZ9Xl7moaXhyDNuPFFDoQR3OSinlbJoU7IYlx1BlM2xMO+7uUJRSym00KdgN7hSNr49ov4JSyqtpUrALDfSjf4dI7VdQSnk1TQo1DE+OYcuhfEorPGt2s1JKOYsmhRqGJcdQXmXjh4N57g5FKaXcQpNCDSlJMYigTUhKKa+lSaGGyGB/esdHsC5VO5uVUt5Jk8IphiXHsDHtOOWVur6CUsr7aFI4xfDkGEorbGw7nO/uUJRSqtlpUjjF0OQYQPsVlFLeSZPCKeLCAukVH86SjYe0CUkp5XU0KdTijxN78lNWES+ualQlb6WU8liaFGoxrlc7JvaN58kv9nIot9jd4SilVLPRpFCH/3dhH3xEuH/ZDl0PWSnlNbwrKWTvdXjX9lHB3H1uD77cncmKHcdcGJRSSrUc3pMUflgEc1Mgc5fDh9wwIole8eH87YMdFOk6C0opL+A9SaHHBPANgA3zHT7Ez9eHhy7tT0Z+KY9//qMLg1NKqZbBe5JCaBz0vRQ2L4ayQocPG9I5munDOjJ/dSq7MgpcGKBSSrmf9yQFgJRZUH4Ctr3doMPundiLyGB//vzuNmw27XRWSrVe3pUUOg6Ddv1h/YvQgBFFUSEB/N/k3mw6mMebGw65MECllHIv70oKIjB0FhzbDofWNejQywd3YHhyDA8v301OYZmLAlRKKffyrqQA0H8qBITDhpcadJiI8I9L+lFUVsk/P97touCUUsq9vC8pBIbBoOmw410oym7Qod3bhTP77C4s3ZTOd/t1zQWlVOvjfUkBIGUmVJXDD681+NDbx3UnMTqYv7y3XQvmKaVaHe9MCm17Q+dR1pwFW1WDDg0O8OXBi/uyL7OQF77VgnlKqdbFO5MCwNCZkJcG+75o8KHjerVjQt92PPWlFsxTSrUu3psUel0IoW0b3OF80v0X9tWCeUqpVsd7k4JfAAy+Dn5cAcfTGnx4zYJ5b6zXuQtKqdbBZUlBROaLSKaIbD/NPmNEZLOI7BCRr10VS52G3GDNXdi4oFGHzxiZzOjucdy/bAfb0nVNZ6WU53Plk8ICYGJdb4pIFPAMcJExpi8w1YWx1C6qI/SYBJtegcqGT0jz9RGeuPIM4kIDuGXRRvKKy10QpFJKNR+XJQVjzDdA7ml2uQp4xxhz0L5/pqtiOa2hM6E4G3Z90KjDY0IDePrqwRwrKOV3b27W2khKKY/mzj6FHkC0iKwUkY0icl1dO4rIbBHZICIbsrKynBtFl3EQnWzVQ2qkMzpF8/8u6MNXe7J4ZuU+JwanlFLNy51JwQ8YApwPTAD+KiI9atvRGDPPGJNijElp06aNc6Pw8bEmsx1cC8d2NPo015zZmUsGteexz37k271OTlxKKdVM3JkU0oEVxpgiY0w28A0w0C2RnHEN+AbC+sYNTwWrNtI/L+tP97Zh3PnGZo7klTgxQKWUah7uTArvA6NExE9EQoDhgONrZTpTSAz0uwy2vgllJxp/mgA/nr1mCOWVNn67aJOWwVBKeRxXDkldDKwFeopIuojMEpE5IjIHwBizC/gE2AqsA140xtQ5fNXlht4I5YVWYmiCrm3CeHTKADYfyuOhj3Y6KTillGoefq46sTFmugP7/Bv4t6tiaJAOQyBhoNWElDLLmr/QSJP7J3DjqGReXHWAwZ2juXhQBycGqpRSruO9M5pPJWIlg8ydVqdzE907qRdDk6K5b+k2fjzW+CYppZRqTpoUauo/BQIjm9ThfJK/rw9zrxpMaKAfc17bSGFZpRMCVEop19KkUFNAqLUAz873obDpc+naRQQx96ozSMsp5t4lW7VwnlKqxdOkcKqUWWCrgI0LnXK6M7vE8ocJPfloWwbPf6PrLyilWjZNCqdq0wN6TIRV/4Wcn5xyypvP7sL5AxJ4ePluXv2u4RVZlVKquWhSqM35/wVff3h3ToNXZquNiPC/aYMY37stf31vO2+uP+iEIJVSyvk0KdQmsgNMfgzS18HqJ5xyygA/H56+ejBn92jDfe9s451N6U45r1JKOZMmhbr0nwJ9LoGv/glHtznllIF+vsy7dghndYnlnre38MGWI045r1JKOYsmhbqIWM1IITHwzs2NWm+hNkH+vrx4fQopnWO4683NfLI9wynnVUopZ9CkcDqhsXDRU5C5w3picJKQAD/mzxjKwMRIbl/8A5/vPOa0cyulVFNoUqhPjwkw+HqrbyGt6TOdTwoL9GPBzGH0SYjgt4s2sXKPe9YYUkqpmjQpOGLCQxDVCd6b06QqqqeKCPLnlZnD6dY2jJtf3cjqfdlOO7dSSjWGJgVHBIbDpc/B8TT49C9OPXVkiD+v3TicpNhQbly4ge/35zj1/Eop1RCaFBzVeQSMuB02LoAfP3XqqWNCA1h003DaRwUxc8F6NqYdd+r5lVLKUZoUGmLcX6BtH1h2GxTnOvXUcWGBLL7pTNpGBHH9/HV8uuOoU8+vlFKO0KTQEH6BcOnzVkL48Hfg5AJ3bSOCeP2m4STHhTL71Y08+MFOXb1NKdWsNCk0VMIAGPsn2PkebF/q/NNHBrPklrO4YUQS81cfYOpzaziUW+z06yilVG00KTTGiDshcRh8dDcUOH9WcqCfLw9c1JfnrhnM/uwiJj/5LZ9s1+YkpZTraVJoDF8/azRSVQW8fyvYXNPEM7FfAh/dPprkuFDmvLaRB5btoKyy6QX6lFKqLpoUGiu2K5z3D/jpS/jiby67TKfYEN6ecxYzRiaxYE0qU55dy8EcbU5SSrmGJoWmSJlp/ax+HNa94LLLBPr5cv+FfXn+2iGk5RRx/pPfsnyb1kxSSjmfQ0lBRLqKSKD99RgRuUNEolwbmgcQgcn/gZ6T4eM/wK4PXXq5CX3j+eiO0XRpG8YtizZx//vbtTlJKeVUjj4pLAWqRKQbMA/oCLzusqg8iY8vXP4SdBgCS2fBoXUuvVzHmBDevvksZo1KZuHaNKY+t5bDeSUuvaZSyns4mhRsxphK4FLgKWPMH4AE14XlYQJC4Ko3IaI9vH4FZO9z7eX8fPjrBX14/toh7M8q4sKnVmndJKWUUziaFCpEZDpwPXCyjcTfNSF5qNA4uHoJiA+8dhkUur7q6YS+8bx/20hiQwO49qXvee7rnzBOnlCnlPIujiaFGcBZwEPGmAMikgy86rqwPFRsV7jqLSshvD4NygpdfsmubcJ479aRTOqXwMPLd/PbRZsoLKt0+XWVUq2TQ0nBGLPTGHOHMWaxiEQD4caYR1wcm2dKHAJTX4aMLbBkBlS5/gs6NNCPuVedwZ8n9+bTnce4eO4q9mW6PiEppVofR0cfrRSRCBGJATYBL4jIf10bmgfrOQnOfwz2fmrNem6GJh0R4aazu/DqrGHkFVdwydOrdalPpVSDOdp8FGmMKQAuA14xxgwHxrsurFYgZSaM/j1sWgjf/KfZLjuiaxwf3D6Krm3DmPPaJh75ZDdVNu1nUEo5xtGk4CciCcA0fu5oPi0RmS8imSKyvZ79hopIpYhMcTAWzzHurzBwOnz1D/hhUbNdtn1UMG/dfCZXDe/Esyt/4vr568gtKm+26yulPJejSeFBYAXwkzFmvYh0AfbWc8wCYOLpdhARX+ARwLmr1rQUInDhk9BlDHxwB+z7otkuHejnyz8v7c+jlw9gXWouFzz5LR9uPaKjk5RSp+VoR/PbxpgBxphb7L/vN8ZcXs8x3wD1rURzO9bEuNa7ar1fAEx7Fdr0grdnQHZ9udS5pg3tyJI5ZxER7M9tr//AJU+vZu1PuuSnUqp2jnY0J4rIu/bmoEwRWSoiiU25sIh0wJoM96wD+84WkQ0isiErK6spl3WPoAiYvtiqrrr4SijJa9bLD0iM4qM7RvOfqQPJPFHG9Be+Y+aC9fx47ESzxqGUavkcbT56GVgGtLf/fGDf1hSPA/caY+qtO22MmWeMSTHGpLRp06aJl3WTqE7WE8PxVFh6I9iat2aRr48wZUgiX90zhvsm9WJ9ai4TH/+GPy7ZwtH80maNRSnVcjmaFNoYY142xlTafxYATf12TgHeEJFUYArwjIhc0sRztmxJI2HSo7DvM5eW2z6dIH9f5pzTlW/+MJYZI5N574cjjPnPVzz6yW4KSivcEpNSquVwNCnkiMg1IuJr/7kGaFLDtDEm2RiTZIxJApYAvzXGvNeUc3qEobPs5bafgK1vuS2M6NAA/npBH774/TlM6BvPMyt/4pxHv2L+qgO6LrRSXszRpDATazjqUSAD6y/7G053gIgsBtYCPUUkXURmicgcEZnThHhbh4mPQOeRsOx2OLzJraF0jAnhiSvP4IPbRtE7IYIHP9zJuMdWsnRjus5vUMoLSWOHKIrIXcaYx50cT71SUlLMhg0bmvuyzleUDfPGWH0Ls7+C8Hh3R4Qxhm/2ZvPvFbvZfriAbm3DuOe8HkzoG4+IuDs8pVQTiMhGY0xKffs1ZeW1u5twrAqNgytfh9I8ePNaqCxzd0SICOf0aMOyW0fxzNWDMcYw57VNXDR3Nd/8mKVzHJTyAk1JCvqnY1MlDIBLnoH0dfBh89RIcoSPjzC5fwIr7jqbf08ZQG5ROdfNX8eV875jQ2p9U0+UUp6sKUmhZXyDebq+l8LZf4DNr8H3z7s7ml/w8/VhakpHvrznHP52UV9+yipiynNrmblgPTuO5Ls7PKWUC5y2T0FETlD7l78AwcYYP1cFVpdW06dQk80Gb14DP34C175jlcVogYrLK1mwJpXnVv5EQWklFwxI4N6JvegYE+Lu0JRS9XC0T6HRHc3u0iqTAkDZCXjxXDiRYXU8x3RxznkrSmDb29YopwkPQUBok0+ZX1LBC9/s58VV+7EZmDUqmVvHdiMssNn/RlBKOUiTgifK3Q8vjIPQNjD2z9D93MZ/iecdgg0vwcYFUHLc2nbu32HkHU4LNyO/hH9/sod3fjhMXFggf5jQgylDOuLro91NSrU0mhQ81YFvYMlMKMoC/xArMfS5GLpPgMCw0x9rDBxcC98/B7s+BAz0Oh+Gz7HWdDi6De7a6pSnhZo2H8rjwQ92sOlgHn3bR/DXC/pwZpdYp15DKdU0mhQ8ma0K0lbDzvdh5zIoygS/IOg2HvpcAj0mWEX2Tqoohe1LrGRwdBsERcGQ62HojVbNJYCD38H8CXDeP2DE7U4P2RjDB1szePjjXRzJL2VSv3j+NKk3nWK1v0GplkCTQmthq7K+0He+D7uWWX0OvoHQ7TfQ+0LI2Wc1ERXnQNs+MPxm6D8NAmr5Mn7lYji2A+7c4vSnhZNKK6p44Zv9PLPyJ6pshpmjkrl1bFfCg/xdcj2llGM0KbRGNps1p2HHe1aSOHEEEHsT0c2QNNpa2KcuaWvh5Ykue1qo6Wh+Kf9esYelm9KJCwvgtrHdmDa0IyEB2hmtlDtoUmjtbDY4ugVCYn9uInJE9dPC1tqfJpxsy6E8HvpoF+tSc4kK8efaMztz3VlJtAkPdPm1lVI/06Sgalf9tPAQjLit2S67ITWX57/Zz+e7juHv68PlgxO5aXQyXdrU03mulHIKTQqqbgsvgsydzfa0UNNPWYW8+O1+lm46TEWVjXN7t+Pmc7owpHNMs8ahlLdpjoJ4ylONuc8a8rphfrNfumubMP512QBW3zuO28Z24/sDuVz+7Fouf3YNK3YcxablupVyK31S8FYLL4LMXfaRSO4bNlpcXslb6w/x4qoDpB8vISk2hGlDO3LZGYnERwa5LS6lWhttPlKnl7YGXp4EE/4JZ93q7miorLKxfPtRXlmbyvrU4/gIjO7ehilDEjm3TzuC/H3dHaJSHk2Tgqrfwgshc7fbnxZOdSC7iHc2pbN0YzpH8kuJCPLjwoHtmZrSkYGJkbrgj1KNoElB1S91NSyY3GKeFk5lsxnW7s/h7Q2H+GTHUUorbHRrG8aUIYlcekYH2kVo85JSjtKkoByz8ELI2mM9LfgHuzuaOhWUVvDx1gyWbExnQ5rVvDSxXzx3/KY7veIj6j+BUl5Ok4JyTPXTwr/grN+6OxqH7M8q5K0N6bz2XRqFZZVMsieH3gmaHJSqiyYF5bgFF0D2jy3+aeFUecXlzF91gJdXp3KirJKJfa3k0Ke9JgelTqXzFJTjxtwHhcdgw8uO7Z+5C5bfay0KlLHVtbGdRlRIAHef15NV947jjt90Z/W+bCY/+S1zXt3IziMFbotLKU+mTwrKUt/TQkWJVYhv48tw6HvwDYCAMKgqh2kLrbLebpZfXMFLqw/w8qoDnCirZELfdtzxm+70bR/p7tCUcjttPlINk7oKFpwPEx+GM2/5efuxnVZp7q1vQGk+xHaDITfAwKugqgwWTbNKZlzwX2t7C5BfXMH81QeYb08O5/Vpx42juzA0KVqHsyqvpUlBNdzJp4Xffgc/fmIlg5NPBb0vsr70k0b9sjx32Ql4+wbY9zmMuhvG/RV8Wkar5MnksGBNKvklFfRtH8GMkclcODCBQD+dDKe8iyYF1XAHvoWFF4CPP9gqILa7/algOoSeZnnNqkr4+PdWEuk3BS55BvxaTmns4vJK3v3hMAtWp7I3s5C4sACuGtaJa87sTFud66C8hCYF1Tgf3g1lBVYy6Dzy9Iv21GQMrH4cPn8AOo2AKxdBSMuqfGqMYfW+HF5efYAv92Ti5yOc3z+BG0YmM6hjlLvDU8qlNCko99i+FN6dA1Gd4eq3ISa5/mOKc2HfF/DjcmtkU8Ig6DzC+onp4nhiaoDU7CIWrk3l7Q3pFJZVckanKGaMTGZSv3j8fVtG85dSzqRJQblP2lp4YzqIL1z1JiSe8t+hMZC910oCP66w1qA2VRDaBhIGwpHNUJxt7Rue8HOC6DwS4no6tc/iRGkFSzams3BNKqk5xXSICubG0clcoUuHqlbG7UlBROYDFwCZxph+tbx/NXAvIMAJ4BZjzJb6zqtJwUNk74VFU+DEUbj8Reg+AQ6usZLAnuVw/IC1X7v+0HMi9JgI7QdbX/jGWB3eaautGddpq+FEhrV/cMzPCaL/VAhr45RwbTbDV3syef7r/axLzSU6xJ8bRiRz/YjORIUEOOUaSrlTS0gKZwOFwCt1JIURwC5jzHERmQQ8YIwZXt95NSl4kKJsWHwlpG+AwHCrr8I3ELqcAz0mWIkgMrH+8xgDx1Otct9p9iRxPBXiesCNn0OQc+chbEjN5dmVP/HF7kxCAny5algnZo1OJiHSc2Z7K3UqtycFexBJwIe1JYVT9osGthtjOtR3Tk0KHqaiBL54EMoLrSTQZQwEhDb9vPtXwmuXQ9dxMP0N8HH+ENPdRwt4/uv9LNtyBB+BS8/owM3ndKWrriutPJCnJYVE0vi0AAAZe0lEQVR7gF7GmBvreH82MBugU6dOQ9LS0pwcqfJI61+Cj+6GkXfBuX9z2WUO5Rbz4rf7eWP9IcqrbEzoE89NZydzRsdofHx0MpzyDB6TFERkLPAMMMoYk1PfOfVJQf3Ch7+z1pq+7AUYMM2ll8ouLGPhmlQWrkmloLSSiCA/hibFMDQ5hmHJMfTvEKkjl1SL5WhScOvwChEZALwITHIkISj1KxMfgawfYdntENsVOgxx2aXiwgL5/Xk9ufmcrny64yjrU3P5/kAuX+zOBCDY35fBnaMYmmQliTM6RhMcoDOnlWdx25OCiHQCvgSuM8ascfSc+qSgfqUoG+aNtWZhz14J4fHNevmsE2WsT81l3QHrZ9fRAowBf19hQGIUZ3aJYVS3NgzuHKXlNZTbuL35SEQWA2OAOOAYcD/gD2CMeU5EXgQuB052EFQ6ErAmBVWro9vhpfOgbW+44SPwd1/5ivySCjalHef7A7l8fyCHren5VNkMQf4+DEuOZVS3WEZ1a0Ov+HDtk1DNxu1JwVU0Kag67VwGb11rVXC95BmXzIRujBOlFXy/P5dV+7JZtS+bfZmFAMSGBjCiWxyjusUyslscidEhbo5UtWYe0aeglFP1uQjG/AlW/gvi+8FZt7o7IgDCg/wZ36cd4/u0A+Bofimr92Wz2p4kPthyBIDkuFDG9mzLuF5tGZYcQ4Cfdlqr5qdPCqp1sdng7eth94dW7aUWsPjP6Rhj2JtZyKq92Xz9YxZr9+dQXmkjLNCP0d3jGNerLWN7tSUurOVUnVWeSZuPlPcqK4T5EyD/ENz4JcR1c3dEDisur2T1vhy+3J3Jl7uPcaygDBEYmBjFuF7WU0Tf9hG6WJBqME0KyrsdT4MXxlq1km76wumlMJqDMYYdRwr4cncmX+zOZGt6HsZAfEQQ4/u0ZVK/BIYnx+CncyOUAzQpKJW6Gl65yCqtMXUhBHp2eYqsE2Ws3JPJF7sy+frHLEoqqogO8ee8PvFM6h/PiK5x2g+h6qRJQSmwVoP74E6I7ATnPwY9znN3RE234s+UdTiTr0hh+fajfLErk8Iya4b1+D7tmNQvgdHd4wjy1zkR6meaFJQ6KW0tfHgXZO2Gvpdas6DD27k7qsZJWwMvT7KWSr1tPYhQWlHF6n3ZfLztKJ/tPEpBaSWhAb6M692OSf3iGd09jvAgf3dHrtxMk4JSNVWWw+on4JtHwT8Yxv8NBl/v1AV7msVrU2DfZ9brGcuttSVqKK+0sXZ/Dp9sz2DFjmPkFpXj5yMMTYphbK82jOvVlq5twrSj2gtpUlCqNtn7rKeG1G+h01lw4RPQpqe7o3JMxhZ4/mwYfQ+smwc9J8Nlz9e5e2WVjY1px/lqTxYr92Sy++gJABKjgxnbsy1je7XhrC5xWp/JS2hSUKouxsDmRfDpX6zhq6PvhlF3u7U0hkPevgH2fg6/2w5f/A02vw6/3w3B0Q4dfiSvhJV7svhydyar92VTUlFFoJ8PZ3WNZWzPtgxNiqFb2zDtrG6lNCkoVZ/CLFjxf7DtLauN/sLHIWmUu6OqXfY+mJsCo+6C8Q9Y61jPOwcm/RuGz27w6coqq1h3IJevdmfx1Z5MDmQXAVYRv25tw+mdEE6fhAj6JETQOyGC6FBdktTTaVJQylH7PocP74a8NKtpZtxfWkzdpGrv3wbb3oa7tkFYW2vb82eDrQrmrGpyvGk5RWxJz2dXRgE7jxSwK6OAzBNl1e/HRwRZiaJ9BAMSozinRxsd3eRhtPaRUo7qNh5++x0s/yN8+x/wC4Rz/ujuqH6Wfxi2vAFDbvg5IYDVUf7R3XB4EyQ2bR2JzrGhdI4N5aKB7au3ZReWsSujwP5zgl0ZBXy7N5tKmyEy2J+LB7VnWkpHnWHdymhSUAogIAQufNL6y/urh8A/BEbc5u6oLGufBmODEbf/cnv/qVa/yKaFTU4KtYkLC2R09zaM7t6meltZZRXrDxzn7Y2HeGP9IV5Zm0av+HCmpXTkkjM6EKPNTB5Pm4+UqqmqEpbOgp3vwQX/g5SZ7o2nKAce7we9L6p9pNF7t1qx/n5Ps8/Yzi+uYNnWI7y94RBb0/Px9xXG927HtJSOjO4ep+U3WhhtPlKqMXz9rPWeK0qsfgb/EBh4ZePPZ0zT2vvXPQ8VxTDqd7W/P+R62PwabF9qvW5GkSH+XHtmZ649szO7jxbw9oZ03v3hMMu3H6VdRCCXDU5kfO+29EmI1GGvHkSfFJSqTUUpvD4VUlfB1AXQ5+KGHW8MbFtiDR1NPhsumtvwiXJlJ+B/fSFpNFy5qO7rPHMmBITCTV827PwuUF5p48vdmby94RBf7cnEZsBHoEe7cPp3iGRAxygGdIikV0K4Lk3azPRJQamm8A+CKxfDa5fBkllwZbDjdZMytsDHf4RD31k1lzYvsr60Jz3asKeGDS9Dab41h6IuIlaH84o/WUuSxv9qOfRmFeDnw8R+8UzsF092YRk/HMxjW3oeW9Lz+WJ3Jm9vTAesoa8948Pp3yGKgYmR9EqIIDY0gMgQf8ID/bTj2o30SUGp0ynJsyqtZu2xFu1JPrvufYuy4cu/w8aFEBIDv7kfzrgGPvt/sHYujPk/GHOvY9etKIUnBlqzra9fdvp9i3PhsZ7W6KTJ/3b41pqbMYbDeSVsS89n6+F8tqbnsTU9nxOllb/Yz9dHiAr2JzLEn+iQgOrXUcEBRIf4c0anaM7qGouvrm/dIPqkoJQzBEfBNe/CgvPh9Svhuveg47Bf7lNVCRteskYtlRXCmbfAOfdaxwKc9w8oOQ4r/2kli2E31X/dLa9D4dHTlrGoFhJjdURvfRPOfdCq7dQCiQiJ0SEkRocwqX8CYCWKtJxi9mYWkldcTl5xBXklJ/+tIK+4nKMFpew+eoK84nKKyqsAa2TU+f3juWhQewZ3itYnCyfSJwWlHHHiqFWdtCgHbvgAEgZa2/d/Dcvvhaxd1roNEx+Btr1+fXxVJbx1LexZDpe/CP2n1H2tqkqYOwRCYuHGLxxrcjrwDSy8EC59vmkd4y1cSXkVK/dksmzLEb7YnUl5pY0OUcFcOLA9Fw1sT++EcE0QddAZzUo5W94hKzFUFMNl86xmol3LIKoTTPgX9Dr/9F/gFSVWldND38H0N6F7HetHb1tiDYu9YhH0vsCx2IyBpwZDWDzMXN7we3NU+karmOCI28HHyR3FNhtUllr/O1UU1/5vVZk12TAokhOlFXy64xgfbD3Ct3uzqbIZurYJ5aKBHbhoUHuS40KdG5+H06SglCvk/GQlhsJj1nDVUXdbk9wcbbIpzYcFF0DOPrju/V83RRkDz44EUwW3rG3YiKVV/4PPH4Bb10ObHo4f54jyIvjyIfjuGcDAuL/C2fc459ybF8PHf4DyE47tnzgMZn7yi6SUW1TOx9syWLblCOtTczEGerYLp1vbMDrFhtA5JsT6NzaU+Iggr+yP0KSglKtk7bEqlA67CSITG358YRbMnwDFOdaaCO36/Pzenk9g8RWNawYqzIT/9rb6NM77R8Pjqsv+lbDsDqs2VMosKMqCPR9bTVvtBzXt3Md2Wmtpt+trPQH4B1vJ9hf/1nh98HtY/geY+LB1n7XIyC/ho60ZfLs3m4O5xaQfL6ai6ufvuQBfHxKjg2ski1ASo4NJiAwiITKY2NAAfFph0tCkoFRLdjzNSgzGwKwVEJ1kvX7pPKv/4o5N4NuI1dLevMZaae7uXeDXxJITJcetMho/vAYxXeGipyBppDXa6dmREBgON3/d+I7tihKYNxaKs2HOasdWwzMGXp9mzR+5ZQ3EJNd7SJXNcCSvhIO5xaTlFJOWW8TBHOv1wdxiCst+OfopwNeHdpGBJEQEkxAVRHxkEO0jg4mPDKJzbAg92oZ7ZNLQpKBUS5e5C+ZPtEYPzVwB2T9ao5wm/8exEUq12fs5LLrcmnDX99LGx7brA/jo99Yw2xG3w5j7fvnl/9NX8OolMOxmmPxo467x4e9gw3y4Zqn1lOCo/MPWhL2EgXD9B02aMW6MIbeonIz8Uo7klXC0oJQjeaUczS/hSH4pR+0/5VW26mMig/0ZlhzD8OQYzuwSS++ECI9ojtIhqUq1dG17W3MfXrnYmiQXGAmhbay5DY3VdSxEdrQ6wRuTFE4cs5pndr4P8f3hqrdqbyLqOhaG3wLfPws9JkC33zTsOjvftxLCiNsblhAAIjvAeX+HD+6EjQsgZUbDjq9BRIgNCyQ2LJB+HSJr3cdmM+QWl5ORV8q+rBN8vz+X7/bn8NnOYwBEBPkxLDmWM7t4VpKoiz4pKOVu+z635kDYKqwJb6NPM4PZESsfseZE3LnFapZyhDGwZTF88ierWWfMvTDijtM3YVWUwLwx1gS/3661nngckXcInhtpNUnNXNG4Zi5jrEmFh3+AW79rXN9OE2Xkl1QniO/255CaUwxAeJAfw5Nj6BUfQWJ0MB2ig0mMDqF9VJBbS3to85FSnmTn+9Zf91NfhqDa/2J1WH46PN7fGhn1m7+eft/SfNj1odVvcHANdDzT6jtwdPRSxhZ44TfQazJMXVh/U05VpdVEdmwHzPkGYro4dp3a5B6AZ0dYq+Vd9ZbbF0aqmSTWHcglLbeYKtvP368i0DY80D6BL9hKGFEhJMWG0C8xkoigRvQhNYDbk4KIzAcuADKNMb8qyCLWDJMngMlAMXCDMWZTfefVpKCUAxZNhaPb4K7tVuXXmipK4MdPrPkQez+zxv5HdbaaclJmNbxw37f/tQr/OTJi6suH4JtHrUq0A6Y17Dq1+e45+OTeFjlpr7LKxtGCUtKPl5B+vITDx0tIP15s/Z5XzJG80uqkIQJd24QxqGNU9U+v+HCnlh9vCUnhbKAQeKWOpDAZuB0rKQwHnjDGDK/vvJoUlHLArg/hzauton69JkNVhTW0dNsS2P2RNScgtC30u8xarKfDkMb/pW2rsuZeHNsOt6y2JvPV5sC31qzrgdPh0mcbfWu/vLbNmjeStRtuXefYCKYWorLKxrETZfyUWcjmQ3nVP7lF5QAE+fvQv0OkPUlEM6hTFO0jgxo9Y9vtScEeRBLwYR1J4XlgpTFmsf33PcAYY0zG6c6pSUEpB1RVWGW3o5OtOQA737PmRQRGQp+LrDIbSaOdNyv5eJo1TDVhgDUi6NTzFuVY/QgBoTD7a+cuCJS917p2j/Pgitecd143MMaQfryEHw7lsflgHpsPHWf7kQLKK63RTzeOSuYvF/Sp5yy184TRRx2AQzV+T7dv+1VSEJHZwGyATp3q+CtEKfUzX39rFNO3j1nt/j0nWYmg23hrDWpni+5sDU197xZY8xSMuuvn94yB92+1ktJVbzp/hbi47jD2T9Zs7h3vQd9LnHv+ZiQidIwJoWNMSPV62eWVNnYfLWDzoTx6tgt3eQweMSTVGDMPmAfWk4Kbw1HKM4y+x+o47jyieZbqHDjdKvj35T+g6zjrqQFg3Tz4cbk1C/lkIUFnO+t2KyF8fI/1BBQa65rruEGAnw8DEqMYkBjVLNdz5yKqh4GONX5PtG9TSjlDQIjVpNJcazeLwAWPW0NT35ltrQlxdJs1K7r7BBg+x3XX9vWDS56xhsd+cp/rruMF3JkUlgHXieVMIL++/gSlVAsXGgsXP2OVEl/xJ1gyE4JjrC9sVw8ZbdfXKtK37S2rhpRqFJc1H4nIYmAMECci6cD9gD+AMeY54GOskUf7sIakNn5aolKq5eg+HobeBOtfAMRaOS40rnmuPepu2LkMPrwLOn3380JHymEuSwrGmOn1vG+AW111faWUG537oFXLqdv40y9h6mx+AXDxXHjxN/Dpn+Hip5vv2q2ER3Q0K6U8TEBI/WtLu0qHwTDyTmt9CbAKDLbQJUpbIk0KSqnWZ9xfwcffmj2dsRWmveJQmW3l3o5mpZRyDR9fGPdnqyZSXhrMOwd+/NTdUXkETQpKqdarxwRrBnVUJ3h9Knz1T6ssh7tUVVoFCw9+D9uXwuonYd0L1vYWQpuPlFKtW0wyzPrMWjTo60cgfQNc/qLjpb4bI22N1WxVkG4tCpSfDgWHrVX1TC1J6cDXcPlLrplt3kCaFJRSrZ9/sDUSKXEoLP8jPH8OTFtodUo7U85P1uS5vfamKt9Aa1GgiA7WKKyIDvbfE61/IxPhh0XWnI5FU+HKRdYyp26k6ykopbzL4Y3w1vVQeMwamTTk+qafs6wQvvk3rH0a/IKsRYoGXGnNz3Bk0t7mxVZ9qISBcPUSl5TpcLQgnvYpKKW8S4chVj9D0ij44A7ry7issHHnMga2vg1zU2D141YZ8ts3WGtThLVxfBb3oOlWhddjO6xS4Pnuq/ijTwpKKe9kq4KVD1vDVn0DIWkkdDsXup8HsV3r/0LP2AIf/xEOfQcJg2Dyv6HjsKbFlLoKFk+3Vt+79l2rAqyTtIj1FFxBk4JSyqnSN8COd61V6LL3WNuik6zk0O1c64kiIOTn/Ytz4cu/w8YFVl2n8ffDoGsavmJdXTK2wKuXWa+vWQrtBznltJoUlFKqoY6nwb7PYO/n1oigimKrjyBplJUgRGDlv6C0AIbdBGP+5Jr6Stn74NVLrKqv0xdD8ugmn1KTglJKNUVFKRxcYz1B7P0McvZa25NGw6RHoV3jVkBzWP5heO0yyD0AUxdYy6o2gSYFpZRyptwDUJRlDWt1dRnwk4pzYdEUOLLZGlI76LR1Rk9LRx8ppZQzxSRbHcnNlRDAmmB33TKr+ei9OfD98y6/pCYFpZRqyQLDrBpO/aZATBeXX05nNCulVEvnFwhTXmqWS+mTglJKqWqaFJRSSlXTpKCUUqqaJgWllFLVNCkopZSqpklBKaVUNU0KSimlqmlSUEopVc3jah+JSBaQ1sjD44BsJ4bTErS2e2pt9wOt755a2/1A67un2u6nszGmTX0HelxSaAoR2eBIQShP0truqbXdD7S+e2pt9wOt756acj/afKSUUqqaJgWllFLVvC0pzHN3AC7Q2u6ptd0PtL57am33A63vnhp9P17Vp6CUUur0vO1JQSml1GloUlBKKVXNa5KCiEwUkT0isk9E7nN3PM4gIqkisk1ENouIxy1cLSLzRSRTRLbX2BYjIp+JyF77v9HujLGh6rinB0TksP1z2iwiTVuBvRmJSEcR+UpEdorIDhG5077dIz+n09yPJ39GQSKyTkS22O/pb/btySLyvf07700RCXDofN7QpyAivsCPwLlAOrAemG6M2enWwJpIRFKBFGOMR066EZGzgULgFWNMP/u2R4FcY8zD9uQdbYy5151xNkQd9/QAUGiM+Y87Y2sMEUkAEowxm0QkHNgIXALcgAd+Tqe5n2l47mckQKgxplBE/IFVwJ3A3cA7xpg3ROQ5YIsx5tn6zuctTwrDgH3GmP3GmHLgDeBiN8fk9Ywx3wC5p2y+GFhof70Q6/+wHqOOe/JYxpgMY8wm++sTwC6gAx76OZ3mfjyWsRTaf/W3/xhgHLDEvt3hz8hbkkIH4FCN39Px8P8Q7AzwqYhsFJHZ7g7GSdoZYzLsr48C7dwZjBPdJiJb7c1LHtHUcioRSQLOAL6nFXxOp9wPePBnJCK+IrIZyAQ+A34C8owxlfZdHP7O85ak0FqNMsYMBiYBt9qbLloNY7Vttob2zWeBrsAgIAN4zL3hNJyIhAFLgbuMMQU13/PEz6mW+/Hoz8gYU2WMGQQkYrWM9GrsubwlKRwGOtb4PdG+zaMZYw7b/80E3sX6j8HTHbO3+55s/810czxNZow5Zv8/rQ14AQ/7nOzt1EuBRcaYd+ybPfZzqu1+PP0zOskYkwd8BZwFRImIn/0th7/zvCUprAe623vjA4ArgWVujqlJRCTU3lGGiIQC5wHbT3+UR1gGXG9/fT3wvhtjcYqTX552l+JBn5O9E/MlYJcx5r813vLIz6mu+/Hwz6iNiETZXwdjDajZhZUcpth3c/gz8orRRwD2IWaPA77AfGPMQ24OqUlEpAvW0wGAH/C6p92TiCwGxmCV+T0G3A+8B7wFdMIqkT7NGOMxHbd13NMYrGYJA6QCN9doj2/RRGQU8C2wDbDZN/8fVju8x31Op7mf6XjuZzQAqyPZF+sP/beMMQ/avyPeAGKAH4BrjDFl9Z7PW5KCUkqp+nlL85FSSikHaFJQSilVTZOCUkqpapoUlFJKVdOkoJRSqpomBaVOISJVNaplbnZmVV0RSapZQVWplsav/l2U8jol9pIBSnkdfVJQykH29Sseta9hsU5Eutm3J4nIl/Zial+ISCf79nYi8q69zv0WERlhP5WviLxgr33/qX0WqlItgiYFpX4t+JTmoytqvJdvjOkPzMWaIQ/wFLDQGDMAWAQ8ad/+JPC1MWYgMBjYYd/eHXjaGNMXyAMud/H9KOUwndGs1ClEpNAYE1bL9lRgnDFmv72o2lFjTKyIZGMt3FJh355hjIkTkSwgsWZpAXu55s+MMd3tv98L+Btj/uH6O1OqfvqkoFTDmDpeN0TN+jNVaN+eakE0KSjVMFfU+Het/fUarMq7AFdjFVwD+AK4BaoXQYlsriCVaiz9C0WpXwu2r2J10ifGmJPDUqNFZCvWX/vT7dtuB14WkT8AWcAM+/Y7gXkiMgvrieAWrAVclGqxtE9BKQfZ+xRSjDHZ7o5FKVfR5iOllFLV9ElBKaVUNX1SUEopVU2TglJKqWqaFJRSSlXTpKCUUqqaJgWllFLV/j9x1Mexy5SjoAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot training & validation loss values\n",
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "plt.title('Model loss')\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Train', 'Test'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Augmentation Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".\t\t\t.git\t\t    Pioneer_main.ipynb\n",
      "..\t\t\t.ipynb_checkpoints  Skimage_learning.ipynb\n",
      "data_aug_cifar10.ipynb\tmodels\t\t    skimage_nupmy.ipynb\n",
      "\n",
      "\n",
      ".\t\t\t.git\t\t    Pioneer_main.ipynb\n",
      "..\t\t\t.ipynb_checkpoints  Skimage_learning.ipynb\n",
      "data_aug_cifar10.ipynb\tmodels\t\t    skimage_nupmy.ipynb\n"
     ]
    }
   ],
   "source": [
    "!ls -a\n",
    "!echo \"\"\n",
    "!rm -rf ./augmented_pictures/\n",
    "!echo \"\"\n",
    "!ls -a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Visualize some results of data augmentation\n",
    "import matplotlib.pyplot as plt\n",
    "import PIL\n",
    "\n",
    "SAVE_PATH = \"./augmented_pictures/\"\n",
    "    \n",
    "def output_augmented(num):\n",
    "    img = None\n",
    "    sub_img= None\n",
    "    name_list = None\n",
    "\n",
    "    if not os.path.isdir(SAVE_PATH):\n",
    "        os.makedirs(SAVE_PATH)\n",
    "\n",
    "    gen_data = datagen.flow(x_train, \n",
    "                                           batch_size=1, \n",
    "                                           shuffle=False, \n",
    "                                           save_to_dir=SAVE_PATH,\n",
    "                                           save_prefix=('generated'+ str(num)))\n",
    "    for i in range(9):\n",
    "        gen_data.next() \n",
    "    \n",
    "# name_list = os.listdir(SAVE_PATH)\n",
    "# # print(name_list)\n",
    "# fig = plt.figure()\n",
    "\n",
    "# for i in range(len(name_list)):\n",
    "#     img = PIL.Image.open(SAVE_PATH+name_list[i])\n",
    "#     sub_img = fig.add_subplot(331 + i)\n",
    "#     sub_img.imshow(img)\n",
    "    \n",
    "# plt.show()\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(5):\n",
    "    output_augmented(i)\n",
    "    name_list = os.listdir(SAVE_PATH)\n",
    "#     fig = plt.figure()\n",
    "#     for i in range(len(name_list)):\n",
    "#     img = PIL.Image.open(SAVE_PATH+name_list[i])\n",
    "#     sub_img = fig.add_subplot(331 + i)\n",
    "#     sub_img.imshow(img)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Augmentation for Semantic Segmentation\n",
    "This section will look at how different approaches can be used to augment the image data, generating the same transformations for both the image and the mask."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "segmentation_datagen = ImageDataGenerator(\n",
    "        featurewise_center=False,  # set input mean to 0 over the dataset\n",
    "        samplewise_center=False,  # set each sample mean to 0\n",
    "        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n",
    "        samplewise_std_normalization=False,  # divide each input by its std\n",
    "        zca_whitening=False,  # apply ZCA whitening\n",
    "        zca_epsilon=1e-06,  # epsilon for ZCA whitening\n",
    "        rotation_range=20,  # randomly rotate images in the range (degrees, 0 to 180)\n",
    "        shear_range=0.2,  # set range for random shear\n",
    "        zoom_range=0.,  # set range for random zoom\n",
    "        channel_shift_range=0.,  # set range for random channel shifts\n",
    "        # set mode for filling points outside the input boundaries\n",
    "        fill_mode='nearest',\n",
    "        cval=0.,  # value used for fill_mode = \"constant\"\n",
    "        horizontal_flip=True,  # randomly flip images\n",
    "        vertical_flip=False,  # randomly flip images\n",
    "        # set rescaling factor (applied before any other transformation)\n",
    "        rescale=None,\n",
    "        # set function that will be applied on each input\n",
    "        preprocessing_function=None,\n",
    "        # image data format, either \"channels_first\" or \"channels_last\"\n",
    "        data_format=None,\n",
    "        # fraction of images reserved for validation (strictly between 0 and 1)\n",
    "        validation_split=0.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Define two folders for saving the image and mask together\n",
    "save_img_path = \"./augmented_images\"\n",
    "save_mask_path = \"./augmented_masks\"\n",
    "\n",
    "seed = 34\n",
    "\n",
    "for counter in range(10): # Generate 20 pairs of images and mask, 2 pairs for each iteration\n",
    "    # The seed stays unchanged, but image_generator and mask_generator are re-initialized every time\n",
    "    # This is to make sure that every time the 2 images and 2 masks are generated \n",
    "    image_generator = segmentation_datagen.flow(x_train,\n",
    "                                            batch_size=1, \n",
    "                                            shuffle=False, \n",
    "                                            save_to_dir=SAVE_PATH,\n",
    "                                            save_prefix=('image_'+ str(num)),\n",
    "                                            seed=seed)\n",
    "\n",
    "    mask_generator = segmentation_datagen.flow(x_train,\n",
    "                                                batch_size=1, \n",
    "                                                shuffle=False, \n",
    "                                                save_to_dir=SAVE_PATH,\n",
    "                                                save_prefix=('mask_'+ str(num)),\n",
    "                                                seed=seed)\n",
    "    \n",
    "    \n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}