(1)CNN_Affine_Valid99p78Test99p24_04272019_99p4ProbBased_LeNet_LabForRpt.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "(1)CNN Affine Valid99p78Test99p24 04272019 99p4ProbBased LeNet-LabForRpt.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/preritt/CS231n-/blob/master/(1)CNN_Affine_Valid99p78Test99p24_04272019_99p4ProbBased_LeNet_LabForRpt.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "HFVqYNiRCy47",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# LeNet Lab\n",
        "![LeNet Architecture](https://github.com/sujaybabruwad/LeNet-in-Tensorflow/blob/master/lenet.png?raw=1)\n",
        "Source: Yan LeCun"
      ]
    },
    {
      "metadata": {
        "id": "k7PXv7fjCy5B",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Load Data\n",
        "\n",
        "Load the MNIST data, which comes pre-loaded with TensorFlow.\n",
        "\n",
        "You do not need to modify this section."
      ]
    },
    {
      "metadata": {
        "id": "t4iqynjrCy5E",
        "colab_type": "code",
        "outputId": "45110226-9697-482b-d001-b149afbcb5c9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        }
      },
      "cell_type": "code",
      "source": [
        "from tensorflow.examples.tutorials.mnist import input_data\n",
        "\n",
        "mnist = input_data.read_data_sets(\"MNIST_data/\", reshape=False)\n",
        "X_train, y_train           = mnist.train.images, mnist.train.labels\n",
        "X_validation, y_validation = mnist.validation.images, mnist.validation.labels\n",
        "X_test, y_test             = mnist.test.images, mnist.test.labels\n",
        "\n",
        "assert(len(X_train) == len(y_train))\n",
        "assert(len(X_validation) == len(y_validation))\n",
        "assert(len(X_test) == len(y_test))\n",
        "\n",
        "print()\n",
        "print(\"Image Shape: {}\".format(X_train[0].shape))\n",
        "print()\n",
        "print(\"Training Set:   {} samples\".format(len(X_train)))\n",
        "print(\"Validation Set: {} samples\".format(len(X_validation)))\n",
        "print(\"Test Set:       {} samples\".format(len(X_test)))"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
            "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
            "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
            "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n",
            "\n",
            "Image Shape: (28, 28, 1)\n",
            "\n",
            "Training Set:   55000 samples\n",
            "Validation Set: 5000 samples\n",
            "Test Set:       10000 samples\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "L7ESI_fVaxBT",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "38221bea-f0cd-407c-d8dd-c99a9f67cbd5"
      },
      "cell_type": "code",
      "source": [
        "X_train.shape"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(55000, 28, 28, 1)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "metadata": {
        "id": "moyj2wnOYMVx",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "from sklearn.model_selection import train_test_split\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "SyyhfjBFXcED",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "total_train_data = np.load('train_total_x.npy')\n",
        "test_data = np.load('test_x.npy')\n",
        "\n",
        "total_train_label = np.load('train_total_y.npy')\n",
        "test_label = np.load('test_y.npy')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "crrDKLpQZqB4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "27596030-d572-4ba2-e7d5-09ada2f002b7"
      },
      "cell_type": "code",
      "source": [
        "10000/600000"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.016666666666666666"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "metadata": {
        "id": "N_aId-4RYmOj",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "\n",
        "# split train data and to create validation data\n",
        "train_X,valid_X,train_ground,valid_ground = train_test_split(total_train_data,\n",
        "                                                             total_train_label,\n",
        "                                                             test_size=.1,\n",
        "                                                             random_state=13)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "LjL8jRvJaP1D",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "train_X_reshaped =  np.reshape(train_X, newshape=(-1,28,28,1))\n",
        "valid_X_reshaped =  np.reshape(valid_X, newshape=(-1,28,28,1))\n",
        "test_X_reshaped =  np.reshape(test_data, newshape=(-1,28,28,1))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "BjSYlqDKa7WZ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "X_train = train_X_reshaped\n",
        "X_validation = valid_X_reshaped\n",
        "X_test = test_X_reshaped"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "7i4JYdahbLo7",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "y_train = train_ground\n",
        "y_validation = valid_ground\n",
        "y_test = test_label"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "WL65sl7Ua2EB",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# test_X_reshaped.shape"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "pV8kvNmMCy5T",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "The MNIST data that TensorFlow pre-loads comes as 28x28x1 images.\n",
        "\n",
        "However, the LeNet architecture only accepts 32x32xC images, where C is the number of color channels.\n",
        "\n",
        "In order to reformat the MNIST data into a shape that LeNet will accept, we pad the data with two rows of zeros on the top and bottom, and two columns of zeros on the left and right (28+2+2 = 32).\n",
        "\n",
        "You do not need to modify this section."
      ]
    },
    {
      "metadata": {
        "id": "sBWJCBw1Cy5U",
        "colab_type": "code",
        "outputId": "af59f5f5-d67e-4e3a-eca5-817fa1384ccf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "\n",
        "# Pad images with 0s\n",
        "X_train      = np.pad(X_train, ((0,0),(2,2),(2,2),(0,0)), 'constant')\n",
        "X_validation = np.pad(X_validation, ((0,0),(2,2),(2,2),(0,0)), 'constant')\n",
        "X_test       = np.pad(X_test, ((0,0),(2,2),(2,2),(0,0)), 'constant')\n",
        "    \n",
        "print(\"Updated Image Shape: {}\".format(X_train[0].shape))"
      ],
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Updated Image Shape: (32, 32, 1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "cW7cCH1xCy5c",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Visualize Data\n",
        "\n",
        "View a sample from the dataset.\n",
        "\n",
        "You do not need to modify this section."
      ]
    },
    {
      "metadata": {
        "id": "h7UbBX8RCy5e",
        "colab_type": "code",
        "outputId": "595921c5-6421-4709-d147-a6bde2e025d4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 123
        }
      },
      "cell_type": "code",
      "source": [
        "import random\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "\n",
        "index = random.randint(0, len(X_train))\n",
        "image = X_train[index].squeeze()\n",
        "\n",
        "plt.figure(figsize=(1,1))\n",
        "plt.imshow(image, cmap=\"gray\")\n",
        "print(y_train[index])"
      ],
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "6\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAFsAAABZCAYAAABR/liSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAABnxJREFUeJztnF1oVEcUx38n0TzVj8SCRhOTWoso\niqnEIEbEl2ItgsmDsVGKhUAqEqhQTaUP0hdB6If6IMXUKkGK8aMBPxCqlgQ1D5KtxjZG20pojJ81\n7UNiEYLN6cO9dz9M1l13bya76/zgsrtzZ3bO/vfsmZm7546oKhYzZI21Aa8SVmyDWLENYsU2iBXb\nIFZsg1ixDZKU2CLyroj8JiK3RWS7X0ZlKpLookZEsoHfgXeAu0A7UK2qXf6Zl1mMS6JtGXBbVbsB\nRKQJWANEFVtEMna5qqoSq04yYWQG0Bv2+q5bFoGI1IpIQEQCSfSVESTj2XGhqg1AA2S2Z8dDMp59\nDygMe13gllmikIzY7cBbIvKGiOQA7wOn/DErM0k4jKjqMxGpA34EsoGDqnrDN8sykISnfgl1lsEx\ne7RnI5aXxIptkFGf+plk+vTpAOzcuZMFCxYAUFNTA8D169fHzC6PjBC7oKAAgJaWFgBmz54dPFda\nWgqkhtg2jBgk7WcjEyZMoL29HYA5c+YEy69duwbAihUrAOjv7/e76wjsbCTFSPuYXVVVFeHRHjt2\n7ABG36NfhrQXe+LEicPKenp6UmJAfB4bRgyS9p6dnZ09rGzv3r309vaOUHtssZ5tElU1dgDq99He\n3q4ebW1t2tbWpllZWb73E+uI5/NbzzZI2sbsZcuWAaHlOMCJEycAGBoaGhObYpG2YtfX1wefX758\nGYB9+/bF1TYry/lBr127FoBt27YBMGvWrOC8vLa2FoBz5875YzB2gDRK2nn2hg0bAFi1alWw7Nix\nYwAMDg7GbF9cXMzFixcBKCwsHHY+NzcXgLNnzwJQXl7OlStXkjPaxXq2QdLWs8eNc0zv7e3l0KFD\nUevn5OQAUFFRAcCePXvIz88H4PHjxwDB9sePH2fx4sWAszACWLRokW+enTZiz5jhJFstXbo0ovzI\nkSM8efJkWP3x48cDzoUqgMOHDwPOuiIQcJKzvFDU19cXbOed8wbIhQsX+vYZbBgxSNp4tudpkyZN\nAmBgYACA/fv3j1i/uroagMbGxojyQCBAWVlZ1H6898/Ly0vO4BGwnm2QtPHsadOmRbxubm4GoLu7\ne1jdkpISDhw4EFHW1NQEhDw+Gp7Xz5w5EwgtmPwgpmeLSKGItIhIl4jcEJGP3fI8ETkvIn+4j7m+\nWZWhxOPZz4BPVPWqiEwAfhaR88CHwE+qusu9xWM78OnomRrJnTt3op6rrKwMzkY6OzuB0JTxRSxZ\nsiS4QLp16xYAFy5cSNbUIDHFVtUHwAP3+YCI3MRJel8DrHCrNQKtGBR7ypQpUc+1trayadMmAObO\nnQvAyZMnAVi/fn1wcPVYvXo14EwPvWnkunXrAHj48KFvNr9UzBaRYuBt4Aow1f0iAB4CU6O0qQVq\nEzcxc4hbbBF5DfgB2KKq/SKhNAlV1Wg5IaN154HnjVu3buXp06cR51paWpg/fz4Ay5cvB0ID3tDQ\nEPPmzQNCV/s2btwIOCvKlStXAtDV5f99WHFN/URkPI7Q36tqs1v8SETy3fP5wF++W5dhxMyIEseF\nG4F/VHVLWPkXwN9hA2SeqtZHex+3TcKe7S2tz5w5A4SuSZ8+fZpLly4BBB/D/+ydPHkyEFq2V1ZW\nBpMu79+/D8DRo0cB2L17d8J/FMeTERVPGCkHPgB+FZEOt+wzYBdwTERqgB6gKiErXyHSLtevoaEB\nCKUCex4eC+9a98DAQHB6t3nz5mTNCRKPZ6ed2B51dXWAcxWwqKgIcDKhnqejw/kxnjrl3FvlzZ/9\nxiZWphhp69mphvXsFMOKbRArtkGs2AaxYhvEim0QK7ZBrNgGsWIbxIptECu2QUznjfQB/7qPqc7r\nxG9nUTyVjF6IAhCRgKqWxq45toyGnTaMGMSKbZCxELthDPpMBN/tNB6zX2VsGDGIMbFTea/tF2Tq\nfi4i90Skwz3eS6ofE2Ek1ffadjO68sMzdYEKnFyYJ6r6pR/9mPLs4F7bqjoIeHttpwSq+kBVr7rP\nBwAvU9dXTIkd117bqcBzmboAdSLyi4gcTDbh3w6QYTyfqQt8A7wJlODkqH+VzPubEjvl99oeKVNX\nVR+p6n+qOgR8ixMOE8aU2Cm917abqfsdcFNVvw4rzw+rVgl0JtOPkat+abDXdrRM3WoRKcHZLedP\n4KNkOrErSIPYAdIgVmyDWLENYsU2iBXbIFZsg1ixDWLFNsj/jkbP0pPc7t4AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 72x72 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "1W6py5mlCy5n",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Preprocess Data\n",
        "\n",
        "Shuffle the training data.\n",
        "\n",
        "You do not need to modify this section."
      ]
    },
    {
      "metadata": {
        "id": "UYJZs7GgCy5p",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from sklearn.utils import shuffle\n",
        "\n",
        "X_train, y_train = shuffle(X_train, y_train)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Q4WFIdiuCy53",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Setup TensorFlow\n",
        "The `EPOCH` and `BATCH_SIZE` values affect the training speed and model accuracy.\n",
        "\n",
        "You do not need to modify this section."
      ]
    },
    {
      "metadata": {
        "id": "est-t83SCy55",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import tensorflow as tf\n",
        "\n",
        "EPOCHS = 200\n",
        "BATCH_SIZE = 512"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "cWVHSK0pCy59",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## TODO: Implement LeNet-5\n",
        "Implement the [LeNet-5](http://yann.lecun.com/exdb/lenet/) neural network architecture.\n",
        "\n",
        "This is the only cell you need to edit.\n",
        "### Input\n",
        "The LeNet architecture accepts a 32x32xC image as input, where C is the number of color channels. Since MNIST images are grayscale, C is 1 in this case.\n",
        "\n",
        "### Architecture\n",
        "**Layer 1: Convolutional.** The output shape should be 28x28x6.\n",
        "\n",
        "**Activation.** Your choice of activation function.\n",
        "\n",
        "**Pooling.** The output shape should be 14x14x6.\n",
        "\n",
        "**Layer 2: Convolutional.** The output shape should be 10x10x16.\n",
        "\n",
        "**Activation.** Your choice of activation function.\n",
        "\n",
        "**Pooling.** The output shape should be 5x5x16.\n",
        "\n",
        "**Flatten.** Flatten the output shape of the final pooling layer such that it's 1D instead of 3D. The easiest way to do is by using `tf.contrib.layers.flatten`, which is already imported for you.\n",
        "\n",
        "**Layer 3: Fully Connected.** This should have 120 outputs.\n",
        "\n",
        "**Activation.** Your choice of activation function.\n",
        "\n",
        "**Layer 4: Fully Connected.** This should have 84 outputs.\n",
        "\n",
        "**Activation.** Your choice of activation function.\n",
        "\n",
        "**Layer 5: Fully Connected (Logits).** This should have 10 outputs.\n",
        "\n",
        "### Output\n",
        "Return the result of the 2nd fully connected layer."
      ]
    },
    {
      "metadata": {
        "id": "Vgad6Ny0OjqN",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "connection_probability = tf.Variable(.9)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "X1nJmcI7Cy5-",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from tensorflow.contrib.layers import flatten\n",
        "\n",
        "def LeNet(x, test_mode = False):    \n",
        "    # Hyperparameters\n",
        "    mu = 0\n",
        "    sigma = 0.1\n",
        "    layer_depth = {\n",
        "        'layer_1' : 6,\n",
        "        'layer_2' : 16,\n",
        "        'layer_3' : 120,\n",
        "        'layer_f1' : 84\n",
        "    }\n",
        "\n",
        "    \n",
        "    # TODO: Layer 1: Convolutional. Input = 32x32x1. Output = 28x28x6.\n",
        "    conv1_w = tf.Variable(tf.truncated_normal(shape = [5,5,1,6],mean = mu, stddev = sigma))\n",
        "    conv1_b = tf.Variable(tf.zeros(6))\n",
        "    conv1 = tf.nn.conv2d(x,conv1_w, strides = [1,1,1,1], padding = 'VALID') + conv1_b \n",
        "    # TODO: Activation.\n",
        "    conv1 = tf.nn.relu(conv1)\n",
        "\n",
        "    # TODO: Pooling. Input = 28x28x6. Output = 14x14x6.\n",
        "    pool_1 = tf.nn.max_pool(conv1,ksize = [1,2,2,1], strides = [1,2,2,1], padding = 'VALID')\n",
        "    \n",
        "    # TODO: Layer 2: Convolutional. Output = 10x10x16.\n",
        "    conv2_w = tf.Variable(tf.truncated_normal(shape = [5,5,6,16], mean = mu, stddev = sigma))\n",
        "    conv2_b = tf.Variable(tf.zeros(16))\n",
        "    conv2 = tf.nn.conv2d(pool_1, conv2_w, strides = [1,1,1,1], padding = 'VALID') + conv2_b\n",
        "    # TODO: Activation.\n",
        "    conv2 = tf.nn.relu(conv2)\n",
        "    \n",
        "    ##### Inset probability connection from x to conv2\n",
        "    conv1p_w = tf.Variable(tf.truncated_normal(shape = [5,5,1,16], mean = mu, stddev = sigma))\n",
        "    conv1p_b = tf.Variable(tf.zeros(16))\n",
        "    conv2_2nd_input = tf.nn.conv2d(x, conv1p_w, strides = [1,3,3,1], padding = 'VALID') + conv1p_b\n",
        "    conv2_2nd_input = tf.nn.relu(conv2_2nd_input) #result = tf.cond(x < y, lambda: tf.add(x, z), lambda: tf.square(y))\n",
        "#     conv2 = tf.cond(tf.random.uniform(shape = tf.shape(connection_probability)) < connection_probability,lambda: conv2 + conv2_2nd_input, lambda: conv2 )\n",
        "    conv2 = tf.cond(tf.logical_and(tf.random.uniform(shape = tf.shape(connection_probability)) < connection_probability, tf.equal(test_mode,False)),lambda: conv2 + conv2_2nd_input, lambda: conv2 )\n",
        "\n",
        "#     if (np.random.random_sample()<connection_probability and test_mode == False):\n",
        "#         conv2 = conv2 + conv2_2nd_input\n",
        "#     if test_mode == True:\n",
        "#       conv2 = conv2 + conv2_2nd_input*connection_probability*0\n",
        "    \n",
        "    # TODO: Pooling. Input = 10x10x16. Output = 5x5x16.\n",
        "    pool_2 = tf.nn.max_pool(conv2, ksize = [1,2,2,1], strides = [1,2,2,1], padding = 'VALID') \n",
        "    \n",
        "    # TODO: Flatten. Input = 5x5x16. Output = 400.\n",
        "    fc1 = flatten(pool_2)\n",
        "    fdense = fc1\n",
        "    \n",
        "    # TODO: Layer 3: Fully Connected. Input = 400. Output = 120.\n",
        "    fc1_w = tf.Variable(tf.truncated_normal(shape = (400,120), mean = mu, stddev = sigma))\n",
        "    fc1_b = tf.Variable(tf.zeros(120))\n",
        "    fc1 = tf.matmul(fc1,fc1_w) + fc1_b\n",
        "    \n",
        "    # TODO: Activation.\n",
        "    fc1 = tf.nn.relu(fc1)\n",
        "\n",
        "    # TODO: Layer 4: Fully Connected. Input = 120. Output = 84.\n",
        "    fc2_w = tf.Variable(tf.truncated_normal(shape = (120,84), mean = mu, stddev = sigma))\n",
        "    fc2_b = tf.Variable(tf.zeros(84))\n",
        "    fc2 = tf.matmul(fc1,fc2_w) + fc2_b\n",
        "    # TODO: Activation.\n",
        "    fc2 = tf.nn.relu(fc2)\n",
        "    \n",
        "    \n",
        "    #################\n",
        "    ##### Inset probability connection from x to conv2\n",
        "    fc2p_w = tf.Variable(tf.truncated_normal(shape = [400,84], mean = mu, stddev = sigma))\n",
        "    fc2p_b = tf.Variable(tf.zeros(84))\n",
        "    fc2_2nd_input = tf.matmul(fdense,fc2p_w) + fc2p_b\n",
        "    fc2_2nd_input = tf.nn.relu(fc2_2nd_input)\n",
        "    fc2 = tf.cond(tf.logical_and(tf.random.uniform(shape = tf.shape(connection_probability)) < connection_probability, tf.equal(test_mode,False)),lambda: fc2 + fc2_2nd_input, lambda: fc2 )\n",
        "\n",
        "#     if (np.random.random_sample()<connection_probability and test_mode == False):\n",
        "#         fc2 = fc2 + fc2_2nd_input\n",
        "#     if test_mode == True:\n",
        "#         fc2 = fc2 + fc2_2nd_input*connection_probability*0\n",
        "    \n",
        "    ################\n",
        "    \n",
        "    # TODO: Layer 5: Fully Connected. Input = 84. Output = 10.\n",
        "    fc3_w = tf.Variable(tf.truncated_normal(shape = (84,10), mean = mu , stddev = sigma))\n",
        "    fc3_b = tf.Variable(tf.zeros(10))\n",
        "    logits = tf.matmul(fc2, fc3_w) + fc3_b\n",
        "    return logits"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "aCx837c9Cy6E",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Features and Labels\n",
        "Train LeNet to classify [MNIST](http://yann.lecun.com/exdb/mnist/) data.\n",
        "\n",
        "`x` is a placeholder for a batch of input images.\n",
        "`y` is a placeholder for a batch of output labels.\n",
        "\n",
        "You do not need to modify this section."
      ]
    },
    {
      "metadata": {
        "id": "Mrb2FHFHCy6G",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "x = tf.placeholder(tf.float32, (None, 32, 32, 1))\n",
        "y = tf.placeholder(tf.int32, (None))\n",
        "one_hot_y = tf.one_hot(y, 10)\n",
        "is_testing= tf.placeholder(tf.bool) "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "uk0pcAjmCy6M",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Training Pipeline\n",
        "Create a training pipeline that uses the model to classify MNIST data.\n",
        "\n",
        "You do not need to modify this section."
      ]
    },
    {
      "metadata": {
        "id": "3oYy_Y3FCy6N",
        "colab_type": "code",
        "outputId": "1ef71a93-858f-4024-9e5b-a47b5dcfb04a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "rate = 0.001\n",
        "decay_rate = 1.00002*(X_train.shape[0]/BATCH_SIZE)\n",
        "decay_rate = 2\n",
        "\n",
        "print(decay_rate)\n",
        "logits = LeNet(x,is_testing)\n",
        "with tf.name_scope('Train'):\n",
        "  cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits = logits, labels = one_hot_y)\n",
        "  loss_operation = tf.reduce_mean(cross_entropy, name='loss')\n",
        "  tf.summary.scalar('loss', loss_operation)\n",
        "# optimizer = tf.train.MomentumOptimizer(learning_rate = rate,momentum=.9)\n",
        "optimizer = tf.train.AdamOptimizer(learning_rate = rate)\n",
        "# tf.train.natural_exp_decay()\n",
        "training_operation = optimizer.minimize(loss_operation)\n",
        "new_prob = connection_probability.assign(connection_probability/decay_rate)\n",
        "\n"
      ],
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "KrMtHo6rCy6S",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Model Evaluation\n",
        "Evaluate how well the loss and accuracy of the model for a given dataset.\n",
        "\n",
        "You do not need to modify this section."
      ]
    },
    {
      "metadata": {
        "id": "OMR9XsLBCy6U",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(one_hot_y, 1))\n",
        "accuracy_operation = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
        "saver = tf.train.Saver()\n",
        "\n",
        "def evaluate(X_data, y_data):\n",
        "    num_examples = len(X_data)\n",
        "    total_accuracy = 0\n",
        "    sess = tf.get_default_session()\n",
        "    for offset in range(0, num_examples, BATCH_SIZE):\n",
        "        batch_x, batch_y = X_data[offset:offset+BATCH_SIZE], y_data[offset:offset+BATCH_SIZE]\n",
        "        accuracy = sess.run(accuracy_operation, feed_dict={x: batch_x, y: batch_y, is_testing: True})\n",
        "        total_accuracy += (accuracy * len(batch_x))\n",
        "    return total_accuracy / num_examples"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "2Jw7iox3Cy6X",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Train the Model\n",
        "Run the training data through the training pipeline to train the model.\n",
        "\n",
        "Before each epoch, shuffle the training set.\n",
        "\n",
        "After each epoch, measure the loss and accuracy of the validation set.\n",
        "\n",
        "Save the model after training.\n",
        "\n",
        "You do not need to modify this section."
      ]
    },
    {
      "metadata": {
        "id": "c98G07hDQG2g",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import math"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "8QC1kt_DL1yU",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def augment(images, labels,\n",
        "            resize=None, # (width, height) tuple or None\n",
        "            horizontal_flip=False,\n",
        "            vertical_flip=False,\n",
        "            rotate=0, # Maximum rotation angle in degrees\n",
        "            crop_probability=0, # How often we do crops\n",
        "            crop_min_percent=0.6, # Minimum linear dimension of a crop\n",
        "            crop_max_percent=1.,  # Maximum linear dimension of a crop\n",
        "            mixup=0):  # Mixup coeffecient, see https://arxiv.org/abs/1710.09412.pdf\n",
        "  if resize is not None:\n",
        "    images = tf.image.resize_bilinear(images, resize)\n",
        "  \n",
        "  # My experiments showed that casting on GPU improves training performance\n",
        "  if images.dtype != tf.float32:\n",
        "    images = tf.image.convert_image_dtype(images, dtype=tf.float32)\n",
        "    images = tf.subtract(images, 0.5)\n",
        "    images = tf.multiply(images, 2.0)\n",
        "  labels = tf.to_float(labels)\n",
        "\n",
        "  with tf.name_scope('augmentation'):\n",
        "    shp = tf.shape(images)\n",
        "    batch_size, height, width = shp[0], shp[1], shp[2]\n",
        "    width = tf.cast(width, tf.float32)\n",
        "    height = tf.cast(height, tf.float32)\n",
        "\n",
        "    # The list of affine transformations that our image will go under.\n",
        "    # Every element is Nx8 tensor, where N is a batch size.\n",
        "    transforms = []\n",
        "    identity = tf.constant([1, 0, 0, 0, 1, 0, 0, 0], dtype=tf.float32)\n",
        "    if horizontal_flip:\n",
        "      coin = tf.less(tf.random_uniform([batch_size], 0, 1.0), 0.5)\n",
        "      flip_transform = tf.convert_to_tensor(\n",
        "          [-1., 0., width, 0., 1., 0., 0., 0.], dtype=tf.float32)\n",
        "      transforms.append(\n",
        "          tf.where(coin,\n",
        "                   tf.tile(tf.expand_dims(flip_transform, 0), [batch_size, 1]),\n",
        "                   tf.tile(tf.expand_dims(identity, 0), [batch_size, 1])))\n",
        "\n",
        "    if vertical_flip:\n",
        "      coin = tf.less(tf.random_uniform([batch_size], 0, 1.0), 0.5)\n",
        "      flip_transform = tf.convert_to_tensor(\n",
        "          [1, 0, 0, 0, -1, height, 0, 0], dtype=tf.float32)\n",
        "      transforms.append(\n",
        "          tf.where(coin,\n",
        "                   tf.tile(tf.expand_dims(flip_transform, 0), [batch_size, 1]),\n",
        "                   tf.tile(tf.expand_dims(identity, 0), [batch_size, 1])))\n",
        "\n",
        "    if rotate > 0:\n",
        "      angle_rad = rotate / 180 * math.pi\n",
        "      angles = tf.random_uniform([batch_size], -angle_rad, angle_rad)\n",
        "      transforms.append(\n",
        "          tf.contrib.image.angles_to_projective_transforms(\n",
        "              angles, height, width))\n",
        "\n",
        "    if crop_probability > 0:\n",
        "      crop_pct = tf.random_uniform([batch_size], crop_min_percent,\n",
        "                                   crop_max_percent)\n",
        "      left = tf.random_uniform([batch_size], 0, width * (1 - crop_pct))\n",
        "      top = tf.random_uniform([batch_size], 0, height * (1 - crop_pct))\n",
        "      crop_transform = tf.stack([\n",
        "          crop_pct,\n",
        "          tf.zeros([batch_size]), top,\n",
        "          tf.zeros([batch_size]), crop_pct, left,\n",
        "          tf.zeros([batch_size]),\n",
        "          tf.zeros([batch_size])\n",
        "      ], 1)\n",
        "\n",
        "      coin = tf.less(\n",
        "          tf.random_uniform([batch_size], 0, 1.0), crop_probability)\n",
        "      transforms.append(\n",
        "          tf.where(coin, crop_transform,\n",
        "                   tf.tile(tf.expand_dims(identity, 0), [batch_size, 1])))\n",
        "\n",
        "    if transforms:\n",
        "      images = tf.contrib.image.transform(\n",
        "          images,\n",
        "          tf.contrib.image.compose_transforms(*transforms),\n",
        "          interpolation='BILINEAR') # or 'NEAREST'\n",
        "\n",
        "    def cshift(values): # Circular shift in batch dimension\n",
        "      return tf.concat([values[-1:, ...], values[:-1, ...]], 0)\n",
        "\n",
        "    if mixup > 0:\n",
        "      mixup = 1.0 * mixup # Convert to float, as tf.distributions.Beta requires floats.\n",
        "      beta = tf.distributions.Beta(mixup, mixup)\n",
        "      lam = beta.sample(batch_size)\n",
        "      ll = tf.expand_dims(tf.expand_dims(tf.expand_dims(lam, -1), -1), -1)\n",
        "      images = ll * images + (1 - ll) * cshift(images)\n",
        "      labels = lam * labels + (1 - lam) * cshift(labels)\n",
        "\n",
        "  return images, labels"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "siI7jT2RUvYP",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def augment_data(dataset, dataset_labels, augementation_factor=1, use_random_rotation=True, use_random_shear=True, use_random_shift=True, use_random_zoom=True):\n",
        "\taugmented_image = []\n",
        "\taugmented_image_labels = []\n",
        "\n",
        "\tfor num in range (0, dataset.shape[0]):\n",
        "\n",
        "\t\tfor i in range(0, augementation_factor):\n",
        "\t\t\t# original image:\n",
        "\t\t\taugmented_image.append(dataset[num])\n",
        "\t\t\taugmented_image_labels.append(dataset_labels[num])\n",
        "\n",
        "\t\t\tif use_random_rotation:\n",
        "\t\t\t\taugmented_image.append(tf.contrib.keras.preprocessing.image.random_rotation(dataset[num], 20, row_axis=0, col_axis=1, channel_axis=2))\n",
        "\t\t\t\taugmented_image_labels.append(dataset_labels[num])\n",
        "\n",
        "\t\t\tif use_random_shear:\n",
        "\t\t\t\taugmented_image.append(tf.contrib.keras.preprocessing.image.random_shear(dataset[num], 0.2, row_axis=0, col_axis=1, channel_axis=2))\n",
        "\t\t\t\taugmented_image_labels.append(dataset_labels[num])\n",
        "\n",
        "\t\t\tif use_random_shift:\n",
        "\t\t\t\taugmented_image.append(tf.contrib.keras.preprocessing.image.random_shift(dataset[num], 0.2, 0.2, row_axis=0, col_axis=1, channel_axis=2))\n",
        "\t\t\t\taugmented_image_labels.append(dataset_labels[num])\n",
        "\n",
        "# \t\t\tif use_random_zoom:\n",
        "# \t\t\t\taugmented_image.append(tf.contrib.keras.preprocessing.image.random_zoom(dataset[num], 0.9, row_axis=0, col_axis=1, channel_axis=2))\n",
        "# \t\t\t\taugmented_image_labels.append(dataset_labels[num])\n",
        "\n",
        "\treturn np.array(augmented_image), np.array(augmented_image_labels)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Of2PvjkIQikg",
        "colab_type": "code",
        "outputId": "e6d56545-5813-4eac-aa2b-cea3de2d2dd4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "(X_train.shape)"
      ],
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(540000, 32, 32, 1)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 58
        }
      ]
    },
    {
      "metadata": {
        "id": "RbERCioKL6dc",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# # These can be any tensors of matching type and dimensions.\n",
        "# images = tf.placeholder(tf.uint8, shape=(None, None, None, 3))\n",
        "# labels = tf.placeholder(tf.uint64, shape=(None))\n",
        "\n",
        "# images, labels = augment(tf.convert_to_tensor(X_train), tf.convert_to_tensor(y_train),\n",
        "#                          horizontal_flip=True, rotate=15, crop_probability=0.8, mixup=4)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "-bwtFMk5U0HH",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "images, labels = (X_train, y_train)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "VZA2k-ZQR7kU",
        "colab_type": "code",
        "outputId": "cfb1ce6e-61d5-40f0-d2e5-d186ca0f26ec",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "images.shape"
      ],
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(540000, 32, 32, 1)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 60
        }
      ]
    },
    {
      "metadata": {
        "id": "MbueMHwfEbi4",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "EPOCHS = 100"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "B0Gm4LylbU25",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import os\n",
        "logs_path = \"./logs/embedding/\"  # path to the folder that we want to save the logs for Tensorboard"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "b8PfIhXqbayL",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "merged = tf.summary.merge_all()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "dyifXDRlcE2s",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "BATCH_SIZE = 512"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "IAw1R-c0cGsn",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "EPOCHS = 100"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "etcSVRRS1y9Z",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# total_steps_for_one_pass"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "o6n9Cv8Wbf_K",
        "colab_type": "code",
        "outputId": "bcf8af6b-caed-431c-a1c4-a10360bb93fb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 8568
        }
      },
      "cell_type": "code",
      "source": [
        "validation_accuracy_track = []\n",
        "train_accuracy_track = []\n",
        "connection_probability_track = []\n",
        "print_every = 1\n",
        "number_of_ex = X_train.shape[0]\n",
        "total_steps_for_one_pass = number_of_ex//BATCH_SIZE + 1\n",
        "\n",
        "with tf.Session() as sess:\n",
        "    sess.run(tf.global_variables_initializer())\n",
        "    num_examples = len(X_train)\n",
        "    best_accuracy_valid = 0\n",
        "    \n",
        "    print(\"Training...\")\n",
        "    print()\n",
        "    for i in range(EPOCHS):\n",
        "        X_train, y_train = shuffle(X_train, y_train)\n",
        "        X_train, y_train = shuffle(images, labels)\n",
        "\n",
        "        for offset in range(0, num_examples, BATCH_SIZE):\n",
        "          end = offset + BATCH_SIZE\n",
        "          batch_x, batch_y = X_train[offset:end], y_train[offset:end]\n",
        "          \n",
        "#         for step in range(0, total_steps_for_one_pass):  \n",
        "# #           print(step)\n",
        "#           if step>=number_of_ex//BATCH_SIZE:\n",
        "#             batch_x, batch_y = X_train[step*BATCH_SIZE:,:,:,:],y_train[step*BATCH_SIZE:]\n",
        "# #             print(step,'Finishing',step*BATCH_SIZE )\n",
        "#             step = 0\n",
        "\n",
        "#           else:\n",
        "\n",
        "#             start = step*BATCH_SIZE\n",
        "#             finish = (step+1)*BATCH_SIZE\n",
        "# #             print(step,'Doing', 'Start = ', start, \"Finish = \", finish)\n",
        "#             batch_x, batch_y = X_train[step:finish,:,:,:],y_train[step:finish]\n",
        "  \n",
        "          tr_op = sess.run([training_operation], feed_dict={x: batch_x, y: batch_y, is_testing : False})\n",
        "#         print(i)  \n",
        "        if i%print_every == 0:\n",
        "          prob = sess.run(new_prob)\n",
        "          print(prob)\n",
        "          validation_accuracy = evaluate(X_validation, y_validation)\n",
        "          validation_accuracy_track.append(validation_accuracy)\n",
        "          tr_accuracy = evaluate(X_train, y_train)\n",
        "          print(\"Train Accuracy = {:.5f}\".format(tr_accuracy))\n",
        "          train_accuracy_track.append(tr_accuracy)\n",
        "          connection_probability_track.append(prob)\n",
        "          print(\"EPOCH {} ...\".format(i+1))\n",
        "          print(\"Validation Accuracy = {:.5f}\".format(validation_accuracy))\n",
        "          print()\n",
        "          if (validation_accuracy >= best_accuracy_valid):\n",
        "            best_accuracy_valid = validation_accuracy\n",
        "            saver.save(sess, './lenet5')\n",
        "        \n",
        "#     saver.save(sess, './lenet')\n",
        "    print(\"Model saved\")"
      ],
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training...\n",
            "\n",
            "0.45\n",
            "Train Accuracy = 0.93708\n",
            "EPOCH 1 ...\n",
            "Validation Accuracy = 0.93677\n",
            "\n",
            "0.225\n",
            "Train Accuracy = 0.97996\n",
            "EPOCH 2 ...\n",
            "Validation Accuracy = 0.97908\n",
            "\n",
            "0.1125\n",
            "Train Accuracy = 0.98826\n",
            "EPOCH 3 ...\n",
            "Validation Accuracy = 0.98722\n",
            "\n",
            "0.05625\n",
            "Train Accuracy = 0.98852\n",
            "EPOCH 4 ...\n",
            "Validation Accuracy = 0.98685\n",
            "\n",
            "0.028125\n",
            "Train Accuracy = 0.99041\n",
            "EPOCH 5 ...\n",
            "Validation Accuracy = 0.98877\n",
            "\n",
            "0.0140625\n",
            "Train Accuracy = 0.99329\n",
            "EPOCH 6 ...\n",
            "Validation Accuracy = 0.99097\n",
            "\n",
            "0.00703125\n",
            "Train Accuracy = 0.99416\n",
            "EPOCH 7 ...\n",
            "Validation Accuracy = 0.99258\n",
            "\n",
            "0.003515625\n",
            "Train Accuracy = 0.99484\n",
            "EPOCH 8 ...\n",
            "Validation Accuracy = 0.99297\n",
            "\n",
            "0.0017578125\n",
            "Train Accuracy = 0.99540\n",
            "EPOCH 9 ...\n",
            "Validation Accuracy = 0.99340\n",
            "\n",
            "0.0008789062\n",
            "Train Accuracy = 0.99607\n",
            "EPOCH 10 ...\n",
            "Validation Accuracy = 0.99438\n",
            "\n",
            "0.0004394531\n",
            "Train Accuracy = 0.99719\n",
            "EPOCH 11 ...\n",
            "Validation Accuracy = 0.99528\n",
            "\n",
            "0.00021972656\n",
            "Train Accuracy = 0.99692\n",
            "EPOCH 12 ...\n",
            "Validation Accuracy = 0.99503\n",
            "\n",
            "0.00010986328\n",
            "Train Accuracy = 0.99676\n",
            "EPOCH 13 ...\n",
            "Validation Accuracy = 0.99447\n",
            "\n",
            "5.493164e-05\n",
            "Train Accuracy = 0.99504\n",
            "EPOCH 14 ...\n",
            "Validation Accuracy = 0.99265\n",
            "\n",
            "2.746582e-05\n",
            "Train Accuracy = 0.99797\n",
            "EPOCH 15 ...\n",
            "Validation Accuracy = 0.99582\n",
            "\n",
            "1.373291e-05\n",
            "Train Accuracy = 0.99801\n",
            "EPOCH 16 ...\n",
            "Validation Accuracy = 0.99588\n",
            "\n",
            "6.866455e-06\n",
            "Train Accuracy = 0.99792\n",
            "EPOCH 17 ...\n",
            "Validation Accuracy = 0.99540\n",
            "\n",
            "3.4332274e-06\n",
            "Train Accuracy = 0.99823\n",
            "EPOCH 18 ...\n",
            "Validation Accuracy = 0.99575\n",
            "\n",
            "1.7166137e-06\n",
            "Train Accuracy = 0.99805\n",
            "EPOCH 19 ...\n",
            "Validation Accuracy = 0.99570\n",
            "\n",
            "8.5830686e-07\n",
            "Train Accuracy = 0.99847\n",
            "EPOCH 20 ...\n",
            "Validation Accuracy = 0.99643\n",
            "\n",
            "4.2915343e-07\n",
            "Train Accuracy = 0.99842\n",
            "EPOCH 21 ...\n",
            "Validation Accuracy = 0.99597\n",
            "\n",
            "2.1457672e-07\n",
            "Train Accuracy = 0.99867\n",
            "EPOCH 22 ...\n",
            "Validation Accuracy = 0.99627\n",
            "\n",
            "1.0728836e-07\n",
            "Train Accuracy = 0.99813\n",
            "EPOCH 23 ...\n",
            "Validation Accuracy = 0.99567\n",
            "\n",
            "5.364418e-08\n",
            "Train Accuracy = 0.99880\n",
            "EPOCH 24 ...\n",
            "Validation Accuracy = 0.99643\n",
            "\n",
            "2.682209e-08\n",
            "Train Accuracy = 0.99850\n",
            "EPOCH 25 ...\n",
            "Validation Accuracy = 0.99648\n",
            "\n",
            "1.3411045e-08\n",
            "Train Accuracy = 0.99789\n",
            "EPOCH 26 ...\n",
            "Validation Accuracy = 0.99540\n",
            "\n",
            "6.7055224e-09\n",
            "Train Accuracy = 0.99873\n",
            "EPOCH 27 ...\n",
            "Validation Accuracy = 0.99672\n",
            "\n",
            "3.3527612e-09\n",
            "Train Accuracy = 0.99766\n",
            "EPOCH 28 ...\n",
            "Validation Accuracy = 0.99522\n",
            "\n",
            "1.6763806e-09\n",
            "Train Accuracy = 0.99792\n",
            "EPOCH 29 ...\n",
            "Validation Accuracy = 0.99575\n",
            "\n",
            "8.381903e-10\n",
            "Train Accuracy = 0.99761\n",
            "EPOCH 30 ...\n",
            "Validation Accuracy = 0.99530\n",
            "\n",
            "4.1909515e-10\n",
            "Train Accuracy = 0.99812\n",
            "EPOCH 31 ...\n",
            "Validation Accuracy = 0.99583\n",
            "\n",
            "2.0954757e-10\n",
            "Train Accuracy = 0.99856\n",
            "EPOCH 32 ...\n",
            "Validation Accuracy = 0.99625\n",
            "\n",
            "1.0477379e-10\n",
            "Train Accuracy = 0.99896\n",
            "EPOCH 33 ...\n",
            "Validation Accuracy = 0.99672\n",
            "\n",
            "5.2386893e-11\n",
            "Train Accuracy = 0.99812\n",
            "EPOCH 34 ...\n",
            "Validation Accuracy = 0.99575\n",
            "\n",
            "2.6193447e-11\n",
            "Train Accuracy = 0.99862\n",
            "EPOCH 35 ...\n",
            "Validation Accuracy = 0.99625\n",
            "\n",
            "1.3096723e-11\n",
            "Train Accuracy = 0.99902\n",
            "EPOCH 36 ...\n",
            "Validation Accuracy = 0.99703\n",
            "\n",
            "6.5483617e-12\n",
            "Train Accuracy = 0.99836\n",
            "EPOCH 37 ...\n",
            "Validation Accuracy = 0.99603\n",
            "\n",
            "3.2741808e-12\n",
            "Train Accuracy = 0.99873\n",
            "EPOCH 38 ...\n",
            "Validation Accuracy = 0.99640\n",
            "\n",
            "1.6370904e-12\n",
            "Train Accuracy = 0.99810\n",
            "EPOCH 39 ...\n",
            "Validation Accuracy = 0.99605\n",
            "\n",
            "8.185452e-13\n",
            "Train Accuracy = 0.99811\n",
            "EPOCH 40 ...\n",
            "Validation Accuracy = 0.99545\n",
            "\n",
            "4.092726e-13\n",
            "Train Accuracy = 0.99904\n",
            "EPOCH 41 ...\n",
            "Validation Accuracy = 0.99673\n",
            "\n",
            "2.046363e-13\n",
            "Train Accuracy = 0.99864\n",
            "EPOCH 42 ...\n",
            "Validation Accuracy = 0.99653\n",
            "\n",
            "1.0231815e-13\n",
            "Train Accuracy = 0.99829\n",
            "EPOCH 43 ...\n",
            "Validation Accuracy = 0.99607\n",
            "\n",
            "5.1159076e-14\n",
            "Train Accuracy = 0.99906\n",
            "EPOCH 44 ...\n",
            "Validation Accuracy = 0.99708\n",
            "\n",
            "2.5579538e-14\n",
            "Train Accuracy = 0.99906\n",
            "EPOCH 45 ...\n",
            "Validation Accuracy = 0.99677\n",
            "\n",
            "1.2789769e-14\n",
            "Train Accuracy = 0.99891\n",
            "EPOCH 46 ...\n",
            "Validation Accuracy = 0.99680\n",
            "\n",
            "6.3948845e-15\n",
            "Train Accuracy = 0.99873\n",
            "EPOCH 47 ...\n",
            "Validation Accuracy = 0.99647\n",
            "\n",
            "3.1974422e-15\n",
            "Train Accuracy = 0.99866\n",
            "EPOCH 48 ...\n",
            "Validation Accuracy = 0.99592\n",
            "\n",
            "1.5987211e-15\n",
            "Train Accuracy = 0.99939\n",
            "EPOCH 49 ...\n",
            "Validation Accuracy = 0.99730\n",
            "\n",
            "7.9936056e-16\n",
            "Train Accuracy = 0.99903\n",
            "EPOCH 50 ...\n",
            "Validation Accuracy = 0.99678\n",
            "\n",
            "3.9968028e-16\n",
            "Train Accuracy = 0.99933\n",
            "EPOCH 51 ...\n",
            "Validation Accuracy = 0.99715\n",
            "\n",
            "1.9984014e-16\n",
            "Train Accuracy = 0.99938\n",
            "EPOCH 52 ...\n",
            "Validation Accuracy = 0.99753\n",
            "\n",
            "9.992007e-17\n",
            "Train Accuracy = 0.99881\n",
            "EPOCH 53 ...\n",
            "Validation Accuracy = 0.99670\n",
            "\n",
            "4.9960035e-17\n",
            "Train Accuracy = 0.99918\n",
            "EPOCH 54 ...\n",
            "Validation Accuracy = 0.99670\n",
            "\n",
            "2.4980017e-17\n",
            "Train Accuracy = 0.99876\n",
            "EPOCH 55 ...\n",
            "Validation Accuracy = 0.99690\n",
            "\n",
            "1.2490009e-17\n",
            "Train Accuracy = 0.99866\n",
            "EPOCH 56 ...\n",
            "Validation Accuracy = 0.99622\n",
            "\n",
            "6.2450043e-18\n",
            "Train Accuracy = 0.99884\n",
            "EPOCH 57 ...\n",
            "Validation Accuracy = 0.99617\n",
            "\n",
            "3.1225022e-18\n",
            "Train Accuracy = 0.99923\n",
            "EPOCH 58 ...\n",
            "Validation Accuracy = 0.99677\n",
            "\n",
            "1.5612511e-18\n",
            "Train Accuracy = 0.99947\n",
            "EPOCH 59 ...\n",
            "Validation Accuracy = 0.99745\n",
            "\n",
            "7.8062554e-19\n",
            "Train Accuracy = 0.99797\n",
            "EPOCH 60 ...\n",
            "Validation Accuracy = 0.99577\n",
            "\n",
            "3.9031277e-19\n",
            "Train Accuracy = 0.99930\n",
            "EPOCH 61 ...\n",
            "Validation Accuracy = 0.99668\n",
            "\n",
            "1.9515639e-19\n",
            "Train Accuracy = 0.99898\n",
            "EPOCH 62 ...\n",
            "Validation Accuracy = 0.99668\n",
            "\n",
            "9.757819e-20\n",
            "Train Accuracy = 0.99926\n",
            "EPOCH 63 ...\n",
            "Validation Accuracy = 0.99688\n",
            "\n",
            "4.8789096e-20\n",
            "Train Accuracy = 0.99888\n",
            "EPOCH 64 ...\n",
            "Validation Accuracy = 0.99658\n",
            "\n",
            "2.4394548e-20\n",
            "Train Accuracy = 0.99814\n",
            "EPOCH 65 ...\n",
            "Validation Accuracy = 0.99572\n",
            "\n",
            "1.2197274e-20\n",
            "Train Accuracy = 0.99940\n",
            "EPOCH 66 ...\n",
            "Validation Accuracy = 0.99722\n",
            "\n",
            "6.098637e-21\n",
            "Train Accuracy = 0.99878\n",
            "EPOCH 67 ...\n",
            "Validation Accuracy = 0.99690\n",
            "\n",
            "3.0493185e-21\n",
            "Train Accuracy = 0.99908\n",
            "EPOCH 68 ...\n",
            "Validation Accuracy = 0.99682\n",
            "\n",
            "1.5246593e-21\n",
            "Train Accuracy = 0.99924\n",
            "EPOCH 69 ...\n",
            "Validation Accuracy = 0.99687\n",
            "\n",
            "7.6232963e-22\n",
            "Train Accuracy = 0.99904\n",
            "EPOCH 70 ...\n",
            "Validation Accuracy = 0.99683\n",
            "\n",
            "3.8116482e-22\n",
            "Train Accuracy = 0.99943\n",
            "EPOCH 71 ...\n",
            "Validation Accuracy = 0.99730\n",
            "\n",
            "1.9058241e-22\n",
            "Train Accuracy = 0.99948\n",
            "EPOCH 72 ...\n",
            "Validation Accuracy = 0.99722\n",
            "\n",
            "9.5291204e-23\n",
            "Train Accuracy = 0.99934\n",
            "EPOCH 73 ...\n",
            "Validation Accuracy = 0.99718\n",
            "\n",
            "4.7645602e-23\n",
            "Train Accuracy = 0.99910\n",
            "EPOCH 74 ...\n",
            "Validation Accuracy = 0.99683\n",
            "\n",
            "2.3822801e-23\n",
            "Train Accuracy = 0.99934\n",
            "EPOCH 75 ...\n",
            "Validation Accuracy = 0.99732\n",
            "\n",
            "1.19114005e-23\n",
            "Train Accuracy = 0.99913\n",
            "EPOCH 76 ...\n",
            "Validation Accuracy = 0.99685\n",
            "\n",
            "5.9557003e-24\n",
            "Train Accuracy = 0.99960\n",
            "EPOCH 77 ...\n",
            "Validation Accuracy = 0.99758\n",
            "\n",
            "2.9778501e-24\n",
            "Train Accuracy = 0.99939\n",
            "EPOCH 78 ...\n",
            "Validation Accuracy = 0.99737\n",
            "\n",
            "1.4889251e-24\n",
            "Train Accuracy = 0.99918\n",
            "EPOCH 79 ...\n",
            "Validation Accuracy = 0.99715\n",
            "\n",
            "7.4446253e-25\n",
            "Train Accuracy = 0.99954\n",
            "EPOCH 80 ...\n",
            "Validation Accuracy = 0.99765\n",
            "\n",
            "3.7223127e-25\n",
            "Train Accuracy = 0.99890\n",
            "EPOCH 81 ...\n",
            "Validation Accuracy = 0.99678\n",
            "\n",
            "1.8611563e-25\n",
            "Train Accuracy = 0.99901\n",
            "EPOCH 82 ...\n",
            "Validation Accuracy = 0.99667\n",
            "\n",
            "9.3057816e-26\n",
            "Train Accuracy = 0.99965\n",
            "EPOCH 83 ...\n",
            "Validation Accuracy = 0.99772\n",
            "\n",
            "4.6528908e-26\n",
            "Train Accuracy = 0.99902\n",
            "EPOCH 84 ...\n",
            "Validation Accuracy = 0.99720\n",
            "\n",
            "2.3264454e-26\n",
            "Train Accuracy = 0.99898\n",
            "EPOCH 85 ...\n",
            "Validation Accuracy = 0.99700\n",
            "\n",
            "1.1632227e-26\n",
            "Train Accuracy = 0.99936\n",
            "EPOCH 86 ...\n",
            "Validation Accuracy = 0.99743\n",
            "\n",
            "5.8161135e-27\n",
            "Train Accuracy = 0.99935\n",
            "EPOCH 87 ...\n",
            "Validation Accuracy = 0.99700\n",
            "\n",
            "2.9080568e-27\n",
            "Train Accuracy = 0.99939\n",
            "EPOCH 88 ...\n",
            "Validation Accuracy = 0.99743\n",
            "\n",
            "1.4540284e-27\n",
            "Train Accuracy = 0.99948\n",
            "EPOCH 89 ...\n",
            "Validation Accuracy = 0.99772\n",
            "\n",
            "7.270142e-28\n",
            "Train Accuracy = 0.99930\n",
            "EPOCH 90 ...\n",
            "Validation Accuracy = 0.99725\n",
            "\n",
            "3.635071e-28\n",
            "Train Accuracy = 0.99944\n",
            "EPOCH 91 ...\n",
            "Validation Accuracy = 0.99753\n",
            "\n",
            "1.8175355e-28\n",
            "Train Accuracy = 0.99926\n",
            "EPOCH 92 ...\n",
            "Validation Accuracy = 0.99705\n",
            "\n",
            "9.0876774e-29\n",
            "Train Accuracy = 0.99899\n",
            "EPOCH 93 ...\n",
            "Validation Accuracy = 0.99697\n",
            "\n",
            "4.5438387e-29\n",
            "Train Accuracy = 0.99913\n",
            "EPOCH 94 ...\n",
            "Validation Accuracy = 0.99718\n",
            "\n",
            "2.2719193e-29\n",
            "Train Accuracy = 0.99951\n",
            "EPOCH 95 ...\n",
            "Validation Accuracy = 0.99752\n",
            "\n",
            "1.1359597e-29\n",
            "Train Accuracy = 0.99901\n",
            "EPOCH 96 ...\n",
            "Validation Accuracy = 0.99710\n",
            "\n",
            "5.6797984e-30\n",
            "Train Accuracy = 0.99949\n",
            "EPOCH 97 ...\n",
            "Validation Accuracy = 0.99760\n",
            "\n",
            "2.8398992e-30\n",
            "Train Accuracy = 0.99929\n",
            "EPOCH 98 ...\n",
            "Validation Accuracy = 0.99692\n",
            "\n",
            "1.4199496e-30\n",
            "Train Accuracy = 0.99965\n",
            "EPOCH 99 ...\n",
            "Validation Accuracy = 0.99783\n",
            "\n",
            "7.099748e-31\n",
            "Train Accuracy = 0.99961\n",
            "EPOCH 100 ...\n",
            "Validation Accuracy = 0.99735\n",
            "\n",
            "Model saved\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "LMkW97BfhrFk",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "77ad1255-3ef1-4c4c-8026-64f08de3a242"
      },
      "cell_type": "code",
      "source": [
        "best_accuracy_valid"
      ],
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9978333333333333"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 68
        }
      ]
    },
    {
      "metadata": {
        "id": "bKU1MIyJCy6Y",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# with tf.Session() as sess:\n",
        "#     sess.run(tf.global_variables_initializer())\n",
        "#     num_examples = len(X_train)\n",
        "#     best_accuracy_valid = 0\n",
        "    \n",
        "#     print(\"Training...\")\n",
        "#     print()\n",
        "#     for i in range(EPOCHS):\n",
        "#         X_train, y_train = shuffle(images, labels)\n",
        "#         for offset in range(0, num_examples, BATCH_SIZE):\n",
        "#             end = offset + BATCH_SIZE\n",
        "#             batch_x, batch_y = X_train[offset:end], y_train[offset:end]\n",
        "#             tr_op = sess.run([training_operation], feed_dict={x: batch_x, y: batch_y, is_testing : False})\n",
        "#             \n",
        "#             prob = sess.run(new_prob)\n",
        "#         print(prob)\n",
        "#         validation_accuracy = evaluate(X_validation, y_validation)\n",
        "#         print(\"EPOCH {} ...\".format(i+1))\n",
        "#         print(\"Validation Accuracy = {:.3f}\".format(validation_accuracy))\n",
        "#         print()\n",
        "#         if (validation_accuracy >= best_accuracy_valid):\n",
        "#           best_accuracy_valid = validation_accuracy\n",
        "#           saver.save(sess, './lenet')\n",
        "        \n",
        "# #     saver.save(sess, './lenet')\n",
        "#     print(\"Model saved\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "yVIeYA_cCy6e",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Evaluate the Model\n",
        "Once you are completely satisfied with your model, evaluate the performance of the model on the test set.\n",
        "\n",
        "Be sure to only do this once!\n",
        "\n",
        "If you were to measure the performance of your trained model on the test set, then improve your model, and then measure the performance of your model on the test set again, that would invalidate your test results. You wouldn't get a true measure of how well your model would perform against real data.\n",
        "\n",
        "You do not need to modify this section."
      ]
    },
    {
      "metadata": {
        "id": "O4b5lSD5H_q9",
        "colab_type": "code",
        "outputId": "50ec1fd9-cab7-46f4-c4ff-bb042664ca55",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "cell_type": "code",
      "source": [
        "with tf.Session() as sess:\n",
        "#     saver.restore(sess, tf.train.latest_checkpoint('.'))\n",
        "    saver.restore(sess, './lenet5')\n",
        "    test_accuracy = evaluate(X_validation, y_validation)\n",
        "    print(\"Validation Accuracy = {:.5f}\".format(test_accuracy))"
      ],
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Restoring parameters from ./lenet5\n",
            "Validation Accuracy = 0.99783\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "bJONJ7CeCy6f",
        "colab_type": "code",
        "outputId": "0c224f59-cde3-41b7-d40c-63f00c216766",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "cell_type": "code",
      "source": [
        "with tf.Session() as sess:\n",
        "#     saver.restore(sess, tf.train.latest_checkpoint('.'))\n",
        "    saver.restore(sess, './lenet5')\n",
        "    test_accuracy = evaluate(X_test, y_test)\n",
        "    print(\"Test Accuracy = {:.5f}\".format(test_accuracy))"
      ],
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Restoring parameters from ./lenet5\n",
            "Test Accuracy = 0.99240\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "1yZ8wJByouiu",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 286
        },
        "outputId": "b33606fc-e79e-477f-a33f-b9ed6d599452"
      },
      "cell_type": "code",
      "source": [
        "plt.plot(validation_accuracy_track)"
      ],
      "execution_count": 76,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7f9d8053ab00>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 76
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8lOW99/HPLytLgLCELYSwiCxF\ndhBXRGuLXURxqVq1drO2tU/PaT092j71nGNrfdpjF22t1mOpemq1Sq1SpVXLokWrArKLSNiTgIQl\nAUKS2X7PHzNJhzBJBgiE3vm+X6+8mHuZmevihm+u+d3XfY+5OyIi0j5ktHUDRETk5FHoi4i0Iwp9\nEZF2RKEvItKOKPRFRNoRhb6ISDui0BcRaUcU+iIi7YhCX0SkHclqaQczmw18Atjl7qNTbDfgPuBj\nwCHgJnd/J7HtM8D/Tez6fXd/rKX369Wrlw8aNCjtDoiICCxbtmy3uxe0tF+LoQ88CvwCeLyJ7ZcA\nwxI/ZwIPAmeaWQ/gP4BJgAPLzGyuu+9r7s0GDRrE0qVL02iWiIjUM7Ot6ezXYnnH3V8D9jazy0zg\ncY97E8g3s37AR4FX3H1vIuhfAWak0ygRETkxWqOmXwhsT1ouTaxrav0RzOxmM1tqZksrKipaoUki\nIpLKKXEi190fdvdJ7j6poKDFkpSIiByj1gj9MqAoaXlAYl1T60VEpI20RujPBW60uKlAlbvvAF4C\nPmJm3c2sO/CRxDoREWkj6UzZfBK4AOhlZqXEZ+RkA7j7Q8A84tM1S4hP2fxsYtteM/sesCTxUne5\ne3MnhEVE5ARrMfTd/doWtjvw1Sa2zQZmH1vTRESktZ0SJ3JFRI5WdV2EWOz4v+41HI1xKBRphRYd\nvYXrd/Hy2p2t0o90pXNxlog04u6sLK3izU17uO7MgXTtkN1qr11xoI7MDKNH55xWe81TxeINu/nj\n8jKG981jYnEPRhd2JTcrs2F7TSjKo29s4Q/vlHLbR05nxuh+KV9nTVkV1z78Jv3yO3D7JSOYPrw3\n8ZsDHK7yUAjD6Nbp8ONzoDbM/HW7mP/eLl5dv4vacIxPTS7ilguGUpjfkeq6CItLdrN1TzWXjS+k\nd5cOLfZt+95DvLahgvFF3RnVv2vD+vU7D/DQqxsZPzCfG6YWN7Tzd29t49t/XA3AqH5d+deLT+fD\nI1P3ozXZqfbF6JMmTXJdkSutIRyNkWlGRsbR/yda8N4HzFlWypfOH8rYovyG9Qdqw8xdWc7v3trG\n2vL9AIzo24VHPzuFvt1aDobmuDvPLCvlP+eupTC/I/O+fh7Zma3zYbwmFOXJt7cxtiif8UX5R/V3\nsqasil+9tol91SF65uXQKy+Xrh2yyc3OoENWBkN753H20F5kJl5zR1UNv1y4kfLKGm46ZxDnntYL\nd/j5ghJ+Nv99OmVnUh2KApCTlcGYwm5MLO5O9845zF68mV0H6uiVl8ve6jq+f9kZXHfmwMPas3VP\nNVc8+AY5mRnkZGWwZc8hzhzcg+unFjOuKJ8B3TuyseIgD7+2ieeWl4PBFRMK+cJ5Q+iUk8lvXt/C\nk29t40BdhF55OUwf3psMM55dXgrAmAH5rC6tIhSNAdAhO4MbzxrEl84fQs+83IZ2RGPOhl0H+PvG\nPbywagfLtv7jZgNnDenJNVOKWLS+gudWlJGVYYSjzvThBfz3VWP508py/utP7zJ9eAEfH9Ofny/Y\nwNY9h5g6pAdPfnHqMQW/mS1z90kt7qfQl5MtFnPuX7CBResreODTEyjM79jq71FWWcPVD/2dUDTG\nJaP78rEz+nF6ny7UhqPUhqP06pLb5Oh8Z1UtH/npq+yvjX/k/8SYflw2rpA/r9nJvNU7qAlHGdmv\nK9edOZA+XXL5xtMr6dIhi998djIj+v5jhHegNszqsipKdh1k+vDeFPXo1GR7q2rCfPuPq3lx1Q6G\n9c5jw66D3DXzQ9x41qBm+7m3OsTK0kr2VYc4b1gBBV1yU+73/Rfe5ZHFmwHo360DHx3dlyEFeRTk\n5dC9Uw7hqHOwLkJ1XYTMDKNDdvyXzdNLS1nw3i66dshiaO889hwMsftgHYcSoV2vd5dcLh9fSF0k\nxu/e2objdOuYw+6DdYwfmE/nnCwWl+xm1vhCvn/5aA7WRXhnayXLtu5l2dZ9rC6rIhx1Jg/qzrdm\njOBD/bvylSfeYdH6Cr5x8el8+YKhZGdmUHGgjisfeoP9NWGeueVsint24sm3t3H//A3sPhgCIL9T\nNpWHwuRmZXDlxAE4MGdZacMgIObOx8f056azixlf1L3hF2BZZQ0PLdrIO9v2MXVITy4a2ZveXXL5\n5cKNPLeijAwzenfJpWdeLh1zMllXvp8DdfF/IyP6duHScf25cERvXl1fwWNvbKG8qpYO2RncdPZg\nvnT+EP60qpzvv7iODlkZ7K+NMONDfbn/2vHkZGUQicZ4dnkZ1XURPnvO4GaPeVMU+nIYd6c2HONg\nXYSsDKN7o9LB9r2H+ObTK7nto8OZMrjHMb/Psq37+OXCEjrkZPLjq8bSITvzsO3VdRG+8fQKXlr7\nAVkZxsAenfj9l85qMqxS2bbnEDfOfotunXIYN6AbY4vyuWhkH7p1jId4xYE6rv7V39l9sI6zh/Zk\n0foK6iKxw14jO9O4YHhvZo7rz0Uj+tAxJ95Od+fG2W+zdMs+nrnlLF5au5NH/raZmnCUvNwsPjm2\nH1dPKmJcUX7DaOzd8v189tG32V8TobB7/BdYKBJj+75D1P/3ysvN4u7LRzNz3JEXpb/y7gfc+fwa\ndh2o4xsXn84t04Zy/SNv8d7O/Sz6t+kN/frN65t5bkV5w/P2VtexfW9Nw7IZTBjYnU+O6ccNZw1q\nGHmvLq1i5gOLuWLCAM4a2pN5q3fw2vu7G0ayzcnvlM0XzxvCDWcVH/ZLMhpz6iJRDoWiLN2ylz+8\nU8bC93bhwFUTB/DV6afRu2suc5aV8suFG6k4UMd/XDqK66YMTDmKrQ1H2VlVS3HPTg3bw9EY/z5n\nFc8uLyPDoE/XDkRjzv7aML/74lQmDOze8PxwNMb6nQdYsb2SVaWVFOZ34vqpAxtG5hUH6njira3U\nhKNcf2Zxs7+AUynZdYA/vFPGrv117D5Yx8G6CMP7dmFScXcmFnenuGfnw/aPRGO8vXkvQ3vn0afr\nPz4BvrdzP9+as4oRfbvwg8vPIKuVPsmBQj+w9laH+LdnVrJt7yFumTaUmeP6k5WZQTTmLN2yl/21\nES4a0bth9BKLOQ8sLOEXC0sagi/D4CdXj+Oy8fEAikRjXPWrv7N8WyUDe3TipX85vyEE0+HuvLlp\nLw8sLGFxyW66dcymqibM9OEFPHTDxIaa7Zbd1Xz5iXdYv3M/3/7YSMYPzOf6R96muGcnnrp5KuWV\ntTy9dDvLtu7j/NN7MWvCAIYW5B32XqFIjKseeoNNu6sZ1a8rq8uqOBSK0iU3i5vOGcRVE4v40m+X\nsWV3Nb/9whQmFvegui7CovUVVByopUN2JrnZGawp28+fVpYnSgk5fOujI7hy4gCeeHsb331uDd+7\nbDQ3TC0GYNf+WlaVVnH2aT3plJP6NFh5ZQ33/XUDBxMjv4wMY1jvPMYW5dO7Sy7ffW4NS7fu4/Lx\nhVw7ZSB5uVlkZMB9f93An9fsZHifLvzwyjGMS5SS1pRV8clfLOYL5w7m2x8byU9feZ/7F5RwRmG3\nhlp/XocsxhR2Y8yAfLp0yGL+ul28tHYn7+7YzyWj+/LTT40jK8OY+cDrVByo45VvTGv4BRKJxthb\nHWL3wRB7q0NkZxp5HbLonJNF1J3acJRQJMbpfbrQOTe9U397q0NEYrEj6t+hSIz9tWF65aX/i71e\nLObMW7OD9z84SOm+Q+w5GOIL5w3mvGG6cr8xhf4prn7kXVkT4kBthIE9Oh0xKm5s5fZKvvLEO1Qc\nqGNgz06U7DpIcc9OTBzYnUXvV7C3Ov7xdlJxd+6ZdQb98zvyzadX8pe1O/nIqD6MG5hPl9ws/rRy\nB8u37+Oxz03h7KG9+PHL6/n5ghJuOnsQj76xhS9fMJR/nzGixT7URaK8sHIHv168mXd37KdXXg43\nnz+ET59ZzPMryvn2H1fz4ZF9+M9LR/Hgoo38fsl2OmZn8vPrxnPB8N5A/MTe5x5dQm52BgdqI+Rk\nZjCqf1dWlVYScxg/MJ/bZ4zgzCE9Abj7xXf5n79t5sFPT+CSM/oRjTmrSit5+LVN/HnNTiA+in/k\nM5OZdnrzwRCNOW9u2sNPXnmfZVv3MbqwKxt3VTN5cA8e++zkVj2hFonG+MXCEu6fv4HkiRq5WRn8\nn4uGcfP5Q46o3//bMyt5bkUZl40r5JllpXxqUhE/mHVGwwi+KY/8bRPff3EdZw3pyZTBPbhv/oaG\nvy8JLoX+Kaw2HOXin7562EfzPl1zuXX6aVw9uYhMM/5Wspvnl5ex60AdeblZ5GZn8tKanRR0yeXB\n6ycwun83/rruA36+oIQte6qZPrw3H/1QX6rrIvzgz+uorovQt1sHyvbVcMclI/nCeYMbQqyqJsyV\nD77Bzv21/PuMEXz3+TVcMWEA9141lm/NWckf3injha+dy8h+XVlbXsUjf9vM4F6dufGsYvI75RCL\nOc+vLONHf1nPjqpahvXO43PnDuaycYWHfUL4379v4bvPrwUgK8O4dspAvnbhafTuevhIcMF7H/Dw\na5uY8aG+XDa+kPxOOezaX8vcleU8+sYWSvfVcM3kIqYO6cm//H4FN0wt5nuXHfHVDqzfeYDfvL6Z\ni0f14aKRfdI+Hu7O3JXl3DPvPWojUf7y9fOP+6RsU7buqWb73hoO1oWprosyeVAPBvZMXWr4YH8t\n0+9dxKFQlBumFvNfl34o7ROwz75TyrfmrCIScz48sg//c+PEEz4rRNqWQv8kisWcjRUHKUpjtA4w\nf90HfP6xpdx09iBO79OFnKwMnnp7G0u37qN/tw6EY07FgTryO2UztCCP6roIB2ojjC3qxg8uP4P8\nTs1P5dtzsI7vv7iO10t28+Orx6b8KFxWWcPlD7zOrgN1DO7VmRe+di6dc7OoPBTioh+/SmH3jozo\n24VnlpU2zLbonJPJ1ZOLWLZ1H6tKqzijsBu3fXQ45w/r1WSg/H7JNlZsr+LL04Y2GW7NORSKcN9f\nN/DI4s1EY87Ifl3541fOTuvv+WjVhqPUhKJHnO9oS/NW76C8sobPnzv4qEN74fpdzF68mR9eMYb+\nJ+BkuZxaFPonSSgS419/v4IXV+8gNyuDyYN6cN6wXlzbzNztb/9xNc8vL+OdOy9uqHe7O4tLdvOr\nVzfROTeTWRMGMH14b3Kyjv1Ej7s3GxRry6u460/v8t1PjGJ0YbeG9c+vKOPrT60gO9O46exB3Dp9\nGOVVNTy4aCMvrCqnT9cOfGvGcGaOLTym6ZDHYm15FY++voWvTj+NQb06t/wEkXZGoX8S1IajfPm3\ny1i4voJbpg0lFInxxsbdvLfzAL3ycrnjkhHMmlB4WPC6O2fds4DxA/N58PqJbdj6prk7L67ewRmF\n3Y6YlbCvOkTHnMwTMtIWkWOXbujritxjdLAuwhceW8Jbm/fyg8sPv4BkdWkV331+Dd98ZiVPvr2N\nh26Y2DBzYW35fnburz2qmvPJZmZ8Ykz/lNtOpdKHiBw93XvnGNSEonzu0SUs2bKPn1497ogrBs8Y\n0I1nv3w2P7piDCtLK/nJK+83bPvrug8wgwuGa8qZiJx8Cv2jVBuO8sXHl7J0y15+9ql/zHVvLCPD\nuHpyEddOGcjTS7azdU81APPX7WJ8Uf4xzVkWETleCv2jEIrE+MoT77C4ZDc/unIsnxybugSS7Nbp\np5GVafzsrxv4YH8tq8uqTunSjogEm2r6aTgUivD7Jdv59eLNlO6r4e7LR3PlxAFpPbd31w7cdPZg\nfvXaxoarIT+s0BeRNqLQb8aW3dU8uWQbT729naqaMJOKu/O9y0YzPXE1abpumTaEJ97cyqNvbGFA\n946c3iev5SeJiJwACv0UVpdW8cO/vMfikt1kZhgXj+zDF88fwsTi7i0/OYX8Tjl88fwh/OSV97lo\nxIm/X7aISFMU+o3UhKLc8ttl1EVifPPi07l6ctFhd8k7Vp87dzAbdh3k04mbeImItAWFfiMPvbqR\nssoanrp5KlMTN/lqDXm5Wfz82vGt9noiIsdCs3eSbN97iIde3cgnxvRr1cAXETlVKPST3P3iOjLM\n+PbHRrZ1U0RETgiFfsLiDbv5y9qdfHX6UN2RUEQCK63QN7MZZrbezErM7PYU24vNbL6ZrTKzRWY2\nIGnbD81sTeLnU63Z+Nbi7vzopfco6tGRL5w3pK2bIyJywrQY+maWCTwAXAKMAq41s1GNdrsXeNzd\nxwB3AfcknvtxYAIwDjgTuM3MunKKeWdbJatKq7j5vCG6e6SIBFo6I/0pQIm7b3L3EPAUMLPRPqOA\nBYnHC5O2jwJec/eIu1cDq4AZx9/s1vWb1zfTpUMWsyakd5WtiMg/q3RCvxDYnrRcmliXbCUwK/H4\ncqCLmfVMrJ9hZp3MrBcwHSg6via3rh1VNfx5zU6umVyU9hdAi4j8s2qtE7m3AdPMbDkwDSgDou7+\nMjAPeAN4Evg7EG38ZDO72cyWmtnSioqKVmpSen775lbcnRvPGnRS31dEpC2kE/plHD46H5BY18Dd\ny919lruPB76TWFeZ+PNudx/n7hcDBrxPI+7+sLtPcvdJBQUn7z7zteEov3trGx8e2YeiHkf//a0i\nIv9s0gn9JcAwMxtsZjnANcDc5B3MrJeZ1b/WHcDsxPrMRJkHMxsDjAFebq3GH6+5K8rZdyjMZ88Z\n3NZNERE5KVosYrt7xMxuBV4CMoHZ7r7WzO4Clrr7XOAC4B4zc+A14KuJp2cDf0vcYGw/cL27R1q/\nG0cvFnN+vXgzI/p2YeqQHm3dHBGRkyKtM5fuPo94bT553Z1Jj+cAc1I8r5b4DJ5Tzktrd7L+gwPc\nd8043fVSRNqNdnlFbizm3Dd/A0MKOjf5BeAiIkHULkP/5Xc/4L2dB/jahaeRmaFRvoi0H+0i9GvD\nUcLRGBC/5cL98zcwuFdnPqlRvoi0M+3iaqSP/uw1DtZGmDWhkP75HXl3x35+fNVYsjLbxe88EZEG\ngQ/9g3URtu45xMAenfjN61uIxJzinp2YOU6jfBFpfwIf+jsqawD45kdO55zTevHiqh2MLcrXKF9E\n2qXAh35ZIvQL8zvSKy+Xz5w9qG0bJCLShgI/3N1RVQtAP30xiohI8EO/vLKGDIM+XXLbuikiIm0u\n8KFfVllD364dVMMXEaEdhH55ZY2+81ZEJKEdhH6tQl9EJCHQoR+LOTuqNNIXEakX6NDffbCOcNQp\nzO/Q1k0RETklBDr06+foa6QvIhIX6NAvr0zM0e+m0BcRgYCH/o6qf1yNKyIiAQ/9ssoaOudk0rVj\n4O82ISKSlkCHfv0cfX0doohIXMBDX3P0RUSSBTz0NUdfRCRZYEO/NhxlT3VIc/RFRJIENvTLNUdf\nROQIaYW+mc0ws/VmVmJmt6fYXmxm881slZktMrMBSdt+ZGZrzWydmd1vJ+msav0cfYW+iMg/tBj6\nZpYJPABcAowCrjWzUY12uxd43N3HAHcB9ySeezZwDjAGGA1MBqa1Wuub0TDS14VZIiIN0hnpTwFK\n3H2Tu4eAp4CZjfYZBSxIPF6YtN2BDkAOkAtkAx8cb6PTUV5Vgxn06aYvTxERqZdO6BcC25OWSxPr\nkq0EZiUeXw50MbOe7v534r8EdiR+XnL3dcfX5PSUV9ZQkJdLblbmyXg7EZF/Cq11Ivc2YJqZLSde\nvikDomZ2GjASGED8F8WFZnZe4yeb2c1mttTMllZUVLRKgzRHX0TkSOmEfhlQlLQ8ILGugbuXu/ss\ndx8PfCexrpL4qP9Ndz/o7geBPwNnNX4Dd3/Y3Se5+6SCgoJj7MrhyitrdM8dEZFG0gn9JcAwMxts\nZjnANcDc5B3MrJeZ1b/WHcDsxONtxD8BZJlZNvFPASe8vOPulFXW0K+b5uiLiCRrMfTdPQLcCrxE\nPLCfdve1ZnaXmV2a2O0CYL2ZvQ/0Ae5OrJ8DbARWE6/7r3T3P7VuF44Ujjp1kRj5nbJP9FuJiPxT\nSev2k+4+D5jXaN2dSY/nEA/4xs+LAl86zjYetXA0BkB2ZmCvPRMROSaBTMVI1AGFvohIY4FMxVD9\nSD8rkN0TETlmgUzFhvJOhu6jLyKSLNihr/KOiMhhApmK4fqavso7IiKHCWQq1o/0czJV3hERSRbo\n0M/KCGT3RESOWSBTMazZOyIiKQUyFRtq+irviIgcJqChX1/TD2T3RESOWSBTsaGmr9AXETlMIFMx\nFFF5R0QklUCGfiSm8o6ISCqBTEWVd0REUgtkKoZV3hERSSmQoR/S7B0RkZQCmYoR3XBNRCSlQKZi\n/cVZWSrviIgcJpChH9JIX0QkpUCmou6nLyKSWiBTMRJ1MjOMTH1zlojIYQIZ+uFojCwFvojIEQIZ\n+qFoTNM1RURSCGQyhqMx3UtfRCSFtJLRzGaY2XozKzGz21NsLzaz+Wa2yswWmdmAxPrpZrYi6afW\nzC5r7U40Fom6rsYVEUmhxdA3s0zgAeASYBRwrZmNarTbvcDj7j4GuAu4B8DdF7r7OHcfB1wIHAJe\nbsX2pxSKxvRViSIiKaSTjFOAEnff5O4h4ClgZqN9RgELEo8XptgOcCXwZ3c/dKyNTVc46uSovCMi\ncoR0krEQ2J60XJpYl2wlMCvx+HKgi5n1bLTPNcCTqd7AzG42s6VmtrSioiKNJjUvHImpvCMikkJr\nDYdvA6aZ2XJgGlAGROs3mlk/4AzgpVRPdveH3X2Su08qKCg47sZEYjFdmCUikkJWGvuUAUVJywMS\n6xq4ezmJkb6Z5QFXuHtl0i5XA3909/DxNTc9oajrXvoiIimkk4xLgGFmNtjMcoiXaeYm72Bmvcys\n/rXuAGY3eo1raaK0cyKEIzFyVN4RETlCi6Hv7hHgVuKlmXXA0+6+1szuMrNLE7tdAKw3s/eBPsDd\n9c83s0HEPym82qotb4bKOyIiqaVT3sHd5wHzGq27M+nxHGBOE8/dwpEnfk+oUNTplKPQFxFpLJDJ\nqNk7IiKpBTP0oyrviIikEshkjMRcoS8ikkIgkzEU0UhfRCSVQCZjvLyjmr6ISGMBDv1Adk1E5LgE\nMhnjt1YOZNdERI5LIJMxFI2RnaXyjohIY4EM/XA0Rrbupy8icoTAJWM05sQclXdERFIIXDKGozEA\nlXdERFIIbOjnaKQvInKEwCVjOOoAZGVopC8i0lgAQ7++vBO4romIHLfAJWND6Ku8IyJyhMAlY315\nRzV9EZEjBS4Z60f6Wbr3jojIEQIX+qGIyjsiIk0JXDJGYirviIg0JXDJqBO5IiJNC1wyhiOq6YuI\nNCVwoR/SSF9EpEmBS8aIpmyKiDQprWQ0sxlmtt7MSszs9hTbi81svpmtMrNFZjYgadtAM3vZzNaZ\n2btmNqj1mn8kTdkUEWlai6FvZpnAA8AlwCjgWjMb1Wi3e4HH3X0McBdwT9K2x4H/dveRwBRgV2s0\nvCkq74iINC2dZJwClLj7JncPAU8BMxvtMwpYkHi8sH574pdDlru/AuDuB939UKu0vAm6IldEpGnp\nJGMhsD1puTSxLtlKYFbi8eVAFzPrCZwOVJrZs2a23Mz+O/HJ4YSJ6H76IiJNaq3h8G3ANDNbDkwD\nyoAokAWcl9g+GRgC3NT4yWZ2s5ktNbOlFRUVx9WQhpq+vi5RROQI6SRjGVCUtDwgsa6Bu5e7+yx3\nHw98J7GukvinghWJ0lAEeA6Y0PgN3P1hd5/k7pMKCgqOsStxIZV3RESalE4yLgGGmdlgM8sBrgHm\nJu9gZr3MrP617gBmJz0338zqk/xC4N3jb3bT9HWJIiJNazH0EyP0W4GXgHXA0+6+1szuMrNLE7td\nAKw3s/eBPsDdiedGiZd25pvZasCA/2n1XiSJaPaOiEiTstLZyd3nAfMarbsz6fEcYE4Tz30FGHMc\nbTwqIX1doohIkwI3HA5HY2RnGmYKfRGRxgIX+pFoTKUdEZEmBC4dw1FX6IuINCFw6RhKlHdERORI\ngQv9cETlHRGRpgQuHSMxlXdERJoSuHRUeUdEpGmBC32Vd0REmha4dAxryqaISJMCl47xmr7KOyIi\nqQQu9EMq74iINClw6ajyjohI0wKXjvErclXeERFJJYChr5G+iEhTApeO4WiM7KzAdUtEpFUELh3D\nUSdb99IXEUkpgKGv8o6ISFMCl47hqKu8IyLShMClYzgaI0cjfRGRlAKXjuFoTN+PKyLShECGvso7\nIiKpBSod3V1flygi0oxApWMk5gDk6IpcEZGU0gp9M5thZuvNrMTMbk+xvdjM5pvZKjNbZGYDkrZF\nzWxF4mduaza+sXA0BkCWRvoiIilltbSDmWUCDwAXA6XAEjOb6+7vJu12L/C4uz9mZhcC9wA3JLbV\nuPu4Vm53SuFIfKSv8o6ISGrppOMUoMTdN7l7CHgKmNlon1HAgsTjhSm2nxThWHykr/KOiEhq6YR+\nIbA9abk0sS7ZSmBW4vHlQBcz65lY7mBmS83sTTO77Lha24L68o5G+iIiqbVWOt4GTDOz5cA0oAyI\nJrYVu/sk4DrgZ2Y2tPGTzezmxC+GpRUVFcfciPryjmr6IiKppZOOZUBR0vKAxLoG7l7u7rPcfTzw\nncS6ysSfZYk/NwGLgPGN38DdH3b3Se4+qaCg4Fj6AUCoYaSv8o6ISCrphP4SYJiZDTazHOAa4LBZ\nOGbWy8zqX+sOYHZifXczy63fBzgHSD4B3KoiDTV9jfRFRFJpMR3dPQLcCrwErAOedve1ZnaXmV2a\n2O0CYL2ZvQ/0Ae5OrB8JLDWzlcRP8P6/RrN+WpVm74iINK/FKZsA7j4PmNdo3Z1Jj+cAc1I87w3g\njONsY9pCDfP0Vd4REUklUEPi+tk7Ku+IiKQWqHSMRBPlHd1wTUQkpUClo+bpi4g0L1Dp2FDT1/30\nRURSClToN9T0Vd4REUkpUOnYUNNXeUdEJKVApaPKOyIizQtU6Ku8IyLSvEClo8o7IiLNC1Q6hnXD\nNRGRZgUq9EOapy8i0qxApaN/L/5zAAAGuElEQVRuuCYi0rxApWMkFiPDIFOzd0REUgpU6IeiMY3y\nRUSaEaiEDEdcoS8i0oxAJWQ4GtPMHRGRZgQq9CMxlXdERJoTqIQMqbwjItKsQCWkyjsiIs0LYOgH\nqksiIq0qUAkZjqq8IyLSnEAlZDga0/fjiog0I1AJGY7GyNbVuCIiTQpe6Ku8IyLSpLQS0sxmmNl6\nMysxs9tTbC82s/lmtsrMFpnZgEbbu5pZqZn9orUanko46irviIg0o8WENLNM4AHgEmAUcK2ZjWq0\n273A4+4+BrgLuKfR9u8Brx1/c5sXjsbI0ZRNEZEmpTMsngKUuPsmdw8BTwEzG+0zCliQeLwwebuZ\nTQT6AC8ff3ObF47GyMrQSF9EpCnpJGQhsD1puTSxLtlKYFbi8eVAFzPraWYZwI+B25p7AzO72cyW\nmtnSioqK9Fqegso7IiLNa62EvA2YZmbLgWlAGRAFvgLMc/fS5p7s7g+7+yR3n1RQUHDMjdAVuSIi\nzctKY58yoChpeUBiXQN3Lycx0jezPOAKd680s7OA88zsK0AekGNmB939iJPBrSFe09dIX0SkKemE\n/hJgmJkNJh721wDXJe9gZr2Ave4eA+4AZgO4+6eT9rkJmHSiAh/i5Z0sjfRFRJrU4rDY3SPArcBL\nwDrgaXdfa2Z3mdmlid0uANab2fvET9refYLa26xwRPP0RUSak85IH3efB8xrtO7OpMdzgDktvMaj\nwKNH3cKjEI6pvCMi0pxAJaRuuCYi0rzAJGQ05kRjqumLiDQnMKEfjsYANNIXEWlGYBIyEnMA1fRF\nRJoRmIQMR+pH+irviIg0JTChn5FhfHxMPwYX5LV1U0RETllpTdn8Z9CtYzYPXDehrZshInJKC8xI\nX0REWqbQFxFpRxT6IiLtiEJfRKQdUeiLiLQjCn0RkXZEoS8i0o4o9EVE2hFz97Zuw2HMrALYehwv\n0QvY3UrN+WfRHvsM7bPf7bHP0D77fbR9Lnb3Fr9k/JQL/eNlZkvdfVJbt+Nkao99hvbZ7/bYZ2if\n/T5RfVZ5R0SkHVHoi4i0I0EM/YfbugFtoD32Gdpnv9tjn6F99vuE9DlwNX0REWlaEEf6IiLShMCE\nvpnNMLP1ZlZiZre3dXtOFDMrMrOFZvauma01s68n1vcws1fMbEPiz+5t3dbWZmaZZrbczF5ILA82\ns7cSx/z3ZpbT1m1sbWaWb2ZzzOw9M1tnZmcF/Vib2b8m/m2vMbMnzaxDEI+1mc02s11mtiZpXcpj\na3H3J/q/ysyO+ctDAhH6ZpYJPABcAowCrjWzUW3bqhMmAnzT3UcBU4GvJvp6OzDf3YcB8xPLQfN1\nYF3S8g+Bn7r7acA+4PNt0qoT6z7gL+4+AhhLvP+BPdZmVgj8H2CSu48GMoFrCOaxfhSY0WhdU8f2\nEmBY4udm4MFjfdNAhD4wBShx903uHgKeAma2cZtOCHff4e7vJB4fIB4ChcT7+1hit8eAy9qmhSeG\nmQ0APg48klg24EJgTmKXIPa5G3A+8GsAdw+5eyUBP9bEv9Gvo5llAZ2AHQTwWLv7a8DeRqubOrYz\ngcc97k0g38z6Hcv7BiX0C4HtSculiXWBZmaDgPHAW0Afd9+R2LQT6NNGzTpRfgZ8C4gllnsCle4e\nSSwH8ZgPBiqA3yTKWo+YWWcCfKzdvQy4F9hGPOyrgGUE/1jXa+rYtlrGBSX02x0zywP+APyLu+9P\n3ubxKVmBmZZlZp8Adrn7srZuy0mWBUwAHnT38UA1jUo5ATzW3YmPagcD/YHOHFkCaRdO1LENSuiX\nAUVJywMS6wLJzLKJB/4T7v5sYvUH9R/3En/uaqv2nQDnAJea2RbipbsLide68xMlAAjmMS8FSt39\nrcTyHOK/BIJ8rD8MbHb3CncPA88SP/5BP9b1mjq2rZZxQQn9JcCwxBn+HOInfua2cZtOiEQt+9fA\nOnf/SdKmucBnEo8/Azx/stt2orj7He4+wN0HET+2C9z908BC4MrEboHqM4C77wS2m9nwxKqLgHcJ\n8LEmXtaZamadEv/W6/sc6GOdpKljOxe4MTGLZypQlVQGOjruHogf4GPA+8BG4Dtt3Z4T2M9ziX/k\nWwWsSPx8jHiNez6wAfgr0KOt23qC+n8B8ELi8RDgbaAEeAbIbev2nYD+jgOWJo73c0D3oB9r4L+A\n94A1wP8CuUE81sCTxM9bhIl/qvt8U8cWMOIzFDcCq4nPbjqm99UVuSIi7UhQyjsiIpIGhb6ISDui\n0BcRaUcU+iIi7YhCX0SkHVHoi4i0Iwp9EZF2RKEvItKO/H9iI78erZepeAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "Mto2Wuj_o13d",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 286
        },
        "outputId": "70858be5-1c4f-4a13-d2cb-eceaf275bd53"
      },
      "cell_type": "code",
      "source": [
        "plt.plot(connection_probability_track)"
      ],
      "execution_count": 77,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7f9d80441da0>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 77
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAE49JREFUeJzt3X+MHHd5x/H3s7v3y8aXOMnxy2ew\nAfPDUELQNQlqhSCkagI0oS2oiVqVSkhRJVJCQWqDQFGb/lFBEG0pESWCUIoKKVDUWtSt24bQioqk\nuQAFkhAwTsA2gVyIifPLPt/d0z927rI+796tnT2vZ/b9ki7emf365hnN5ePvPTM7E5mJJKlaav0u\nQJLUe4a7JFWQ4S5JFWS4S1IFGe6SVEGGuyRVkOEuSRVkuEtSBRnuklRBjX5t+KyzzsotW7b0a/OS\nVEp33HHHg5k5sdq4voX7li1bmJ6e7tfmJamUIuKH3YyzLSNJFWS4S1IFGe6SVEGGuyRVkOEuSRVk\nuEtSBRnuklRBpQv32+97iA/uuoe5+YV+lyJJp6zShfs3fnSAj9yym0NzhrskdVK6cB9p1AE4fGS+\nz5VI0qmrhOHeLPmwM3dJ6qh84T5kuEvSasoX7ottmTnbMpLUSQnDvZi5H3HmLkmdlDDcF2fuhrsk\ndVK+cC967rOGuyR1VL5wX7paxp67JHVSwnC3LSNJqylhuDtzl6TVlC/ch7xaRpJWU75wty0jSasq\nYbjblpGk1ZQ33G3LSFJHpQv3Rr1GvRa2ZSRpBaULd2jO3m3LSFJnJQ53Z+6S1ElJw71uz12SVtBV\nuEfERRFxT0TsjoirVxj3mxGRETHVuxKPNTJkW0aSVrJquEdEHbgeuBjYDlweEdvbjNsAXAXc1usi\nl7MtI0kr62bmfi6wOzP3ZOYscBNwaZtxfwa8HzjUw/raGmnUDXdJWkE34b4J2NuyvK9YtyQiXgls\nzsx/WekbRcQVETEdEdMzMzPHXeyiYa+WkaQVPeUTqhFRAz4EvHu1sZl5Q2ZOZebUxMTECW9zpFHz\nhKokraCbcN8PbG5ZnizWLdoAvAz4SkTcB5wP7FjLk6r23CVpZd2E++3AtojYGhHDwGXAjsU3M/Ph\nzDwrM7dk5hbgVuCSzJxek4pZ7LnblpGkTlYN98ycA64EdgF3A5/LzDsj4tqIuGStC2yneSmkM3dJ\n6qTRzaDM3AnsXLbumg5jX/PUy1qZPXdJWll5P6FqW0aSOippuNuWkaSVlDPc7blL0orKGe6NOvML\nydy8AS9J7ZQ03BcftWe4S1I7hrskVVA5w32oDsCs4S5JbZUz3Jdm7l4OKUntlDTcmzN32zKS1F5J\nw72YufspVUlqq5zhPmRbRpJWUs5wty0jSSsqabg7c5eklZQz3IfsuUvSSsoZ7rZlJGlFJQ132zKS\ntJKSh7szd0lqp5zhXtx+wJ67JLVXznC3LSNJKypluDdqQS1sy0hSJ6UM94hg2EftSVJHpQx3KB6S\nfcS2jCS1U+Jwd+YuSZ2UN9x9SLYkdVTecG/UvVpGkjoocbjXvM5dkjood7jblpGktkoc7rZlJKmT\n8oa7J1QlqaPyhrs9d0nqqMThbltGkjopcbjblpGkTsob7vbcJamj8oa795aRpI66CveIuCgi7omI\n3RFxdZv3fz8ivh0R34yIr0bE9t6XerSRRo3ZeWfuktTOquEeEXXgeuBiYDtweZvw/kxm/kJmvgL4\nAPChnle6zEijzpH5ZH4h13pTklQ63czczwV2Z+aezJwFbgIubR2QmQdbFtcDa564I0PN0mftu0vS\nMRpdjNkE7G1Z3gect3xQRLwdeBcwDFzQk+pW0PqovbHh+lpvTpJKpWcnVDPz+sx8PvDHwPvajYmI\nKyJiOiKmZ2ZmntL2RhrFQ7KduUvSMboJ9/3A5pblyWJdJzcBb2r3RmbekJlTmTk1MTHRfZVtLM3c\n/ZSqJB2jm3C/HdgWEVsjYhi4DNjROiAitrUsvgH4fu9KbG+x5+6nVCXpWKv23DNzLiKuBHYBdeDG\nzLwzIq4FpjNzB3BlRFwIHAEOAG9dy6LBtowkraSbE6pk5k5g57J117S8vqrHda2q9YSqJOloJf6E\nqj13SeqkvOE+ZFtGkjopb7jblpGkjioQ7s7cJWm50ob7sD13SeqotOH+5KWQtmUkabnyhvuQbRlJ\n6qS84W7PXZI6Km24D9cXe+62ZSRpudKGe0T4kGxJ6qC04Q4Y7pLUQbnDfaju1TKS1Ea5w71R8zp3\nSWqj/OFuW0aSjlHycLctI0ntlDvch5y5S1I75Q53e+6S1FbJw922jCS1U/Jwty0jSe2UO9yH6swa\n7pJ0jHKHuzN3SWqrAuFuz12Slit5uNe9WkaS2ih3uHuduyS1Ve5wb9SYnV9gYSH7XYoknVJKHu7N\n56jOzjt7l6RWJQ/3xacxGe6S1Krc4b70kGyvmJGkVqUO93XDzbbMo4fn+lyJJJ1aSh3u46NDADxy\nyHCXpFalDvcNRbgfPHSkz5VI0qml1OE+PtYA4OATztwlqVW5w32pLePMXZJalTvcx2zLSFI7pQ73\n9cN1amFbRpKW6yrcI+KiiLgnInZHxNVt3n9XRNwVEd+KiJsj4rm9L7VtXYyPDTlzl6RlVg33iKgD\n1wMXA9uByyNi+7Jh3wCmMvPlwBeAD/S60E7GR4c4+IThLkmtupm5nwvszsw9mTkL3ARc2jogM2/J\nzMeLxVuByd6W2dn4WIODXucuSUfpJtw3AXtblvcV6zp5G/CvT6Wo4+HMXZKO1dMTqhHxO8AUcF2H\n96+IiOmImJ6ZmenJNsdH7blL0nLdhPt+YHPL8mSx7igRcSHwXuCSzDzc7htl5g2ZOZWZUxMTEydS\n7zHGxxpeLSNJy3QT7rcD2yJia0QMA5cBO1oHRMQ5wMdoBvsDvS+zM2fuknSsVcM9M+eAK4FdwN3A\n5zLzzoi4NiIuKYZdBzwN+HxEfDMidnT4dj03PjbE47PzHPGBHZK0pNHNoMzcCexctu6altcX9riu\nro2PNnfhkUNznLF+uF9lSNIppdSfUIUnb0Hg/WUk6UnlD/fF2/56UlWSlpQ+3DcUbRlPqkrSk0of\n7kt3hvSDTJK0pDrh7sxdkpaUP9xHfRqTJC1X+nBfP9xo3tPdmbskLSl9uNdqwQZvHiZJRyl9uIO3\n/ZWk5aoR7s7cJeko1Ql3e+6StKQa4T7W4BHbMpK0pBrhbltGko5SjXAfG/KEqiS1qEa4jw7x6OE5\n5rynuyQBVQn3seanVB897OxdkqAq4e5tfyXpKJUId2/7K0lHq0S4e9tfSTpaNcJ91Nv+SlKraoT7\nmLf9laRWFQl3Z+6S1KoS4f604QYR9twlaVElwr1WCzaMeNtfSVpUiXCHxVsQOHOXJKhSuI8OeUJV\nkgrVCfexhjN3SSpUJ9y97a8kLalOuI8N+cAOSSpUJ9yduUvSkuqE+1iDRw7PMb+Q/S5FkvquOuFe\n3F/mUVszklSdcPe2v5L0pMqE+2nF/WUOPD7b50okqf8qE+7PPn0MgB///Ik+VyJJ/ddVuEfERRFx\nT0Tsjoir27z/6oj4ekTMRcSbe1/m6jZvXAfAvgOGuyStGu4RUQeuBy4GtgOXR8T2ZcN+BPwe8Jle\nF9it8bEGG0YahrskAY0uxpwL7M7MPQARcRNwKXDX4oDMvK94b2ENauxKRLBp45jhLkl015bZBOxt\nWd5XrDvlTG5cx74Dj/e7DEnqu5N6QjUiroiI6YiYnpmZ6fn3n9w4xv4DT5DpB5kkDbZuwn0/sLll\nebJYd9wy84bMnMrMqYmJiRP5Fiua3DjGI4fnvPWvpIHXTbjfDmyLiK0RMQxcBuxY27JOzOTG5uWQ\ne23NSBpwq4Z7Zs4BVwK7gLuBz2XmnRFxbURcAhARvxgR+4C3AB+LiDvXsuhOJr0cUpKA7q6WITN3\nAjuXrbum5fXtNNs1fbU4c9/vB5kkDbjKfEIVmrcgeNpIwytmJA28SoV7RDDpte6SVK1wB9h0uuEu\nSZUL9+bM3baMpMFWwXBfxyOH5njYR+5JGmAVDPfmFTPO3iUNsgqGe/Na9/323SUNsAqG++LM3XCX\nNLgqF+6nrxti/XDdcJc00CoX7k/e192eu6TBVblwh8X7ujtzlzS4KhruztwlDbbKhvvBQ3McPOS1\n7pIGU0XD3cshJQ22Sob78ybWA3D3/Qf7XIkk9Uclw/2FT9/AaWND3LbnoX6XIkl9Uclwr9WCc7ee\nwW33/qzfpUhSX1Qy3AHOf96Z3Pezx7n/YfvukgZPZcP9vK1nANiakTSQKhvuL3nWOOOjDVszkgZS\nZcO9XgvO3XomtzpzlzSAKhvuAOc/7wzuffAxfvLwoX6XIkknVcXD/UwAWzOSBk6lw/0lzxpnw2jD\n1oykgVPpcK/XgvO2nsFte5y5SxoslQ53gPO2nsmeBx/jgYP23SUNjsqH+6ue3+y777rrp32uRJJO\nnsqH+0ufPc7Zm0/nb77yA2bnFvpdjiSdFJUP94jgnRduY//Pn+CLX9/X73Ik6aSofLgDvOaFE5w9\neRofuWU3R+advUuqvoEI94jgqgu3se+As3dJg2Egwh3gtS96Oi939i5pQAxMuEcEV71uG3sfeoK/\nvvn7/S5HktbUwIQ7wAUvfjq/8cpNfPjLu/nIlw14SdXV6HcBJ1NEcN2bz4aED/7794gI3v7aF/S7\nLEnqua5m7hFxUUTcExG7I+LqNu+PRMQ/FO/fFhFbel1or9RrwXVvOZtfP2cT1+26h3fe9A3uffCx\nfpclST216sw9IurA9cCvAPuA2yNiR2be1TLsbcCBzHxBRFwGvB/4rbUouBfqteCDbzmbZ502yo3/\ncy87/u/HvOmcTVx+7nM4e/J0hhsD1a2SVEGRmSsPiHgV8CeZ+avF8nsAMvPPW8bsKsZ8LSIawE+A\niVzhm09NTeX09HQPduGpmXnkMB/7rx/w6Vt/yOG5BUaHakw99wxeummcyY3rmNw4xtM3jDA+OsT4\n6BDrR+o06oa/pP6IiDsyc2q1cd303DcBe1uW9wHndRqTmXMR8TBwJvBgd+X2z8SGEd73xu38wQXb\n+Nqen3Fr8fXJr97HbIdLJuu1YKRRY7hRo1EL6rWgHkFEUKtBENSi2eMPoPmfppaXRATH4/hGSzpV\nveN12/i1s5+9pts4qSdUI+IK4AqA5zznOSdz06s6bd0QF73smVz0smcCsLCQzDx6mL0PPc7PHpvl\n4BNHePiJIzw+O8/huXkOH1lgdn6B+YVkfiGZW0gyITNZyCShuUxzHTRfL1n5F6Zj5PH+BUmnrNPG\nhtZ8G92E+35gc8vyZLGu3Zh9RVvmNOCYm6hn5g3ADdBsy5xIwSdLrRY8Y3yUZ4yP9rsUSTpu3TSP\nbwe2RcTWiBgGLgN2LBuzA3hr8frNwJdX6rdLktbWqjP3ood+JbALqAM3ZuadEXEtMJ2ZO4BPAJ+O\niN3AQzT/AZAk9UlXPffM3AnsXLbumpbXh4C39LY0SdKJ8po+Saogw12SKshwl6QKMtwlqYIMd0mq\noFXvLbNmG46YAX54gn/9LEpwa4M1MIj7PYj7DIO534O4z3D8+/3czJxYbVDfwv2piIjpbm6cUzWD\nuN+DuM8wmPs9iPsMa7fftmUkqYIMd0mqoLKG+w39LqBPBnG/B3GfYTD3exD3GdZov0vZc5ckrays\nM3dJ0gpKF+6rPay7CiJic0TcEhF3RcSdEXFVsf6MiPiPiPh+8efGftfaaxFRj4hvRMSXiuWtxUPX\ndxcPYR/ud429FhGnR8QXIuK7EXF3RLxqQI71HxY/39+JiM9GxGjVjndE3BgRD0TEd1rWtT220fTh\nYt+/FRGvfCrbLlW4tzys+2JgO3B5RGzvb1VrYg54d2ZuB84H3l7s59XAzZm5Dbi5WK6aq4C7W5bf\nD/xFZr4AOEDzYexV81fAv2Xmi4Gzae5/pY91RGwC3gFMZebLaN5O/DKqd7z/Frho2bpOx/ZiYFvx\ndQXw0aey4VKFO3AusDsz92TmLHATcGmfa+q5zLw/M79evH6E5v/sm2ju66eKYZ8C3tSfCtdGREwC\nbwA+XiwHcAHwhWJIFff5NODVNJ+JQGbOZubPqfixLjSAseLpbeuA+6nY8c7M/6b5jItWnY7tpcDf\nZdOtwOkR8awT3XbZwr3dw7o39amWkyIitgDnALcBz8jM+4u3fgI8o09lrZW/BP4IWHwy+ZnAzzNz\nrliu4vHeCswAnyzaUR+PiPVU/Fhn5n7gg8CPaIb6w8AdVP94Q+dj29N8K1u4D5SIeBrwj8A7M/Ng\n63vFYwwrc6lTRLwReCAz7+h3LSdZA3gl8NHMPAd4jGUtmKoda4Ciz3wpzX/cng2s59j2ReWt5bEt\nW7h387DuSoiIIZrB/veZ+cVi9U8Xf00r/nygX/WtgV8CLomI+2i22y6g2Ys+vfi1Hap5vPcB+zLz\ntmL5CzTDvsrHGuBC4N7MnMnMI8AXaf4MVP14Q+dj29N8K1u4d/Ow7tIres2fAO7OzA+1vNX6IPK3\nAv98smtbK5n5nsyczMwtNI/rlzPzt4FbaD50HSq2zwCZ+RNgb0S8qFj1OuAuKnysCz8Czo+IdcXP\n++J+V/p4Fzod2x3A7xZXzZwPPNzSvjl+mVmqL+D1wPeAHwDv7Xc9a7SPv0zzV7VvAd8svl5Pswd9\nM/B94D+BM/pd6xrt/2uALxWvnwf8L7Ab+Dww0u/61mB/XwFMF8f7n4CNg3CsgT8Fvgt8B/g0MFK1\n4w18luY5hSM0f0t7W6djCwTNqwF/AHyb5pVEJ7xtP6EqSRVUtraMJKkLhrskVZDhLkkVZLhLUgUZ\n7pJUQYa7JFWQ4S5JFWS4S1IF/T8/W/C09qY3ewAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "GSJ1mqpGmeqG",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "39677980-495c-4b12-8c29-c5a1e7f47633"
      },
      "cell_type": "code",
      "source": [
        "(1-test_accuracy)*100"
      ],
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.7600000286102304"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 72
        }
      ]
    },
    {
      "metadata": {
        "id": "GAN9CwuEmngK",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import scipy.io as sio\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "wxlg_sUWmndo",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "epochTrack = [k for k in range(1,110,10)]\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "S3vZjD2Imnau",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "sio.savemat('LeNEt5PRobBasedValid99p78Test99p24.mat', {'ValidationTracked':validation_accuracy_track,\n",
        "                                       'train_accuracy_track':train_accuracy_track,\n",
        "                                       'connection_probability_track':connection_probability_track,\n",
        "                                       'epochTrack':epochTrack, 'TestAcc':test_accuracy,\n",
        "                                                         'BestValidation':best_accuracy_valid})"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "qvRqnUrrCy6m",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def write_sprite_image(filename, images):\n",
        "    \"\"\"\n",
        "        Create a sprite image consisting of sample images\n",
        "        :param filename: name of the file to save on disk\n",
        "        :param shape: tensor of flattened images\n",
        "    \"\"\"\n",
        "\n",
        "    # Invert grayscale image\n",
        "    images = 1 - images\n",
        "\n",
        "    # Calculate number of plot\n",
        "    n_plots = int(np.ceil(np.sqrt(images.shape[0])))\n",
        "\n",
        "    # Make the background of sprite image\n",
        "    sprite_image = np.ones((img_h * n_plots, img_w * n_plots))\n",
        "\n",
        "    for i in range(n_plots):\n",
        "        for j in range(n_plots):\n",
        "            img_idx = i * n_plots + j\n",
        "            if img_idx < images.shape[0]:\n",
        "                img = images[img_idx]\n",
        "                sprite_image[i * img_h:(i + 1) * img_h,\n",
        "                j * img_w:(j + 1) * img_w] = img\n",
        "\n",
        "    plt.imsave(filename, sprite_image, cmap='gray')\n",
        "    print('Sprite image saved in {}'.format(filename))\n",
        "\n",
        "def write_metadata(filename, labels):\n",
        "    \"\"\"\n",
        "            Create a metadata file image consisting of sample indices and labels\n",
        "            :param filename: name of the file to save on disk\n",
        "            :param shape: tensor of labels\n",
        "    \"\"\"\n",
        "    with open(filename, 'w') as f:\n",
        "        f.write(\"Index\\tLabel\\n\")\n",
        "        for index, label in enumerate(labels):\n",
        "            f.write(\"{}\\t{}\\n\".format(index, label))\n",
        "\n",
        "    print('Metadata file saved in {}'.format(filename))\n",
        " "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "e2S6lFLBySfi",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "img_w = 32\n",
        "img_h = 32"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "BuqVjiWmwqJ-",
        "colab_type": "code",
        "outputId": "a5469f6e-d005-47a2-f12d-8f670ec3d6bc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 225
        }
      },
      "cell_type": "code",
      "source": [
        "# Reshape images from vector to matrix\n",
        "x_test_images = np.reshape(np.array(X_test), (-1, img_w, img_h))\n",
        "# Reshape labels from one-hot-encode to index\n",
        "x_test_labels = y_test\n",
        "\n",
        "write_sprite_image(os.path.join(logs_path, 'sprite_images.png'), x_test_images)\n",
        "write_metadata(os.path.join(logs_path, 'metadata.tsv'), x_test_labels)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-25-eec30453d89c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mx_test_labels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0my_test\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mwrite_sprite_image\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogs_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'sprite_images.png'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx_test_images\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0mwrite_metadata\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogs_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'metadata.tsv'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx_test_labels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'os' is not defined"
          ]
        }
      ]
    },
    {
      "metadata": {
        "id": "ma8GQKVWwswP",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!wget https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip\n",
        "!unzip -o ngrok-stable-linux-amd64.zip\n",
        "# logdir=logs/embedding/\n",
        "LOG_DIR = 'logs/embedding/'\n",
        "get_ipython().system_raw(\n",
        "    'tensorboard --logdir {} --host 0.0.0.0 --port 6006 &'\n",
        "    .format(LOG_DIR)\n",
        ")\n",
        "\n",
        "get_ipython().system_raw('./ngrok http 6006 &')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "b_PM5kqHwuZP",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "ngrok_url = !curl -s http://localhost:4040/api/tunnels | python -c \\\n",
        "    \"import sys, json; print(json.load(sys.stdin)['tunnels'][0]['public_url'])\"\n",
        "        \n",
        "ngrok_url = ngrok_url[0].replace(\"'\", '')\n",
        "print(ngrok_url)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "0WeHkK34wvyk",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from IPython.display import IFrame\n",
        "\n",
        "IFrame(ngrok_url, width=700, height=900)"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}