Visualize_04012019_99p3ProbBased_LeNet_LabAugmentedData.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Visualize 04012019 99p3ProbBased LeNet-LabAugmentedData.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/preritt/CS231n-/blob/master/Visualize_04012019_99p3ProbBased_LeNet_LabAugmentedData.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "HFVqYNiRCy47",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# LeNet Lab\n",
        "![LeNet Architecture](https://github.com/sujaybabruwad/LeNet-in-Tensorflow/blob/master/lenet.png?raw=1)\n",
        "Source: Yan LeCun"
      ]
    },
    {
      "metadata": {
        "id": "oWv2YnX4IxqB",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from tensorflow.contrib.tensorboard.plugins import projector\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "k7PXv7fjCy5B",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Load Data\n",
        "\n",
        "Load the MNIST data, which comes pre-loaded with TensorFlow.\n",
        "\n",
        "You do not need to modify this section."
      ]
    },
    {
      "metadata": {
        "id": "t4iqynjrCy5E",
        "colab_type": "code",
        "outputId": "ee138add-1c57-41f1-c8fc-e2ce5948a69e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 619
        }
      },
      "cell_type": "code",
      "source": [
        "from tensorflow.examples.tutorials.mnist import input_data\n",
        "\n",
        "mnist = input_data.read_data_sets(\"MNIST_data/\", reshape=False)\n",
        "X_train, y_train           = mnist.train.images, mnist.train.labels\n",
        "X_validation, y_validation = mnist.validation.images, mnist.validation.labels\n",
        "X_test, y_test             = mnist.test.images, mnist.test.labels\n",
        "\n",
        "assert(len(X_train) == len(y_train))\n",
        "assert(len(X_validation) == len(y_validation))\n",
        "assert(len(X_test) == len(y_test))\n",
        "\n",
        "print()\n",
        "print(\"Image Shape: {}\".format(X_train[0].shape))\n",
        "print()\n",
        "print(\"Training Set:   {} samples\".format(len(X_train)))\n",
        "print(\"Validation Set: {} samples\".format(len(X_validation)))\n",
        "print(\"Test Set:       {} samples\".format(len(X_test)))"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From <ipython-input-2-566519a95339>:3: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please write your own downloading logic.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/base.py:252: _internal_retry.<locals>.wrap.<locals>.wrapped_fn (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use urllib or similar directly.\n",
            "Successfully downloaded train-images-idx3-ubyte.gz 9912422 bytes.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use tf.data to implement this functionality.\n",
            "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
            "Successfully downloaded train-labels-idx1-ubyte.gz 28881 bytes.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use tf.data to implement this functionality.\n",
            "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
            "Successfully downloaded t10k-images-idx3-ubyte.gz 1648877 bytes.\n",
            "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
            "Successfully downloaded t10k-labels-idx1-ubyte.gz 4542 bytes.\n",
            "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
            "\n",
            "Image Shape: (28, 28, 1)\n",
            "\n",
            "Training Set:   55000 samples\n",
            "Validation Set: 5000 samples\n",
            "Test Set:       10000 samples\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "0Jm9y17KI5QD",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import os\n",
        "logs_path = \"./logs/embedding/\"  # path to the folder that we want to save the logs for Tensorboard"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "pV8kvNmMCy5T",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "The MNIST data that TensorFlow pre-loads comes as 28x28x1 images.\n",
        "\n",
        "However, the LeNet architecture only accepts 32x32xC images, where C is the number of color channels.\n",
        "\n",
        "In order to reformat the MNIST data into a shape that LeNet will accept, we pad the data with two rows of zeros on the top and bottom, and two columns of zeros on the left and right (28+2+2 = 32).\n",
        "\n",
        "You do not need to modify this section."
      ]
    },
    {
      "metadata": {
        "id": "sBWJCBw1Cy5U",
        "colab_type": "code",
        "outputId": "b8b68f5d-0115-4e24-b40c-4f5c1d641d9f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "\n",
        "# Pad images with 0s\n",
        "X_train      = np.pad(X_train, ((0,0),(2,2),(2,2),(0,0)), 'constant')\n",
        "X_validation = np.pad(X_validation, ((0,0),(2,2),(2,2),(0,0)), 'constant')\n",
        "X_test       = np.pad(X_test, ((0,0),(2,2),(2,2),(0,0)), 'constant')\n",
        "    \n",
        "print(\"Updated Image Shape: {}\".format(X_train[0].shape))"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Updated Image Shape: (32, 32, 1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "cW7cCH1xCy5c",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Visualize Data\n",
        "\n",
        "View a sample from the dataset.\n",
        "\n",
        "You do not need to modify this section."
      ]
    },
    {
      "metadata": {
        "id": "h7UbBX8RCy5e",
        "colab_type": "code",
        "outputId": "ae5ced4a-91c4-463f-c1b6-761f56f6a25c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 124
        }
      },
      "cell_type": "code",
      "source": [
        "import random\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "\n",
        "index = random.randint(0, len(X_train))\n",
        "image = X_train[index].squeeze()\n",
        "\n",
        "plt.figure(figsize=(1,1))\n",
        "plt.imshow(image, cmap=\"gray\")\n",
        "print(y_train[index])"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAFcAAABYCAYAAACAnmu5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAABchJREFUeJztnE1oXFUUx3/TStvUtmJNxUYoxRBO\nbdNFDS2JYKZGSU0IukgkixLEFlSwJWC0MbhxUyM21H4sCiUottmoBELUElLrIosuLO1EOiCn0kIq\niZAvjI6GUppxMe9NxiaTzrzOfcmb3B+EeXPfnfsO/xzOzLv3/24oHo9jMcOKxQ4gn7HiGsSKaxAr\nrkGsuAax4hrkEa8fFJHPgXIgDjSr6pWcRZUneMpcEQkDJapaARwETuU0qjzBa1l4CegBUNVfgcdF\nZEO6zqFQKB6NRuOhUCgQf9nEupBIXsvCU8DVlPdjTttf83W+fv06paWlBOluMBexeq659xFa6OTO\nnTuJx+OEQgt2WzJkE+tC/wSvZWGERKa6FAF/eBwrb/Eqbj/QACAizwEjqvp3zqLKE0Jea4uIfApU\nAjPAu6r6S9qLhELxPC4LaTt6Fjcblqu4ufpCM05paSkAR44cAWBqaoqRkREAJiYmADh79uziBJcG\ne/trkMBkbk9PDwBbt26dc25mZgaA3bt3c/LkSQCi0ahvsaUjMDX3+PHjAFRVVQFw7do1CgsLAait\nrU32Gx0dBaC6uhrwJnKuaq4tCwYJTOa6bNq0CYCxsTEKCgoAKCsrA6C1tZWamprkeYDNmzdnfQ2b\nuQEgcJm7EKtWreLcuXMANDQ0AHD48GEAzpw5k/E49iYiDa2trQAcPXoUgNu3bwOwY8cOpqenMxrD\nloUAkHeZu27dOgBu3rwJkPy5VlJSwq1btzIaw2ZuAMg7cWOxGLFYjM7OTjo7O5Ptzc3NvseSd+Iu\nJay4Blk24vb39/t+zWUj7mIQmClHFxEB4OLFi/T19QHQ0dEBwI0bN1i9ejUAFRUVAKxYkcifgYEB\nv0O1mWuSwGXu9u3bASgqKuLAgQPA7DJPW1sb4XAYgMrKSmB2In0xyIvMrauro66ujrVr19LS0kJL\nS8ucPu7am5/khbhLlcCVhfkoLi4GIBKJJI/vZ3h42M+QAJu5Rglc5roLkJFIhF27dgGJSXLgf1nr\nzort378/2d9vMhJXRD4DXnD6twNXgPPAShIGvCZVvWMqyKDywPlcEXkR+EBVa0XkCSACXAIuqOq3\nIvIJ8Luqpl1HMTGfu3HjRhobGwE4ffp0st09PnHiBABDQ0NZj+2nnWkA+Nk5/hN4FNgLvOO0fQe8\nD2S+SJUDJicn2bZt25z2qakpgOTKcGoft1TcvXvXhwizXIkQkbdIlId9qvqk01YMnFfV59N9LhqN\nxl2vVx7y8EY8EXmNxMMl1cBvmQzukktnuZuRe/bsoaurC1jYm+BeMx6P093dDUB7ezsw620YHx/n\nzp3ZrwxfneUisg/4CKhR1SkgJiIFzumnSTjNLffxwMwVkceAY8DLqjrpNP8I1ANdzmufsQgd1q9f\nD8CpU4mnspqamub0mZ6enlNP16xZAyR+rtXX1wMkX116e3sZHBwEZj1puSCTstAIFALfuNN9wBtA\np4i8DQwBX+UsojwiMEvrrrmjqKhozjm39nZ0dMxxNZaXlwMQDoeThpENG9I+Msfw8DBbtmyxjhtT\nWN9CALDiGsSKaxArrkGsuAax4hrEimsQK65BrLgGseIaxIprECuuQay4BvFlVmy5YjPXIFZcg1hx\nDWLFNYgV1yBWXINYcQ3iiz93qW9kPI9F9lWgDJhwuhxT1R+yHde4uKkbGYvIs8AXQIXp62aKY5Et\ndeJzLbI/AW2q+v3DjO1HWchqI+NFYAB43Tl2LbIrczGwH2Uhq42M/UZV7wH/OG8PAheAe8AhEXkP\nGAUOqep4tmMvxhfakrTdpFhkD5F4JOFDVa0CBoGPvYzpR+Yu+Y2MUyyyrzgW2Uspp3vx6Jr3I3OX\n9EbGKRbZOtciKyLdIvKM02Uv4GljSOOZq6qXReSqiFzG2cjY9DWzZD6L7JfA1yLyLxAD3vQysJ3P\nNYi9QzOIFdcgVlyDWHENYsU1iBXXIFZcg/wHPkNiK33kuk4AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 72x72 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "1W6py5mlCy5n",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Preprocess Data\n",
        "\n",
        "Shuffle the training data.\n",
        "\n",
        "You do not need to modify this section."
      ]
    },
    {
      "metadata": {
        "id": "UYJZs7GgCy5p",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from sklearn.utils import shuffle\n",
        "\n",
        "X_train, y_train = shuffle(X_train, y_train)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Q4WFIdiuCy53",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Setup TensorFlow\n",
        "The `EPOCH` and `BATCH_SIZE` values affect the training speed and model accuracy.\n",
        "\n",
        "You do not need to modify this section."
      ]
    },
    {
      "metadata": {
        "id": "est-t83SCy55",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import tensorflow as tf\n",
        "\n",
        "EPOCHS = 100\n",
        "BATCH_SIZE = 128"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "cWVHSK0pCy59",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## TODO: Implement LeNet-5\n",
        "Implement the [LeNet-5](http://yann.lecun.com/exdb/lenet/) neural network architecture.\n",
        "\n",
        "This is the only cell you need to edit.\n",
        "### Input\n",
        "The LeNet architecture accepts a 32x32xC image as input, where C is the number of color channels. Since MNIST images are grayscale, C is 1 in this case.\n",
        "\n",
        "### Architecture\n",
        "**Layer 1: Convolutional.** The output shape should be 28x28x6.\n",
        "\n",
        "**Activation.** Your choice of activation function.\n",
        "\n",
        "**Pooling.** The output shape should be 14x14x6.\n",
        "\n",
        "**Layer 2: Convolutional.** The output shape should be 10x10x16.\n",
        "\n",
        "**Activation.** Your choice of activation function.\n",
        "\n",
        "**Pooling.** The output shape should be 5x5x16.\n",
        "\n",
        "**Flatten.** Flatten the output shape of the final pooling layer such that it's 1D instead of 3D. The easiest way to do is by using `tf.contrib.layers.flatten`, which is already imported for you.\n",
        "\n",
        "**Layer 3: Fully Connected.** This should have 120 outputs.\n",
        "\n",
        "**Activation.** Your choice of activation function.\n",
        "\n",
        "**Layer 4: Fully Connected.** This should have 84 outputs.\n",
        "\n",
        "**Activation.** Your choice of activation function.\n",
        "\n",
        "**Layer 5: Fully Connected (Logits).** This should have 10 outputs.\n",
        "\n",
        "### Output\n",
        "Return the result of the 2nd fully connected layer."
      ]
    },
    {
      "metadata": {
        "id": "Vgad6Ny0OjqN",
        "colab_type": "code",
        "outputId": "0ce6a2ce-32a5-44a9-932e-785415fd0ec3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 92
        }
      },
      "cell_type": "code",
      "source": [
        "connection_probability = tf.Variable(.9999)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Colocations handled automatically by placer.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "X1nJmcI7Cy5-",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from tensorflow.contrib.layers import flatten\n",
        "\n",
        "def LeNet(x, test_mode = False):    \n",
        "    # Hyperparameters\n",
        "    mu = 0\n",
        "    sigma = 0.1\n",
        "    layer_depth = {\n",
        "        'layer_1' : 6,\n",
        "        'layer_2' : 16,\n",
        "        'layer_3' : 120,\n",
        "        'layer_f1' : 84\n",
        "    }\n",
        "\n",
        "    \n",
        "    # TODO: Layer 1: Convolutional. Input = 32x32x1. Output = 28x28x6.\n",
        "    conv1_w = tf.Variable(tf.truncated_normal(shape = [5,5,1,6],mean = mu, stddev = sigma))\n",
        "    conv1_b = tf.Variable(tf.zeros(6))\n",
        "    conv1 = tf.nn.conv2d(x,conv1_w, strides = [1,1,1,1], padding = 'VALID') + conv1_b \n",
        "    # TODO: Activation.\n",
        "    conv1 = tf.nn.relu(conv1)\n",
        "\n",
        "    # TODO: Pooling. Input = 28x28x6. Output = 14x14x6.\n",
        "    pool_1 = tf.nn.max_pool(conv1,ksize = [1,2,2,1], strides = [1,2,2,1], padding = 'VALID')\n",
        "    \n",
        "    # TODO: Layer 2: Convolutional. Output = 10x10x16.\n",
        "    conv2_w = tf.Variable(tf.truncated_normal(shape = [5,5,6,16], mean = mu, stddev = sigma))\n",
        "    conv2_b = tf.Variable(tf.zeros(16))\n",
        "    conv2 = tf.nn.conv2d(pool_1, conv2_w, strides = [1,1,1,1], padding = 'VALID') + conv2_b\n",
        "    # TODO: Activation.\n",
        "    conv2 = tf.nn.relu(conv2)\n",
        "    \n",
        "    ##### Inset probability connection from x to conv2\n",
        "    conv1p_w = tf.Variable(tf.truncated_normal(shape = [5,5,1,16], mean = mu, stddev = sigma))\n",
        "    conv1p_b = tf.Variable(tf.zeros(16))\n",
        "    conv2_2nd_input = tf.nn.conv2d(x, conv1p_w, strides = [1,3,3,1], padding = 'VALID') + conv1p_b\n",
        "    conv2_2nd_input = tf.nn.relu(conv2_2nd_input) #result = tf.cond(x < y, lambda: tf.add(x, z), lambda: tf.square(y))\n",
        "#     conv2 = tf.cond(tf.random.uniform(shape = tf.shape(connection_probability)) < connection_probability,lambda: conv2 + conv2_2nd_input, lambda: conv2 )\n",
        "    conv2 = tf.cond(tf.logical_and(tf.random.uniform(shape = tf.shape(connection_probability)) < connection_probability, tf.equal(test_mode,False)),lambda: conv2 + conv2_2nd_input, lambda: conv2 )\n",
        "\n",
        "#     if (np.random.random_sample()<connection_probability and test_mode == False):\n",
        "#         conv2 = conv2 + conv2_2nd_input\n",
        "#     if test_mode == True:\n",
        "#       conv2 = conv2 + conv2_2nd_input*connection_probability*0\n",
        "    \n",
        "    # TODO: Pooling. Input = 10x10x16. Output = 5x5x16.\n",
        "    pool_2 = tf.nn.max_pool(conv2, ksize = [1,2,2,1], strides = [1,2,2,1], padding = 'VALID') \n",
        "    \n",
        "    # TODO: Flatten. Input = 5x5x16. Output = 400.\n",
        "    fc1 = flatten(pool_2)\n",
        "    fdense = fc1\n",
        "    \n",
        "    # TODO: Layer 3: Fully Connected. Input = 400. Output = 120.\n",
        "    fc1_w = tf.Variable(tf.truncated_normal(shape = (400,120), mean = mu, stddev = sigma))\n",
        "    fc1_b = tf.Variable(tf.zeros(120))\n",
        "    fc1 = tf.matmul(fc1,fc1_w) + fc1_b\n",
        "    \n",
        "    # TODO: Activation.\n",
        "    fc1 = tf.nn.relu(fc1)\n",
        "\n",
        "    # TODO: Layer 4: Fully Connected. Input = 120. Output = 84.\n",
        "    fc2_w = tf.Variable(tf.truncated_normal(shape = (120,84), mean = mu, stddev = sigma))\n",
        "    fc2_b = tf.Variable(tf.zeros(84))\n",
        "    fc2 = tf.matmul(fc1,fc2_w) + fc2_b\n",
        "    # TODO: Activation.\n",
        "    fc2 = tf.nn.relu(fc2)\n",
        "    \n",
        "    \n",
        "    #################\n",
        "    ##### Inset probability connection from x to conv2\n",
        "    fc2p_w = tf.Variable(tf.truncated_normal(shape = [400,84], mean = mu, stddev = sigma))\n",
        "    fc2p_b = tf.Variable(tf.zeros(84))\n",
        "    fc2_2nd_input = tf.matmul(fdense,fc2p_w) + fc2p_b\n",
        "    fc2_2nd_input = tf.nn.relu(fc2_2nd_input)\n",
        "    fc2 = tf.cond(tf.logical_and(tf.random.uniform(shape = tf.shape(connection_probability)) < connection_probability, tf.equal(test_mode,False)),lambda: fc2 + fc2_2nd_input, lambda: fc2 )\n",
        "\n",
        "#     if (np.random.random_sample()<connection_probability and test_mode == False):\n",
        "#         fc2 = fc2 + fc2_2nd_input\n",
        "#     if test_mode == True:\n",
        "#         fc2 = fc2 + fc2_2nd_input*connection_probability*0\n",
        "    \n",
        "    ################\n",
        "    \n",
        "    # TODO: Layer 5: Fully Connected. Input = 84. Output = 10.\n",
        "    fc3_w = tf.Variable(tf.truncated_normal(shape = (84,10), mean = mu , stddev = sigma))\n",
        "    fc3_b = tf.Variable(tf.zeros(10))\n",
        "    logits = tf.matmul(fc2, fc3_w) + fc3_b\n",
        "    return logits"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "aCx837c9Cy6E",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Features and Labels\n",
        "Train LeNet to classify [MNIST](http://yann.lecun.com/exdb/mnist/) data.\n",
        "\n",
        "`x` is a placeholder for a batch of input images.\n",
        "`y` is a placeholder for a batch of output labels.\n",
        "\n",
        "You do not need to modify this section."
      ]
    },
    {
      "metadata": {
        "id": "iafVK0DgLRTY",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Mrb2FHFHCy6G",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "with tf.name_scope('Input'):\n",
        "\n",
        "  x = tf.placeholder(tf.float32, (None, 32, 32, 1), name='X')\n",
        "  tf.summary.image('input_image', x, max_outputs=5)\n",
        "  y = tf.placeholder(tf.int32, (None), name = 'Y')\n",
        "one_hot_y = tf.one_hot(y, 10)\n",
        "is_testing= tf.placeholder(tf.bool) "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "uk0pcAjmCy6M",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Training Pipeline\n",
        "Create a training pipeline that uses the model to classify MNIST data.\n",
        "\n",
        "You do not need to modify this section."
      ]
    },
    {
      "metadata": {
        "id": "3oYy_Y3FCy6N",
        "colab_type": "code",
        "outputId": "9385149e-d0b7-4c7d-cb02-b746c915be3e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 237
        }
      },
      "cell_type": "code",
      "source": [
        "rate = 0.001\n",
        "\n",
        "logits = LeNet(x,is_testing)\n",
        "with tf.name_scope('Train'):\n",
        "  cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits = logits, labels = one_hot_y)\n",
        "  loss_operation = tf.reduce_mean(cross_entropy, name='loss')\n",
        "  tf.summary.scalar('loss', loss_operation)\n",
        "optimizer = tf.train.AdamOptimizer(learning_rate = rate)\n",
        "training_operation = optimizer.minimize(loss_operation)\n",
        "new_prob = connection_probability.assign(connection_probability/1.00005)\n",
        "\n"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/layers/python/layers/layers.py:1624: flatten (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use keras.layers.flatten instead.\n",
            "WARNING:tensorflow:From <ipython-input-11-450f4e83c3de>:5: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "\n",
            "Future major versions of TensorFlow will allow gradients to flow\n",
            "into the labels input on backprop by default.\n",
            "\n",
            "See `tf.nn.softmax_cross_entropy_with_logits_v2`.\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "KrMtHo6rCy6S",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Model Evaluation\n",
        "Evaluate how well the loss and accuracy of the model for a given dataset.\n",
        "\n",
        "You do not need to modify this section."
      ]
    },
    {
      "metadata": {
        "id": "OMR9XsLBCy6U",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(one_hot_y, 1))\n",
        "accuracy_operation = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
        "saver = tf.train.Saver()\n",
        "\n",
        "def evaluate(X_data, y_data):\n",
        "    num_examples = len(X_data)\n",
        "    total_accuracy = 0\n",
        "    sess = tf.get_default_session()\n",
        "    for offset in range(0, num_examples, BATCH_SIZE):\n",
        "        batch_x, batch_y = X_data[offset:offset+BATCH_SIZE], y_data[offset:offset+BATCH_SIZE]\n",
        "        accuracy = sess.run(accuracy_operation, feed_dict={x: batch_x, y: batch_y, is_testing: True})\n",
        "        total_accuracy += (accuracy * len(batch_x))\n",
        "    tot_acc = total_accuracy / num_examples\n",
        "    with tf.name_scope('Accuracy'):\n",
        "      tf.summary.scalar('accuracy', tot_acc)\n",
        "    return total_accuracy / num_examples"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "2Jw7iox3Cy6X",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Train the Model\n",
        "Run the training data through the training pipeline to train the model.\n",
        "\n",
        "Before each epoch, shuffle the training set.\n",
        "\n",
        "After each epoch, measure the loss and accuracy of the validation set.\n",
        "\n",
        "Save the model after training.\n",
        "\n",
        "You do not need to modify this section."
      ]
    },
    {
      "metadata": {
        "id": "c98G07hDQG2g",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import math"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "8QC1kt_DL1yU",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def augment(images, labels,\n",
        "            resize=None, # (width, height) tuple or None\n",
        "            horizontal_flip=False,\n",
        "            vertical_flip=False,\n",
        "            rotate=0, # Maximum rotation angle in degrees\n",
        "            crop_probability=0, # How often we do crops\n",
        "            crop_min_percent=0.6, # Minimum linear dimension of a crop\n",
        "            crop_max_percent=1.,  # Maximum linear dimension of a crop\n",
        "            mixup=0):  # Mixup coeffecient, see https://arxiv.org/abs/1710.09412.pdf\n",
        "  if resize is not None:\n",
        "    images = tf.image.resize_bilinear(images, resize)\n",
        "  \n",
        "  # My experiments showed that casting on GPU improves training performance\n",
        "  if images.dtype != tf.float32:\n",
        "    images = tf.image.convert_image_dtype(images, dtype=tf.float32)\n",
        "    images = tf.subtract(images, 0.5)\n",
        "    images = tf.multiply(images, 2.0)\n",
        "  labels = tf.to_float(labels)\n",
        "\n",
        "  with tf.name_scope('augmentation'):\n",
        "    shp = tf.shape(images)\n",
        "    batch_size, height, width = shp[0], shp[1], shp[2]\n",
        "    width = tf.cast(width, tf.float32)\n",
        "    height = tf.cast(height, tf.float32)\n",
        "\n",
        "    # The list of affine transformations that our image will go under.\n",
        "    # Every element is Nx8 tensor, where N is a batch size.\n",
        "    transforms = []\n",
        "    identity = tf.constant([1, 0, 0, 0, 1, 0, 0, 0], dtype=tf.float32)\n",
        "    if horizontal_flip:\n",
        "      coin = tf.less(tf.random_uniform([batch_size], 0, 1.0), 0.5)\n",
        "      flip_transform = tf.convert_to_tensor(\n",
        "          [-1., 0., width, 0., 1., 0., 0., 0.], dtype=tf.float32)\n",
        "      transforms.append(\n",
        "          tf.where(coin,\n",
        "                   tf.tile(tf.expand_dims(flip_transform, 0), [batch_size, 1]),\n",
        "                   tf.tile(tf.expand_dims(identity, 0), [batch_size, 1])))\n",
        "\n",
        "    if vertical_flip:\n",
        "      coin = tf.less(tf.random_uniform([batch_size], 0, 1.0), 0.5)\n",
        "      flip_transform = tf.convert_to_tensor(\n",
        "          [1, 0, 0, 0, -1, height, 0, 0], dtype=tf.float32)\n",
        "      transforms.append(\n",
        "          tf.where(coin,\n",
        "                   tf.tile(tf.expand_dims(flip_transform, 0), [batch_size, 1]),\n",
        "                   tf.tile(tf.expand_dims(identity, 0), [batch_size, 1])))\n",
        "\n",
        "    if rotate > 0:\n",
        "      angle_rad = rotate / 180 * math.pi\n",
        "      angles = tf.random_uniform([batch_size], -angle_rad, angle_rad)\n",
        "      transforms.append(\n",
        "          tf.contrib.image.angles_to_projective_transforms(\n",
        "              angles, height, width))\n",
        "\n",
        "    if crop_probability > 0:\n",
        "      crop_pct = tf.random_uniform([batch_size], crop_min_percent,\n",
        "                                   crop_max_percent)\n",
        "      left = tf.random_uniform([batch_size], 0, width * (1 - crop_pct))\n",
        "      top = tf.random_uniform([batch_size], 0, height * (1 - crop_pct))\n",
        "      crop_transform = tf.stack([\n",
        "          crop_pct,\n",
        "          tf.zeros([batch_size]), top,\n",
        "          tf.zeros([batch_size]), crop_pct, left,\n",
        "          tf.zeros([batch_size]),\n",
        "          tf.zeros([batch_size])\n",
        "      ], 1)\n",
        "\n",
        "      coin = tf.less(\n",
        "          tf.random_uniform([batch_size], 0, 1.0), crop_probability)\n",
        "      transforms.append(\n",
        "          tf.where(coin, crop_transform,\n",
        "                   tf.tile(tf.expand_dims(identity, 0), [batch_size, 1])))\n",
        "\n",
        "    if transforms:\n",
        "      images = tf.contrib.image.transform(\n",
        "          images,\n",
        "          tf.contrib.image.compose_transforms(*transforms),\n",
        "          interpolation='BILINEAR') # or 'NEAREST'\n",
        "\n",
        "    def cshift(values): # Circular shift in batch dimension\n",
        "      return tf.concat([values[-1:, ...], values[:-1, ...]], 0)\n",
        "\n",
        "    if mixup > 0:\n",
        "      mixup = 1.0 * mixup # Convert to float, as tf.distributions.Beta requires floats.\n",
        "      beta = tf.distributions.Beta(mixup, mixup)\n",
        "      lam = beta.sample(batch_size)\n",
        "      ll = tf.expand_dims(tf.expand_dims(tf.expand_dims(lam, -1), -1), -1)\n",
        "      images = ll * images + (1 - ll) * cshift(images)\n",
        "      labels = lam * labels + (1 - lam) * cshift(labels)\n",
        "\n",
        "  return images, labels"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "siI7jT2RUvYP",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def augment_data(dataset, dataset_labels, augementation_factor=1, use_random_rotation=True, use_random_shear=True, use_random_shift=True, use_random_zoom=True):\n",
        "\taugmented_image = []\n",
        "\taugmented_image_labels = []\n",
        "\n",
        "\tfor num in range (0, dataset.shape[0]):\n",
        "\n",
        "\t\tfor i in range(0, augementation_factor):\n",
        "\t\t\t# original image:\n",
        "\t\t\taugmented_image.append(dataset[num])\n",
        "\t\t\taugmented_image_labels.append(dataset_labels[num])\n",
        "\n",
        "\t\t\tif use_random_rotation:\n",
        "\t\t\t\taugmented_image.append(tf.contrib.keras.preprocessing.image.random_rotation(dataset[num], 20, row_axis=0, col_axis=1, channel_axis=2))\n",
        "\t\t\t\taugmented_image_labels.append(dataset_labels[num])\n",
        "\n",
        "\t\t\tif use_random_shear:\n",
        "\t\t\t\taugmented_image.append(tf.contrib.keras.preprocessing.image.random_shear(dataset[num], 0.2, row_axis=0, col_axis=1, channel_axis=2))\n",
        "\t\t\t\taugmented_image_labels.append(dataset_labels[num])\n",
        "\n",
        "\t\t\tif use_random_shift:\n",
        "\t\t\t\taugmented_image.append(tf.contrib.keras.preprocessing.image.random_shift(dataset[num], 0.2, 0.2, row_axis=0, col_axis=1, channel_axis=2))\n",
        "\t\t\t\taugmented_image_labels.append(dataset_labels[num])\n",
        "\n",
        "# \t\t\tif use_random_zoom:\n",
        "# \t\t\t\taugmented_image.append(tf.contrib.keras.preprocessing.image.random_zoom(dataset[num], 0.9, row_axis=0, col_axis=1, channel_axis=2))\n",
        "# \t\t\t\taugmented_image_labels.append(dataset_labels[num])\n",
        "\n",
        "\treturn np.array(augmented_image), np.array(augmented_image_labels)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Of2PvjkIQikg",
        "colab_type": "code",
        "outputId": "119fd4de-b704-42d7-f5ef-a0143333c68e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "(X_train.shape)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(55000, 32, 32, 1)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "metadata": {
        "id": "RbERCioKL6dc",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# # These can be any tensors of matching type and dimensions.\n",
        "# images = tf.placeholder(tf.uint8, shape=(None, None, None, 3))\n",
        "# labels = tf.placeholder(tf.uint64, shape=(None))\n",
        "\n",
        "# images, labels = augment(tf.convert_to_tensor(X_train), tf.convert_to_tensor(y_train),\n",
        "#                          horizontal_flip=True, rotate=15, crop_probability=0.8, mixup=4)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "-bwtFMk5U0HH",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "images, labels = augment_data(X_train, y_train)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "VZA2k-ZQR7kU",
        "colab_type": "code",
        "outputId": "37a6b247-61c9-4cb2-f531-b89b0d8c77e8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "images.shape"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(220000, 32, 32, 1)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "metadata": {
        "id": "Eb8X_yO6QDIx",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "merged = tf.summary.merge_all()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "MbueMHwfEbi4",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "EPOCHS = 66"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "INEVMxw4R0Nu",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# mnist.test.images"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "jYoZ96svRxfO",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Load the test set\n",
        "x_test = mnist.test.images\n",
        "y_test = mnist.test.labels\n",
        "\n",
        "# Initialize the embedding variable with the shape of our desired tensor\n",
        "tensor_shape = (x_test.shape[0] , logits.get_shape()[1].value) # [test_set , h1] = [10000 , 200]\n",
        "embedding_var = tf.Variable(tf.zeros(tensor_shape), \n",
        "                            name='logits_embedding')\n",
        "# assign the tensor that we want to visualize to the embedding variable\n",
        "embedding_assign = embedding_var.assign(logits) "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "8QrvPzlGSN6S",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "EPOCHS = 2"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "bKU1MIyJCy6Y",
        "colab_type": "code",
        "outputId": "502298a4-4380-4445-8ce7-e1d57a79945a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 217
        }
      },
      "cell_type": "code",
      "source": [
        "with tf.Session() as sess:\n",
        "    sess.run(tf.global_variables_initializer())\n",
        "    num_examples = len(X_train)\n",
        "    best_accuracy_valid = 0\n",
        "    train_writer = tf.summary.FileWriter(logs_path, sess.graph)\n",
        "\n",
        "    print(\"Training...\")\n",
        "    print()\n",
        "    for i in range(EPOCHS):\n",
        "        X_train, y_train = shuffle(images, labels)\n",
        "        for offset in range(0, num_examples, BATCH_SIZE):\n",
        "            end = offset + BATCH_SIZE\n",
        "            batch_x, batch_y = X_train[offset:end], y_train[offset:end]\n",
        "            tr_op,summary_tr  = sess.run([training_operation,merged], feed_dict={x: batch_x, y: batch_y, is_testing : False})\n",
        "            train_writer.add_summary(summary_tr, i)\n",
        "            prob = sess.run(new_prob)\n",
        "        print(prob)\n",
        "        validation_accuracy = evaluate(X_validation, y_validation)\n",
        "        print(\"EPOCH {} ...\".format(i+1))\n",
        "        print(\"Validation Accuracy = {:.3f}\".format(validation_accuracy))\n",
        "        print()\n",
        "        if (validation_accuracy >= best_accuracy_valid):\n",
        "          best_accuracy_valid = validation_accuracy\n",
        "          saver.save(sess, './lenet')\n",
        "        \n",
        "#     saver.save(sess, './lenet')\n",
        "    print(\"Model saved\")"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training...\n",
            "\n",
            "0.9176275\n",
            "EPOCH 1 ...\n",
            "Validation Accuracy = 0.976\n",
            "\n",
            "0.8421243\n",
            "EPOCH 2 ...\n",
            "Validation Accuracy = 0.983\n",
            "\n",
            "Model saved\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "hUc5BqhdSVtq",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "img_w = 32\n",
        "img_h = 32"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "tr1bfaPASS3n",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from tensorflow.contrib.tensorboard.plugins import projector\n",
        "\n",
        "# Create a config object to write the configuration parameters\n",
        "config = projector.ProjectorConfig()\n",
        "\n",
        "# Add embedding variable\n",
        "embedding = config.embeddings.add()\n",
        "embedding.tensor_name = embedding_var.name\n",
        "\n",
        "# Link this tensor to its metadata file (e.g. labels) -> we will create this file later\n",
        "embedding.metadata_path = 'metadata.tsv'\n",
        "\n",
        "# Specify where you find the sprite. -> we will create this image later\n",
        "embedding.sprite.image_path = 'sprite_images.png'\n",
        "embedding.sprite.single_image_dim.extend([img_w, img_h])\n",
        "\n",
        "# Write a projector_config.pbtxt in the logs_path.\n",
        "# TensorBoard will read this file during startup.\n",
        "projector.visualize_embeddings(train_writer, config)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "6JnCs7sUUcLl",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# # Run session to evaluate the tensor\n",
        "# with tf.Session() as sess:\n",
        "#   x_test_fc1 = sess.run(embedding_assign, feed_dict={x: X_test})\n",
        "\n",
        "#   # Save the tensor in model.ckpt file\n",
        "#   saver = tf.train.Saver()\n",
        "#   saver.save(sess, os.path.join(logs_path, \"model.ckpt\"), i)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "yVIeYA_cCy6e",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Evaluate the Model\n",
        "Once you are completely satisfied with your model, evaluate the performance of the model on the test set.\n",
        "\n",
        "Be sure to only do this once!\n",
        "\n",
        "If you were to measure the performance of your trained model on the test set, then improve your model, and then measure the performance of your model on the test set again, that would invalidate your test results. You wouldn't get a true measure of how well your model would perform against real data.\n",
        "\n",
        "You do not need to modify this section."
      ]
    },
    {
      "metadata": {
        "id": "O4b5lSD5H_q9",
        "colab_type": "code",
        "outputId": "fbc4435c-e059-4ddb-b5cb-c683146c096c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "cell_type": "code",
      "source": [
        "with tf.Session() as sess:\n",
        "    saver.restore(sess, tf.train.latest_checkpoint('.'))\n",
        "\n",
        "    test_accuracy = evaluate(X_validation, y_validation)\n",
        "    print(\"Validation Accuracy = {:.3f}\".format(test_accuracy))"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Restoring parameters from ./lenet\n",
            "Validation Accuracy = 0.983\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "bJONJ7CeCy6f",
        "colab_type": "code",
        "outputId": "414040ee-6230-416c-b4bb-4c376487c143",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "cell_type": "code",
      "source": [
        "with tf.Session() as sess:\n",
        "    saver.restore(sess, tf.train.latest_checkpoint('.'))\n",
        "    x_test_fc1 = sess.run(embedding_assign, feed_dict={x: X_test,is_testing: True})\n",
        "\n",
        "    # Save the tensor in model.ckpt file\n",
        "    saver = tf.train.Saver()\n",
        "    saver.save(sess, os.path.join(logs_path, \"model.ckpt\"), i)\n",
        "    test_accuracy = evaluate(X_test, y_test)\n",
        "    print(\"Test Accuracy = {:.3f}\".format(test_accuracy))"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Restoring parameters from ./lenet\n",
            "Test Accuracy = 0.986\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "qvRqnUrrCy6m",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def write_sprite_image(filename, images):\n",
        "    \"\"\"\n",
        "        Create a sprite image consisting of sample images\n",
        "        :param filename: name of the file to save on disk\n",
        "        :param shape: tensor of flattened images\n",
        "    \"\"\"\n",
        "\n",
        "    # Invert grayscale image\n",
        "    images = 1 - images\n",
        "\n",
        "    # Calculate number of plot\n",
        "    n_plots = int(np.ceil(np.sqrt(images.shape[0])))\n",
        "\n",
        "    # Make the background of sprite image\n",
        "    sprite_image = np.ones((img_h * n_plots, img_w * n_plots))\n",
        "\n",
        "    for i in range(n_plots):\n",
        "        for j in range(n_plots):\n",
        "            img_idx = i * n_plots + j\n",
        "            if img_idx < images.shape[0]:\n",
        "                img = images[img_idx]\n",
        "                sprite_image[i * img_h:(i + 1) * img_h,\n",
        "                j * img_w:(j + 1) * img_w] = img\n",
        "\n",
        "    plt.imsave(filename, sprite_image, cmap='gray')\n",
        "    print('Sprite image saved in {}'.format(filename))\n",
        "\n",
        "def write_metadata(filename, labels):\n",
        "    \"\"\"\n",
        "            Create a metadata file image consisting of sample indices and labels\n",
        "            :param filename: name of the file to save on disk\n",
        "            :param shape: tensor of labels\n",
        "    \"\"\"\n",
        "    with open(filename, 'w') as f:\n",
        "        f.write(\"Index\\tLabel\\n\")\n",
        "        for index, label in enumerate(labels):\n",
        "            f.write(\"{}\\t{}\\n\".format(index, label))\n",
        "\n",
        "    print('Metadata file saved in {}'.format(filename))\n",
        " "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "vE0ULEJ7XGlH",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "d54cd52c-239b-41a4-b087-646e9ec38096"
      },
      "cell_type": "code",
      "source": [
        "y_test.shape"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(10000,)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 44
        }
      ]
    },
    {
      "metadata": {
        "id": "F-zHFsYHWvrv",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "be2cafa7-2c82-4239-fbf5-258f53b4ad30"
      },
      "cell_type": "code",
      "source": [
        "# Reshape images from vector to matrix\n",
        "x_test_images = np.reshape(np.array(X_test), (-1, img_w, img_h))\n",
        "# Reshape labels from one-hot-encode to index\n",
        "x_test_labels = y_test\n",
        "\n",
        "write_sprite_image(os.path.join(logs_path, 'sprite_images.png'), x_test_images)\n",
        "write_metadata(os.path.join(logs_path, 'metadata.tsv'), x_test_labels)"
      ],
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sprite image saved in ./logs/embedding/sprite_images.png\n",
            "Metadata file saved in ./logs/embedding/metadata.tsv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "uPVpeenpW8sp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 254
        },
        "outputId": "fbede865-efcb-4d7c-f74d-09063730074b"
      },
      "cell_type": "code",
      "source": [
        "!wget https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip\n",
        "!unzip -o ngrok-stable-linux-amd64.zip\n",
        "# logdir=logs/embedding/\n",
        "LOG_DIR = 'logs/embedding/'\n",
        "get_ipython().system_raw(\n",
        "    'tensorboard --logdir {} --host 0.0.0.0 --port 6006 &'\n",
        "    .format(LOG_DIR)\n",
        ")\n",
        "\n",
        "get_ipython().system_raw('./ngrok http 6006 &')"
      ],
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2019-04-01 05:29:06--  https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip\n",
            "Resolving bin.equinox.io (bin.equinox.io)... 54.173.32.212, 52.7.169.168, 34.196.237.103, ...\n",
            "Connecting to bin.equinox.io (bin.equinox.io)|54.173.32.212|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 14977695 (14M) [application/octet-stream]\n",
            "Saving to: ‘ngrok-stable-linux-amd64.zip.1’\n",
            "\n",
            "ngrok-stable-linux- 100%[===================>]  14.28M  14.4MB/s    in 1.0s    \n",
            "\n",
            "2019-04-01 05:29:07 (14.4 MB/s) - ‘ngrok-stable-linux-amd64.zip.1’ saved [14977695/14977695]\n",
            "\n",
            "Archive:  ngrok-stable-linux-amd64.zip\n",
            "  inflating: ngrok                   \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "sULv8PgRXp2j",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "e4cd2565-b617-4399-c2fe-47c0a35d8556"
      },
      "cell_type": "code",
      "source": [
        "ngrok_url = !curl -s http://localhost:4040/api/tunnels | python -c \\\n",
        "    \"import sys, json; print(json.load(sys.stdin)['tunnels'][0]['public_url'])\"\n",
        "        \n",
        "ngrok_url = ngrok_url[0].replace(\"'\", '')\n",
        "print(ngrok_url)"
      ],
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "https://15966ccb.ngrok.io\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "q7TjVgm-X6N1",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 921
        },
        "outputId": "deea439e-89cf-499a-fe67-e686b64b23fe"
      },
      "cell_type": "code",
      "source": [
        "from IPython.display import IFrame\n",
        "\n",
        "IFrame(ngrok_url, width=700, height=900)"
      ],
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "        <iframe\n",
              "            width=\"700\"\n",
              "            height=\"900\"\n",
              "            src=\"https://15966ccb.ngrok.io\"\n",
              "            frameborder=\"0\"\n",
              "            allowfullscreen\n",
              "        ></iframe>\n",
              "        "
            ],
            "text/plain": [
              "<IPython.lib.display.IFrame at 0x7fe269bc6d30>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 51
        }
      ]
    },
    {
      "metadata": {
        "id": "u5OZDSZ7X9eO",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}