Deepdream_vgg16_pytorch.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "toc": {
      "base_numbering": 1,
      "nav_menu": {},
      "number_sections": true,
      "sideBar": true,
      "skip_h1_title": false,
      "title_cell": "Table of Contents",
      "title_sidebar": "Contents",
      "toc_cell": false,
      "toc_position": {},
      "toc_section_display": true,
      "toc_window_display": false
    },
    "varInspector": {
      "cols": {
        "lenName": 16,
        "lenType": 16,
        "lenVar": 40
      },
      "kernels_config": {
        "python": {
          "delete_cmd_postfix": "",
          "delete_cmd_prefix": "del ",
          "library": "var_list.py",
          "varRefreshCmd": "print(var_dic_list())"
        },
        "r": {
          "delete_cmd_postfix": ") ",
          "delete_cmd_prefix": "rm(",
          "library": "var_list.r",
          "varRefreshCmd": "cat(var_dic_list()) "
        }
      },
      "types_to_exclude": [
        "module",
        "function",
        "builtin_function_or_method",
        "instance",
        "_Feature"
      ],
      "window_display": false
    },
    "colab": {
      "name": "Deepdream_vgg16_pytorch.ipynb",
      "provenance": []
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "collapsed": true,
        "pycharm": {
          "name": "#%% md\n"
        },
        "id": "Qbjyg5YeaPU1"
      },
      "source": [
        "# Deep dream with VGG16 model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-03-04T12:43:02.228433Z",
          "start_time": "2021-03-04T12:43:00.300596Z"
        },
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "fUAQ7_RVaPVM"
      },
      "source": [
        "import os\n",
        "import torch\n",
        "import numpy as np\n",
        "from matplotlib import pyplot as plt\n",
        "from torch.autograd import Variable\n",
        "%matplotlib inline\n",
        "import torch\n",
        "from torch.autograd import Variable\n",
        "from torch.optim import SGD\n",
        "from torchvision import models, transforms\n",
        "import PIL\n",
        "import os\n",
        "import matplotlib\n",
        "from matplotlib import animation\n",
        "from IPython.display import HTML\n",
        "\n",
        "import scipy.ndimage as ndimage\n",
        "\n",
        "%matplotlib inline\n",
        "\n",
        "import scipy.ndimage as nd\n",
        "import PIL.Image\n",
        "from IPython.display import clear_output, Image, display\n",
        "from io import BytesIO\n",
        "import torchvision\n",
        "from torchvision.models import vgg16\n",
        "from torchvision.transforms import transforms\n",
        "from torchsummary import summary\n",
        "torch.manual_seed(0)\n",
        "torch.backends.cudnn.deterministic = True\n",
        "torch.backends.cudnn.benchmark = False\n",
        "np.random.seed(0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mpmt7XmGb39Q"
      },
      "source": [
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\r\n",
        "device"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "start_time": "2021-03-04T12:43:00.306Z"
        },
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "QwtcxljYaPVO"
      },
      "source": [
        "model = vgg16(pretrained=True)\n",
        "model.load_state_dict(torch.load('/root/.cache/torch/hub/checkpoints/vgg16-397923af.pth'))\n",
        "model.to(device)\n",
        "summary(model, input_size=(3,32,32))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "start_time": "2021-03-04T12:43:00.322Z"
        },
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "lLbSSBo3aPVT"
      },
      "source": [
        "def showarray(a, fmt='jpeg'):\n",
        "    a = np.uint8(np.clip(a, 0, 255))\n",
        "    f = BytesIO()\n",
        "    PIL.Image.fromarray(a).save(f, fmt)\n",
        "    display(Image(data=f.getvalue()))\n",
        "\n",
        "def showtensor(a):\n",
        "    mean = np.array([0.485, 0.456, 0.406]).reshape([1, 1, 3])\n",
        "    std = np.array([0.229, 0.224, 0.225]).reshape([1, 1, 3])\n",
        "    inp = a[0, :, :, :]\n",
        "    inp = inp.transpose(1, 2, 0)\n",
        "    inp = std * inp + mean\n",
        "    inp *= 255\n",
        "    showarray(inp)\n",
        "    clear_output(wait=True)\n",
        "\n",
        "normalise = transforms.Compose([\n",
        "    transforms.ToTensor(),\n",
        "    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
        "])\n",
        "\n",
        "normalise_resize = transforms.Compose([\n",
        "    transforms.Resize((224, 224)),\n",
        "    transforms.ToTensor(),\n",
        "    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
        "])\n",
        "invTrans = transforms.Compose([\n",
        "               transforms.Normalize([-0.485, -0.456, -0.406], [1/0.229, 1/0.224, 1/0.225]),\n",
        "           ])\n",
        "\n",
        "def init_image(size=(400, 400, 3)):\n",
        "    img = PIL.Image.fromarray(np.uint8(np.full(size, 150)))\n",
        "    img = PIL.Image.fromarray(np.uint8(np.random.uniform(150, 180, size)))\n",
        "    img_tensor = normalise(img).unsqueeze(0)\n",
        "    img_np = img_tensor.numpy()\n",
        "    return img, img_tensor, img_np\n",
        "\n",
        "def load_image(path, resize=False, size=None):\n",
        "    img = PIL.Image.open(path)\n",
        "\n",
        "    if size is not None:\n",
        "        img.thumbnail(size, PIL.Image.ANTIALIAS)\n",
        "\n",
        "    if resize:\n",
        "        img_tensor = normalise_resize(img).unsqueeze(0)\n",
        "    else:\n",
        "        img_tensor = normalise(img).unsqueeze(0)\n",
        "    img_np = img_tensor.numpy()\n",
        "    return img, img_tensor, img_np\n",
        "\n",
        "\n",
        "\n",
        "def imshow(img, lables = None ):\n",
        "    try:\n",
        "        img = invTrans(img)    # unnormalize\n",
        "        npimg = img.numpy()\n",
        "    except Exception as e:\n",
        "        npimg = img\n",
        "    fig = plt.figure(figsize = (200,50))\n",
        "    ax = fig.add_subplot(1,1,1)\n",
        "    if bool(lables):\n",
        "        ax.set_xticks(range(len(lables)))\n",
        "        ax.set_xticklabels(lables,fontsize=\"medium\")\n",
        "    ax.imshow(np.transpose(npimg, (1, 2, 0)))\n",
        "    plt.show()\n",
        "\n",
        "def image_to_variable(image, requires_grad=False):\n",
        "    image = Variable(image.to(device), requires_grad=requires_grad)\n",
        "    return image\n",
        "\n",
        "for param in model.parameters():\n",
        "    param.requires_grad = False"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "start_time": "2021-03-04T12:43:00.330Z"
        },
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "jHJcFAqBaPVW"
      },
      "source": [
        "def octaver_fn(model, base_img, step_fn, octave_n=6, octave_scale=1.4, iter_n=10, **step_args):\n",
        "    octaves = [base_img]\n",
        "\n",
        "    for i in range(octave_n - 1):\n",
        "        octaves.append(nd.zoom(octaves[-1], (1, 1, 1.0 / octave_scale, 1.0 / octave_scale), order=1))\n",
        "\n",
        "    detail = np.zeros_like(octaves[-1])\n",
        "    for octave, octave_base in enumerate(octaves[::-1]):\n",
        "        h, w = octave_base.shape[-2:]\n",
        "\n",
        "        if octave > 0:\n",
        "            h1, w1 = detail.shape[-2:]\n",
        "            detail = nd.zoom(detail, (1, 1, 1.0 * h / h1, 1.0 * w / w1), order=1)\n",
        "\n",
        "        src = octave_base + detail\n",
        "\n",
        "        for i in range(iter_n):\n",
        "            src = step_fn(model, src, **step_args)\n",
        "\n",
        "        detail = src.numpy() - octave_base\n",
        "\n",
        "    return src"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "start_time": "2021-03-04T12:43:00.336Z"
        },
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "YGw4F6UcaPVY"
      },
      "source": [
        "def filter_step(model, img, layer_index, filter_index, step_size=5, display=True, use_L2=False):\n",
        "\n",
        "    mean = np.array([0.485, 0.456, 0.406]).reshape([3, 1, 1])\n",
        "    std = np.array([0.229, 0.224, 0.225]).reshape([3, 1, 1])\n",
        "\n",
        "    model.zero_grad()\n",
        "\n",
        "    img_var = image_to_variable(torch.Tensor(img), requires_grad=True)\n",
        "    optimizer = SGD([img_var], lr=step_size, weight_decay=1e-4)\n",
        "\n",
        "    x = img_var\n",
        "    for index, layer in enumerate(model.features):\n",
        "        x = layer(x)\n",
        "        if index == layer_index:\n",
        "            break\n",
        "\n",
        "    output = x[0, filter_index]\n",
        "    loss = output.norm() #torch.mean(output)\n",
        "    loss.backward()\n",
        "\n",
        "    if use_L2:\n",
        "        #L2 normalization on gradients\n",
        "        mean_square = torch.Tensor([torch.mean(img_var.grad.data ** 2) + 1e-5])\n",
        "        mean_square = mean_square.to(device)\n",
        "        img_var.grad.data /= torch.sqrt(mean_square)\n",
        "        img_var.data.add_(img_var.grad.data * step_size)\n",
        "    else:\n",
        "        optimizer.step()\n",
        "\n",
        "    result = img_var.data.cpu().numpy()\n",
        "    result[0, :, :, :] = np.clip(result[0, :, :, :], -mean / std, (1 - mean) / std)\n",
        "\n",
        "    # if display:\n",
        "        # showtensor(result)\n",
        "\n",
        "    return torch.Tensor(result)\n",
        "\n",
        "def visualize_filter(model, base_img, layer_index, filter_index,\n",
        "                     octave_n=6, octave_scale=1.4, iter_n=10,\n",
        "                     step_size=5, display=True, use_L2=False):\n",
        "\n",
        "    return octaver_fn(\n",
        "                model, base_img, step_fn=filter_step,\n",
        "                octave_n=octave_n, octave_scale=octave_scale,\n",
        "                iter_n=iter_n, layer_index=layer_index,\n",
        "                filter_index=filter_index, step_size=step_size,\n",
        "                display=display, use_L2=use_L2\n",
        "            )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "start_time": "2021-03-04T12:43:00.340Z"
        },
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "4H9SiHNJaPVa"
      },
      "source": [
        "def show_layer(model, layer_num, filter_start=10, filter_end=20, step_size=7, use_L2=False):\n",
        "    filters = []\n",
        "    titles = []\n",
        "\n",
        "    _, _, img_np = init_image(size=(600, 600, 3))\n",
        "    for i in range(filter_start, filter_end):\n",
        "        title = \"Layer {} Filter {}\".format(layer_num , i)\n",
        "        print(title)\n",
        "        filter = visualize_filter(model, img_np, layer_num, filter_index=i, octave_n=2, iter_n=20, step_size=step_size, display=True, use_L2=use_L2)\n",
        "        # filter_img = tensor_to_img(filter)\n",
        "        # filter_img.save(title + \".jpg\")\n",
        "        filters.append(filter.view((-1,600,600)))\n",
        "        titles.append(title)\n",
        "\n",
        "    imshow(torchvision.utils.make_grid(filters, nrow=len(titles), scale_each=4), lables=titles)\n",
        "    return filters, titles"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "start_time": "2021-03-04T12:43:00.346Z"
        },
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "IsgE4W7laPVc"
      },
      "source": [
        "images, titles = show_layer(model, 1, use_L2=True, step_size=0.05)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "flVomQemaPVc"
      },
      "source": [
        "images, titles = show_layer(model, 25,filter_start=10, filter_end=12, use_L2=True, step_size=0.05)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "MGfGJGR0aPVd"
      },
      "source": [
        "def deepDream(model, layer_num, filter_start=10, filter_end=20, step_size=7, use_L2=False):\n",
        "    filters = []\n",
        "    titles = []\n",
        "\n",
        "    _, _, img_np = load_image('.pytorch/sample/image/Hacktoberfest-Blog-983x1024.jpg', size=(600, 600))\n",
        "    for i in range(filter_start, filter_end):\n",
        "        title = \"Layer {} Filter {}\".format(layer_num , i)\n",
        "        print(title)\n",
        "        filter = visualize_filter(model, img_np, layer_num, filter_index=i, octave_n=2, iter_n=20, step_size=step_size, display=True, use_L2=use_L2)\n",
        "        # filter_img = tensor_to_img(filter)\n",
        "        # filter_img.save(title + \".jpg\")\n",
        "        filters.append(filter.view((-1,600,600)))\n",
        "        titles.append(title)\n",
        "\n",
        "    imshow(torchvision.utils.make_grid(filters, nrow=len(titles), scale_each=4), lables=titles)\n",
        "    return filters, titles"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "gVYPUP-MaPVe"
      },
      "source": [
        "!curl -L 'https://source.unsplash.com/qkV6WPoVrN8/600x600' --create-dirs -o .pytorch/sample/image/Hacktoberfest-Blog-983x1024.jpg\r\n",
        "imshow(torch.from_numpy(load_image('.pytorch/sample/image/Hacktoberfest-Blog-983x1024.jpg', size=(600, 600))[-1][0]))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "pPmwl6fFaPVf"
      },
      "source": [
        "images, titles = deepDream(model, 25,filter_start=10, filter_end=20, use_L2=True, step_size=0.05)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "-eZdzTp8aPVg"
      },
      "source": [
        "images, titles = deepDream(model, 29,filter_start=10, filter_end=20, use_L2=True, step_size=0.05)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "swpSO8UUcfPd"
      },
      "source": [
        "!git clone https://github.com/realsarm/deepLearningRecipes.git # to restore the saved data\r\n",
        "!mv deepLearningRecipes/* ."
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}