pytorch-tutorial.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Untitled1.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "145d5b1ad8de42639c5e24ead2acefde": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0bfc4415442547b087d7ae75b44d7cd5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e4b6601efcd24d44bad1223256c1c09d",
              "IPY_MODEL_c620fc6ef0504c9fabd649e8f7729527"
            ]
          }
        },
        "0bfc4415442547b087d7ae75b44d7cd5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e4b6601efcd24d44bad1223256c1c09d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2f7abe16be104d2a95da02bded69dece",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_567931982f0a488abd21fb97a9578fbc"
          }
        },
        "c620fc6ef0504c9fabd649e8f7729527": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0dce977dd7424df687dceac745d17c93",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 170500096/? [00:20&lt;00:00, 52960154.52it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d0e53f22b4194cf0bd2766207b5cd0cd"
          }
        },
        "2f7abe16be104d2a95da02bded69dece": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "567931982f0a488abd21fb97a9578fbc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0dce977dd7424df687dceac745d17c93": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d0e53f22b4194cf0bd2766207b5cd0cd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6f347c8cd2bf477ab1f98b0fef342d3f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d33bc4af435b4d7594e21acf07f7139b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_af5dddd10bf94020b67aba3b6445bfd8",
              "IPY_MODEL_9c7facac951e47d5a7b95952ff0ae426"
            ]
          }
        },
        "d33bc4af435b4d7594e21acf07f7139b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "af5dddd10bf94020b67aba3b6445bfd8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e30087e7c8dd480e8c4c260a55acf600",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 46827520,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 46827520,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5d1933afd499484ca73ac1d3a01edcea"
          }
        },
        "9c7facac951e47d5a7b95952ff0ae426": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_39543067be5b44a7a6f69a7cc80c0772",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 44.7M/44.7M [00:00&lt;00:00, 99.4MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b76255a84ebd485cac680ce7e036d36d"
          }
        },
        "e30087e7c8dd480e8c4c260a55acf600": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5d1933afd499484ca73ac1d3a01edcea": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "39543067be5b44a7a6f69a7cc80c0772": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b76255a84ebd485cac680ce7e036d36d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/xhxuciedu/CS175/blob/master/pytorch-tutorial.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7FKQCFzXDnGH"
      },
      "source": [
        "import numpy as np\n",
        "import sys\n",
        "import torch \n",
        "import torchvision\n",
        "import torch.nn as nn\n",
        "import torchvision.transforms as transforms"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q_w92gMaP6N1"
      },
      "source": [
        "### Check Package Versions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uYr2YRtWPkel",
        "outputId": "61553dc9-da01-4ee6-b09c-940b39f33c69",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "print('__Python VERSION:', sys.version)\n",
        "print('__PyTorch VERSION:', torch.__version__)\n",
        "print('__CUDNN VERSION:', torch.backends.cudnn.version())\n",
        "print('__Number CUDA Devices:', torch.cuda.device_count())"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "__Python VERSION: 3.6.9 (default, Oct  8 2020, 12:12:24) \n",
            "[GCC 8.4.0]\n",
            "__PyTorch VERSION: 1.7.0+cu101\n",
            "__CUDNN VERSION: 7603\n",
            "__Number CUDA Devices: 1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5Eq_6EZ0QM23"
      },
      "source": [
        "### PyTorch\n",
        "What is PyTorch?\n",
        "\n",
        "It’s a Python based scientific computing package targeted at two sets of audiences:\n",
        "\n",
        "* A replacement for numpy to use the power of GPUs\n",
        "* a deep learning research platform that provides maximum flexibility and speed\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cSa2TTQfQ3nS"
      },
      "source": [
        "### Tensors\n",
        "\n",
        "Tensors are similar to numpy’s ndarrays, with the addition being that Tensors can also be used on a GPU to accelerate computing.\n",
        "\n",
        "\n",
        "Construct a 5x3 matrix, uninitialized"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xTW9C9zVRLWy",
        "outputId": "f447dd76-6dcf-4266-f2b0-488d6e600c3f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "x = torch.Tensor(5, 3)\n",
        "print(x)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[5.1905e-36, 0.0000e+00, 3.7835e-44],\n",
            "        [0.0000e+00,        nan, 0.0000e+00],\n",
            "        [1.3733e-14, 6.4069e+02, 4.3066e+21],\n",
            "        [1.1824e+22, 4.3066e+21, 6.3828e+28],\n",
            "        [3.8016e-39, 0.0000e+00, 0.0000e+00]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i74pPBySRRDZ",
        "outputId": "3745996d-f1b8-42bd-e138-6ec94f4a10f0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "# get its size\n",
        "y = torch.rand(5, 3)\n",
        "print(x + y)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[4.2567e-01, 8.7422e-01, 4.7108e-01],\n",
            "        [5.0496e-01,        nan, 2.6444e-01],\n",
            "        [6.7579e-01, 6.4092e+02, 4.3066e+21],\n",
            "        [1.1824e+22, 4.3066e+21, 6.3828e+28],\n",
            "        [8.4644e-01, 1.8018e-01, 2.2816e-01]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O0xeSB-3Rd1L",
        "outputId": "b696a376-5d3e-4d1f-f67f-558e145ca8dd",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "# Addition: in-place\n",
        "y.add_(x)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[4.2567e-01, 8.7422e-01, 4.7108e-01],\n",
              "        [5.0496e-01,        nan, 2.6444e-01],\n",
              "        [6.7579e-01, 6.4092e+02, 4.3066e+21],\n",
              "        [1.1824e+22, 4.3066e+21, 6.3828e+28],\n",
              "        [8.4644e-01, 1.8018e-01, 2.2816e-01]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QagJi5H9YT_M"
      },
      "source": [
        "### Create tensors"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zF881O3lYfwc"
      },
      "source": [
        "# random\n",
        "v = torch.rand(2, 3)            # Initialize with random number (uniform distribution)\n",
        "v = torch.randn(2, 3)           # With normal distribution (SD=1, mean=0)\n",
        "v = torch.randperm(4)   \n",
        "\n",
        "# ones\n",
        "eye = torch.eye(3)              # Create an identity 3x3 tensor\n",
        "v = torch.ones(10)              # A tensor of size 10 containing all ones\n",
        "v = torch.ones(2, 1, 2, 1)      # Size 2x1x2x1\n",
        "v = torch.ones_like(eye)        # A tensor with same shape as eye. Fill it with 1.\n",
        "\n",
        "# zeros\n",
        "v = torch.zeros(10) \n",
        "\n",
        "# range of values\n",
        "v = torch.arange(5)             # similar to range(5) but creating a Tensor\n",
        "v = torch.arange(0, 5, step=1)  # Size 5. Similar to range(0, 5, 1)\n",
        "\n",
        "# linear or log scale\n",
        "v = torch.linspace(1, 10, steps=10) # Create a Tensor with 10 linear points for (1, 10) inclusively\n",
        "v = torch.logspace(start=-10, end=10, steps=5) # Size 5: 1.0e-10 1.0e-05 1.0e+00, 1.0e+05, 1.0e+10"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j0_vX6QFbP9r"
      },
      "source": [
        "### Dot product, component-wide product, matrix multiplication, "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hbf5A2rTYonK"
      },
      "source": [
        "# Dot product of 2 tensors\n",
        "r = torch.dot(torch.Tensor([4, 2]), torch.Tensor([3, 1])) # 14"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F-AZEEfEYosW",
        "outputId": "46e5cc53-28ea-4530-dc3a-b568ed158fcc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "# component-wise product\n",
        "torch.Tensor([4, 2])* torch.Tensor([3, 1])"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([12.,  2.])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bnfFZpz4a0JG"
      },
      "source": [
        "# Matrix x Matrix\n",
        "# Size 2x4\n",
        "mat1 = torch.randn(2, 3)\n",
        "mat2 = torch.randn(3, 4)\n",
        "r = torch.mm(mat1, mat2)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9fNQ8Aw-ahBH"
      },
      "source": [
        "# Batch Matrix x Matrix\n",
        "# Size 10x3x5\n",
        "batch1 = torch.randn(10, 3, 4)\n",
        "batch2 = torch.randn(10, 4, 5)\n",
        "r = torch.bmm(batch1, batch2)"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zygHVhSba0eN"
      },
      "source": [
        "###Squeeze and unsqueeze"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vylLWkFfaicW"
      },
      "source": [
        "t = torch.ones(2,1,2,1) # Size 2x1x2x1\n",
        "r = torch.squeeze(t)     # Size 2x2\n",
        "r = torch.squeeze(t, 1)  # Squeeze dimension 1: Size 2x2x1\n",
        "\n",
        "# Un-squeeze a dimension\n",
        "x = torch.Tensor([1, 2, 3])\n",
        "r = torch.unsqueeze(x, 0)       # Size: 1x3\n",
        "r = torch.unsqueeze(x, 1)       # Size: 3x1"
      ],
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v6gEsNF3a-26"
      },
      "source": [
        "### Transpose\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TwCePtvPa_hO",
        "outputId": "7ed9bf60-3811-4e77-ecba-22375c02c814",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "# Transpose dim 0 and 1\n",
        "v = torch.randn(3,2)\n",
        "r = torch.transpose(v, 0, 1)\n",
        "print(r.shape)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([2, 3])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G6wj7JQJRna5"
      },
      "source": [
        "### Numpy Bridge\n",
        "Converting a torch Tensor to a numpy array and vice versa is a breeze.\n",
        "\n",
        "The torch Tensor and numpy array will share their underlying memory locations, and changing one will change the other.\n",
        "\n",
        "Converting torch Tensor to numpy Array"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M-V3P-KeOdFY"
      },
      "source": [
        "# Create a numpy array.\n",
        "x = np.array([[1, 2], [3, 4]])\n",
        "\n",
        "# Convert the numpy array to a torch tensor.\n",
        "y = torch.from_numpy(x)\n",
        "\n",
        "# Convert the torch tensor to a numpy array.\n",
        "z = y.numpy()"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DrxFA9zrOhcw"
      },
      "source": [
        "# Conversion\n",
        "a = np.array([1, 2, 3])\n",
        "v = torch.from_numpy(a)         # Convert a numpy array to a Tensor\n",
        "\n",
        "b = v.numpy()                   # Tensor to numpy\n",
        "b[1] = -1                       # Numpy and Tensor share the same memory\n",
        "assert(a[1] == b[1])            # Change Numpy will also change the Tensor"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PkrFAPSzYDLa"
      },
      "source": [
        "### Reshape tensor"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TVJAK82AYB78",
        "outputId": "9e2843e5-a5d5-46b7-b206-98b74ef68b88",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "### Tensor resizing\n",
        "x = torch.randn(2, 3)            # Size 2x3\n",
        "y = x.view(6)                    # Resize x to size 6\n",
        "z = x.view(-1, 2)                # Size 3x2\n",
        "print(y.shape, z.shape)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([6]) torch.Size([3, 2])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cfoEcojSYTra"
      },
      "source": [
        ""
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CJnuUJgeSH5S"
      },
      "source": [
        "###CUDA Tensors\n",
        "\n",
        "All the Tensors on the CPU except a CharTensor support converting to NumPy and back.\n",
        "\n",
        "\n",
        "Tensors can be moved onto GPU using the .cuda function."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IPUVPxqbSP_8"
      },
      "source": [
        "# let us run this cell only if CUDA is available\n",
        "\n",
        "x = torch.rand(3,2)\n",
        "y = torch.rand(3,2)\n",
        "if torch.cuda.is_available():\n",
        "    x = x.cuda()\n",
        "    y = y.cuda()\n",
        "    x + y"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fUYQkqCUSiRT",
        "outputId": "ecf4a4bf-0567-4acc-b31d-a7849d3831a4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "x"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[0.9372, 0.0962],\n",
              "        [0.1581, 0.4592],\n",
              "        [0.4736, 0.4794]], device='cuda:0')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N20kEqktSQKm",
        "outputId": "a8aebea7-d8be-451e-af0d-ba09d41bc8da",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "y"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[0.6448, 0.5404],\n",
              "        [0.1072, 0.0278],\n",
              "        [0.8307, 0.5756]], device='cuda:0')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1mS3kiy9UNoR"
      },
      "source": [
        "## Autograd: automatic differentiation\n",
        "\n",
        "Central to all neural networks in PyTorch is autograd, a core torch package for automatic differentiation. \n",
        "\n",
        "\n",
        "The autograd package provides automatic differentiation for all operations on Tensors. It is a define-by-run framework, which means that your backprop is defined by how your code is run, and that every single iteration can be different.\n",
        "\n",
        "Let us see this in more simple terms with some examples."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PmxgzX_6U4rn"
      },
      "source": [
        "# create an variable\n",
        "x = torch.ones((2,2), requires_grad=True)\n",
        "\n",
        "# Do an operation of variable:\n",
        "y = x + 2\n",
        "\n",
        "# Do more operations on y\n",
        "z = y * y * 3\n",
        "out = z.mean()"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jdNxo2rxU4zr"
      },
      "source": [
        "# Gradients\n",
        "# ---------\n",
        "# let's backprop now\n",
        "# ``out.backward()`` is equivalent to doing ``out.backward(torch.Tensor([1.0]))``\n",
        "out.backward()"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UNK-RRBmU43f",
        "outputId": "6aa7d102-9a03-438c-f9da-dc9ffa0a2cca",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "###############################################################\n",
        "# print gradients d(out)/dx\n",
        "#\n",
        "\n",
        "print(x.grad)"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[4.5000, 4.5000],\n",
            "        [4.5000, 4.5000]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VUDiTPCeWGlU"
      },
      "source": [
        "You should have got a matrix of ``4.5``. Let’s call the ``out`` *Variable* $o$.\n",
        "We have that: $o = \\frac{1}{4}\\sum_i z_i$, \n",
        "$z_i = 3(x_i+2)^2$ and $z_i\\bigr\\rvert_{x_i=1} = 27$\n",
        "\n",
        "Therefore,\n",
        "$$\\frac{\\partial o}{\\partial x_i} = \\frac{3}{2}(x_i+2)$$ hence\n",
        "$$\\frac{\\partial o}{\\partial x_i}\\bigr\\rvert_{x_i=1} = \\frac{9}{2} = 4.5$$"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bKL_ex-DWs_L",
        "outputId": "7e235541-c8a0-428d-c7e0-e1770d61b717",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "# You can do many crazy things with autograd!\n",
        "x = torch.randn(3, requires_grad=True)\n",
        "\n",
        "y = x * 2\n",
        "while y.data.norm() < 1000:\n",
        "    y = y * 2\n",
        "\n",
        "print(y)"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([ 788.4969,   30.4932, -620.1824], grad_fn=<MulBackward0>)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vmSVgbJNWyqu",
        "outputId": "679ad403-d907-4049-8c68-e0ab5fb581c4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "gradients = torch.FloatTensor([0.1, 1.0, 0.0001])\n",
        "y.backward(gradients)\n",
        "\n",
        "print(x.grad)"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([5.1200e+01, 5.1200e+02, 5.1200e-02])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mQ5oVt1kFv6N"
      },
      "source": [
        "### Basic autograd example 1 "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Lx-5ce4DE2y-"
      },
      "source": [
        "# Create tensors.\n",
        "x = torch.tensor(1., requires_grad=True)\n",
        "w = torch.tensor(2., requires_grad=True)\n",
        "b = torch.tensor(3., requires_grad=True)"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mcy6p4nSE5Yt"
      },
      "source": [
        "# Build a computational graph.\n",
        "y = w * x + b    # y = 2 * x + 3"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gzfF4EDBGyJB"
      },
      "source": [
        "# Compute gradients.\n",
        "y.backward()"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i3hIZDh1GrJn",
        "outputId": "aace97a0-fe3e-428b-85f5-d21b0a39814e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "# Print out the gradients.\n",
        "print(x.grad)    # x.grad = 2 \n",
        "print(w.grad)    # w.grad = 1 \n",
        "print(b.grad)    # b.grad = 1 "
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor(2.)\n",
            "tensor(1.)\n",
            "tensor(1.)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iiS-D364E6JC",
        "outputId": "8e9520ff-0632-4525-a7fd-7af36066dc84",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "y.detach().numpy()"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(5., dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ahapqf3VF8Fn"
      },
      "source": [
        "### Basic autograd example 2  "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lgah0xn0GCxA",
        "outputId": "1ec1d3e5-f584-442b-8874-81d1efc3a1cb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "# Create tensors of shape (10, 3) and (10, 2).\n",
        "x = torch.randn(10, 3)\n",
        "y = torch.randn(10, 2)\n",
        "print(x.shape, y.shape)"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([10, 3]) torch.Size([10, 2])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oiQkdoJXGLeJ",
        "outputId": "9c97c190-879b-4d67-a1fd-dbba742d2fdc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "# Build a fully connected layer.\n",
        "linear = nn.Linear(3, 2)\n",
        "print ('w: ', linear.weight)\n",
        "print ('b: ', linear.bias)"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "w:  Parameter containing:\n",
            "tensor([[ 0.3203, -0.0737,  0.1438],\n",
            "        [-0.0329,  0.5661, -0.1516]], requires_grad=True)\n",
            "b:  Parameter containing:\n",
            "tensor([ 0.4003, -0.0575], requires_grad=True)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IvOuGobUG_c4",
        "outputId": "8e90d0c5-8ad3-4ad2-86e9-ff8cf264f86f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "loss = torch.sum((linear(x)-y)**2)/y.shape[0]\n",
        "print('loss: ', loss.data.numpy())"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "loss:  2.7102962\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-Jg-jZqxIswR"
      },
      "source": [
        "loss.backward()"
      ],
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UIJ9YkofIszl",
        "outputId": "851ff4b5-031b-4455-feaa-cd9009cf30aa",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "print('w grad: ', linear.weight.grad)\n",
        "print('b grad: ', linear.bias.grad)"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "w grad:  tensor([[ 1.7091, -0.0946,  0.4207],\n",
            "        [ 1.2066,  1.3790,  1.4064]])\n",
            "b grad:  tensor([ 1.3754, -0.6619])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vPtT7MCXJJsj",
        "outputId": "ba121788-b5ea-488b-f95d-18e33fdd4eaf",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "# check grad\n",
        "print('w grad:', (linear(x)-y).transpose(0,1).mm(x)/y.shape[0]*2)\n",
        "print('b grad:', 2*torch.mean(linear(x)-y, dim=0))"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "w grad: tensor([[ 1.7091, -0.0946,  0.4207],\n",
            "        [ 1.2066,  1.3790,  1.4064]], grad_fn=<MulBackward0>)\n",
            "b grad: tensor([ 1.3754, -0.6619], grad_fn=<MulBackward0>)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Yy7Cv9ugMN0K"
      },
      "source": [
        "###. Basic autograd example 3"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hon-LU88MN6L"
      },
      "source": [
        "# Create tensors of shape (10, 3) and (10, 2).\n",
        "x = torch.randn(10, 3)\n",
        "y = torch.randn(10, 2)\n",
        "linear = nn.Linear(3, 2)"
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ltmT6QfoFMal",
        "outputId": "78f302ec-f3fe-4e08-ad7d-23012d642539",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "# Build loss function and optimizer.\n",
        "criterion = nn.MSELoss()\n",
        "optimizer = torch.optim.SGD(linear.parameters(), lr=0.01)\n",
        "\n",
        "# Forward pass.\n",
        "pred = linear(x)\n",
        "\n",
        "# Compute loss.\n",
        "loss = criterion(pred, y)\n",
        "print('loss: ', loss.item())\n",
        "\n",
        "# Backward pass.\n",
        "loss.backward()\n",
        "\n",
        "# Print out the gradients.\n",
        "print ('dL/dw: ', linear.weight.grad) \n",
        "print ('dL/db: ', linear.bias.grad)\n",
        "\n",
        "# 1-step gradient descent.\n",
        "optimizer.step()\n",
        "\n",
        "# You can also perform gradient descent at the low level.\n",
        "# linear.weight.data.sub_(0.01 * linear.weight.grad.data)\n",
        "# linear.bias.data.sub_(0.01 * linear.bias.grad.data)\n",
        "\n",
        "# Print out the loss after 1-step gradient descent.\n",
        "pred = linear(x)\n",
        "loss = criterion(pred, y)\n",
        "print('loss after 1 step optimization: ', loss.item())"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "loss:  0.8296538591384888\n",
            "dL/dw:  tensor([[ 0.0308, -0.2376,  0.5363],\n",
            "        [-0.3689, -0.1521,  0.1391]])\n",
            "dL/db:  tensor([0.1237, 0.2327])\n",
            "loss after 1 step optimization:  0.8237482905387878\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WO2btHYRjJ6D"
      },
      "source": [
        "## Input pipeline, Data loader"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YC-xoV6kPmlm",
        "outputId": "2ff6dfe7-5cf0-4c1b-dbb8-40f251f41dde",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119,
          "referenced_widgets": [
            "145d5b1ad8de42639c5e24ead2acefde",
            "0bfc4415442547b087d7ae75b44d7cd5",
            "e4b6601efcd24d44bad1223256c1c09d",
            "c620fc6ef0504c9fabd649e8f7729527",
            "2f7abe16be104d2a95da02bded69dece",
            "567931982f0a488abd21fb97a9578fbc",
            "0dce977dd7424df687dceac745d17c93",
            "d0e53f22b4194cf0bd2766207b5cd0cd"
          ]
        }
      },
      "source": [
        "# Download and construct CIFAR-10 dataset.\n",
        "train_dataset = torchvision.datasets.CIFAR10(root='../../data/',\n",
        "                                             train=True, \n",
        "                                             transform=transforms.ToTensor(),\n",
        "                                             download=True)\n",
        "\n",
        "# Fetch one data pair (read data from disk).\n",
        "image, label = train_dataset[0]\n",
        "print (image.size())\n",
        "print (label)\n",
        "\n",
        "# Data loader (this provides queues and threads in a very simple way).\n",
        "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n",
        "                                           batch_size=64, \n",
        "                                           shuffle=True)\n",
        "\n",
        "# When iteration starts, queue and thread start to load data from files.\n",
        "data_iter = iter(train_loader)\n",
        "\n",
        "# Mini-batch images and labels.\n",
        "images, labels = data_iter.next()\n",
        "\n",
        "# Actual usage of the data loader is as below.\n",
        "for images, labels in train_loader:\n",
        "    # Training code should be written here.\n",
        "    pass"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ../../data/cifar-10-python.tar.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "145d5b1ad8de42639c5e24ead2acefde",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting ../../data/cifar-10-python.tar.gz to ../../data/\n",
            "torch.Size([3, 32, 32])\n",
            "6\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bI16kN0tjnRl"
      },
      "source": [
        "### Input pipeline for custom dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TLfA6_yGjlwF"
      },
      "source": [
        "# # ================================================================== #\n",
        "# #                  Input pipeline for custom dataset                 #\n",
        "# # ================================================================== #\n",
        "\n",
        "# # You should build your custom dataset as below.\n",
        "# class CustomDataset(torch.utils.data.Dataset):\n",
        "#     def __init__(self):\n",
        "#         # TODO\n",
        "#         # 1. Initialize file paths or a list of file names. \n",
        "#         pass\n",
        "#     def __getitem__(self, index):\n",
        "#         # TODO\n",
        "#         # 1. Read one data from file (e.g. using numpy.fromfile, PIL.Image.open).\n",
        "#         # 2. Preprocess the data (e.g. torchvision.Transform).\n",
        "#         # 3. Return a data pair (e.g. image and label).\n",
        "#         pass\n",
        "#     def __len__(self):\n",
        "#         # You should change 0 to the total size of your dataset.\n",
        "#         return 0 \n",
        "\n",
        "# # You can then use the prebuilt data loader. \n",
        "# custom_dataset = CustomDataset()\n",
        "# train_loader = torch.utils.data.DataLoader(dataset=custom_dataset,\n",
        "#                                            batch_size=64, \n",
        "#                                            shuffle=True)\n"
      ],
      "execution_count": 51,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "olrd4PlTjgv7"
      },
      "source": [
        "### Pretrained model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xRsDNLcKjfGz",
        "outputId": "d08d673a-1c33-42dc-cc44-b52ecd6f5808",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121,
          "referenced_widgets": [
            "6f347c8cd2bf477ab1f98b0fef342d3f",
            "d33bc4af435b4d7594e21acf07f7139b",
            "af5dddd10bf94020b67aba3b6445bfd8",
            "9c7facac951e47d5a7b95952ff0ae426",
            "e30087e7c8dd480e8c4c260a55acf600",
            "5d1933afd499484ca73ac1d3a01edcea",
            "39543067be5b44a7a6f69a7cc80c0772",
            "b76255a84ebd485cac680ce7e036d36d"
          ]
        }
      },
      "source": [
        "# ================================================================== #\n",
        "#                           Pretrained model                         #\n",
        "# ================================================================== #\n",
        "\n",
        "# Download and load the pretrained ResNet-18.\n",
        "resnet = torchvision.models.resnet18(pretrained=True)\n",
        "\n",
        "# If you want to finetune only the top layer of the model, set as below.\n",
        "for param in resnet.parameters():\n",
        "    param.requires_grad = False\n",
        "\n",
        "# Replace the top layer for finetuning.\n",
        "resnet.fc = nn.Linear(resnet.fc.in_features, 100)  # 100 is an example.\n",
        "\n",
        "# Forward pass.\n",
        "images = torch.randn(64, 3, 224, 224)\n",
        "outputs = resnet(images)\n",
        "print (outputs.size())     # (64, 100)"
      ],
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading: \"https://download.pytorch.org/models/resnet18-5c106cde.pth\" to /root/.cache/torch/hub/checkpoints/resnet18-5c106cde.pth\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6f347c8cd2bf477ab1f98b0fef342d3f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=46827520.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "torch.Size([64, 100])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9VbbrBlBjY6P"
      },
      "source": [
        "### Save and load the model    "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qfYBD7T_jW6f",
        "outputId": "050345ee-9a94-4ea0-e3f6-7204d7217a7f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "# Save and load the entire model.\n",
        "torch.save(resnet, 'model.ckpt')\n",
        "model = torch.load('model.ckpt')\n",
        "\n",
        "# Save and load only the model parameters (recommended).\n",
        "torch.save(resnet.state_dict(), 'params.ckpt')\n",
        "resnet.load_state_dict(torch.load('params.ckpt'))"
      ],
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<All keys matched successfully>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gx1gFIdtjdcE"
      },
      "source": [
        ""
      ],
      "execution_count": 53,
      "outputs": []
    }
  ]
}