best_model_train.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "6f854391",
   "metadata": {},
   "source": [
    "# Notebook for using model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ebd905fe",
   "metadata": {},
   "source": [
    "Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d546fb0a-54ad-4914-86f6-1bb1708d7634",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.backends.cudnn as cudnn\n",
    "\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "from torchsummary import summary\n",
    "\n",
    "import os\n",
    "import argparse\n",
    "\n",
    "from models.dropoutresnet import DropoutResNet18"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8890fbb7",
   "metadata": {},
   "source": [
    "Training Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b4acb759-15ed-4d98-9993-225d439836ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(epoch):\n",
    "    print('\\nEpoch: %d' % epoch)\n",
    "    # switch to train mode\n",
    "    net.train()\n",
    "    train_loss = 0\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    # start training epoch\n",
    "    for batch_idx, (inputs, targets) in enumerate(trainloader):\n",
    "        # move to device\n",
    "        inputs, targets = inputs.to(device), targets.to(device)\n",
    "        \n",
    "        # forward pass\n",
    "        optimizer.zero_grad()\n",
    "        outputs = net(inputs)\n",
    "        \n",
    "        # backpropogate\n",
    "        loss = criterion(outputs, targets)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        # calculate loss and accuracy\n",
    "        train_loss += loss.item()\n",
    "        _, predicted = outputs.max(1)\n",
    "        total += targets.size(0)\n",
    "        correct += predicted.eq(targets).sum().item()\n",
    "\n",
    "    print('Loss: %.3f | Acc: %.3f%% (%d/%d)' % (train_loss/(batch_idx+1), 100.*correct/total, correct, total))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4cd8c87f",
   "metadata": {},
   "source": [
    "Test Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "342fd9b5-6818-4df1-a51e-bcd64263759d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(epoch):\n",
    "    global best_acc\n",
    "    # switch to eval mode\n",
    "    net.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    with torch.no_grad():\n",
    "        # start test epoch\n",
    "        for batch_idx, (inputs, targets) in enumerate(testloader):\n",
    "            # move to device\n",
    "            inputs, targets = inputs.to(device), targets.to(device)\n",
    "            \n",
    "            # forward pass\n",
    "            outputs = net(inputs)\n",
    "            \n",
    "            # calculate loss and accuracy\n",
    "            loss = criterion(outputs, targets)\n",
    "            test_loss += loss.item()\n",
    "            _, predicted = outputs.max(1)\n",
    "            total += targets.size(0)\n",
    "            correct += predicted.eq(targets).sum().item()\n",
    "\n",
    "        print('Loss: %.3f | Acc: %.3f%% (%d/%d)' % (test_loss/(batch_idx+1), 100.*correct/total, correct, total))\n",
    "        \n",
    "    # Save checkpoint.\n",
    "    acc = 100.*correct/total\n",
    "    \n",
    "    # Save best epoch checkpoint\n",
    "    if acc > best_acc:\n",
    "        print('Saving..')\n",
    "        state = {\n",
    "            'net': net.state_dict(),\n",
    "            'acc': acc,\n",
    "            'epoch': epoch,\n",
    "        }\n",
    "        if not os.path.isdir('checkpoint'):\n",
    "            os.mkdir('checkpoint')\n",
    "        torch.save(state, './checkpoint/final.pth')\n",
    "        best_acc = acc\n",
    "    \n",
    "    # Save model\n",
    "    if epoch == 199:\n",
    "        state = {\n",
    "            'net': net.state_dict(),\n",
    "            'acc': acc,\n",
    "            'epoch': epoch,\n",
    "        }\n",
    "        torch.save(state, './checkpoint/final.pth')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c301c0fa",
   "metadata": {},
   "source": [
    "Loading variables for training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "62a3a412-586d-4630-8264-5c5b2f3148a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "best_acc = 0  # best test accuracy\n",
    "start_epoch = 0  # start from epoch 0 or last checkpoint epoch"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d9bd3043",
   "metadata": {},
   "source": [
    "Data Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f92cabe1-7f40-498c-a64e-6c1dc67a4e20",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==> Preparing data..\n",
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "print('==> Preparing data..')\n",
    "\n",
    "# Data augmentation and normalization for training\n",
    "transform_train = transforms.Compose([\n",
    "    transforms.RandomCrop(32, padding=4),\n",
    "    transforms.RandomHorizontalFlip(),\n",
    "    transforms.ToTensor(),\n",
    "])\n",
    "\n",
    "transform_test = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "])\n",
    "\n",
    "# Create dataloaders\n",
    "trainset = torchvision.datasets.CIFAR10(\n",
    "    root='./deep-learning-mini-project-spring-24-nyu/cifar-10-python', train=True, download=True, transform=transform_train)\n",
    "trainloader = torch.utils.data.DataLoader(\n",
    "    trainset, batch_size=128, shuffle=True, num_workers=2)\n",
    "\n",
    "# Load dataset with tranformation\n",
    "testset = torchvision.datasets.CIFAR10(\n",
    "    root='./deep-learning-mini-project-spring-24-nyu/cifar-10-python', train=False, download=True, transform=transform_test)\n",
    "testloader = torch.utils.data.DataLoader(\n",
    "    testset, batch_size=100, shuffle=False, num_workers=2)\n",
    "\n",
    "# List of classes\n",
    "classes = ('plane', 'car', 'bird', 'cat', 'deer',\n",
    "        'dog', 'frog', 'horse', 'ship', 'truck')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48229d1f",
   "metadata": {},
   "source": [
    "Initializing Model and training tools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f9ffbb39-f3ef-4e50-9f49-f97109d4cdcc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==> Building model..\n",
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "            Conv2d-1           [-1, 64, 32, 32]           1,728\n",
      "       BatchNorm2d-2           [-1, 64, 32, 32]             128\n",
      "            Conv2d-3           [-1, 64, 32, 32]          36,864\n",
      "       BatchNorm2d-4           [-1, 64, 32, 32]             128\n",
      "            Conv2d-5           [-1, 64, 32, 32]          36,864\n",
      "       BatchNorm2d-6           [-1, 64, 32, 32]             128\n",
      "ModifiedBasicBlock-7           [-1, 64, 32, 32]               0\n",
      "            Conv2d-8           [-1, 64, 32, 32]          36,864\n",
      "       BatchNorm2d-9           [-1, 64, 32, 32]             128\n",
      "           Conv2d-10           [-1, 64, 32, 32]          36,864\n",
      "      BatchNorm2d-11           [-1, 64, 32, 32]             128\n",
      "ModifiedBasicBlock-12           [-1, 64, 32, 32]               0\n",
      "           Conv2d-13           [-1, 64, 32, 32]          36,864\n",
      "      BatchNorm2d-14           [-1, 64, 32, 32]             128\n",
      "           Conv2d-15           [-1, 64, 32, 32]          36,864\n",
      "      BatchNorm2d-16           [-1, 64, 32, 32]             128\n",
      "ModifiedBasicBlock-17           [-1, 64, 32, 32]               0\n",
      "           Conv2d-18           [-1, 64, 32, 32]          36,864\n",
      "      BatchNorm2d-19           [-1, 64, 32, 32]             128\n",
      "           Conv2d-20           [-1, 64, 32, 32]          36,864\n",
      "      BatchNorm2d-21           [-1, 64, 32, 32]             128\n",
      "ModifiedBasicBlock-22           [-1, 64, 32, 32]               0\n",
      "           Conv2d-23          [-1, 128, 16, 16]          73,728\n",
      "      BatchNorm2d-24          [-1, 128, 16, 16]             256\n",
      "           Conv2d-25          [-1, 128, 16, 16]         147,456\n",
      "      BatchNorm2d-26          [-1, 128, 16, 16]             256\n",
      "           Conv2d-27          [-1, 128, 16, 16]           8,192\n",
      "      BatchNorm2d-28          [-1, 128, 16, 16]             256\n",
      "ModifiedBasicBlock-29          [-1, 128, 16, 16]               0\n",
      "           Conv2d-30          [-1, 128, 16, 16]         147,456\n",
      "      BatchNorm2d-31          [-1, 128, 16, 16]             256\n",
      "           Conv2d-32          [-1, 128, 16, 16]         147,456\n",
      "      BatchNorm2d-33          [-1, 128, 16, 16]             256\n",
      "ModifiedBasicBlock-34          [-1, 128, 16, 16]               0\n",
      "           Conv2d-35          [-1, 128, 16, 16]         147,456\n",
      "      BatchNorm2d-36          [-1, 128, 16, 16]             256\n",
      "           Conv2d-37          [-1, 128, 16, 16]         147,456\n",
      "      BatchNorm2d-38          [-1, 128, 16, 16]             256\n",
      "ModifiedBasicBlock-39          [-1, 128, 16, 16]               0\n",
      "           Conv2d-40          [-1, 128, 16, 16]         147,456\n",
      "      BatchNorm2d-41          [-1, 128, 16, 16]             256\n",
      "           Conv2d-42          [-1, 128, 16, 16]         147,456\n",
      "      BatchNorm2d-43          [-1, 128, 16, 16]             256\n",
      "ModifiedBasicBlock-44          [-1, 128, 16, 16]               0\n",
      "           Conv2d-45            [-1, 256, 8, 8]         294,912\n",
      "      BatchNorm2d-46            [-1, 256, 8, 8]             512\n",
      "           Conv2d-47            [-1, 256, 8, 8]         589,824\n",
      "      BatchNorm2d-48            [-1, 256, 8, 8]             512\n",
      "           Conv2d-49            [-1, 256, 8, 8]          32,768\n",
      "      BatchNorm2d-50            [-1, 256, 8, 8]             512\n",
      "ModifiedBasicBlock-51            [-1, 256, 8, 8]               0\n",
      "           Conv2d-52            [-1, 256, 8, 8]         589,824\n",
      "      BatchNorm2d-53            [-1, 256, 8, 8]             512\n",
      "           Conv2d-54            [-1, 256, 8, 8]         589,824\n",
      "      BatchNorm2d-55            [-1, 256, 8, 8]             512\n",
      "ModifiedBasicBlock-56            [-1, 256, 8, 8]               0\n",
      "           Conv2d-57            [-1, 256, 8, 8]         589,824\n",
      "      BatchNorm2d-58            [-1, 256, 8, 8]             512\n",
      "           Conv2d-59            [-1, 256, 8, 8]         589,824\n",
      "      BatchNorm2d-60            [-1, 256, 8, 8]             512\n",
      "ModifiedBasicBlock-61            [-1, 256, 8, 8]               0\n",
      "AdaptiveAvgPool2d-62            [-1, 256, 1, 1]               0\n",
      "          Dropout-63                  [-1, 256]               0\n",
      "           Linear-64                   [-1, 10]           2,570\n",
      "    DropoutResNet-65                   [-1, 10]               0\n",
      "================================================================\n",
      "Total params: 4,697,162\n",
      "Trainable params: 4,697,162\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.01\n",
      "Forward/backward pass size (MB): 18.63\n",
      "Params size (MB): 17.92\n",
      "Estimated Total Size (MB): 36.56\n",
      "----------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "print('==> Building model..')\n",
    "# Initialize the model\n",
    "net = DropoutResNet18(0.4)\n",
    "net = net.to(device)\n",
    "\n",
    "# If the model is on cuda, use DataParallel\n",
    "if device == 'cuda':\n",
    "    net = torch.nn.DataParallel(net)\n",
    "    cudnn.benchmark = True\n",
    "\n",
    "# Define the loss function and optimizer\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(net.parameters(), lr=0.1, momentum=0.9, weight_decay=5e-4)\n",
    "scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=200)\n",
    "\n",
    "# Print out the model parameters\n",
    "summary(net, (3,32,32))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "db636fde",
   "metadata": {},
   "source": [
    "Running Training and Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "dde0dd12-3655-4fe4-b403-c80529afcca6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch: 0\n",
      "Loss: 2.029 | Acc: 24.178% (12089/50000)\n",
      "Loss: 2.346 | Acc: 22.850% (2285/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 1\n",
      "Loss: 1.587 | Acc: 40.508% (20254/50000)\n",
      "Loss: 1.422 | Acc: 46.580% (4658/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 2\n",
      "Loss: 1.353 | Acc: 50.404% (25202/50000)\n",
      "Loss: 1.477 | Acc: 48.010% (4801/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 3\n",
      "Loss: 1.173 | Acc: 57.910% (28955/50000)\n",
      "Loss: 1.882 | Acc: 45.350% (4535/10000)\n",
      "\n",
      "Epoch: 4\n",
      "Loss: 1.005 | Acc: 64.542% (32271/50000)\n",
      "Loss: 1.056 | Acc: 64.190% (6419/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 5\n",
      "Loss: 0.872 | Acc: 69.550% (34775/50000)\n",
      "Loss: 0.871 | Acc: 69.950% (6995/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 6\n",
      "Loss: 0.761 | Acc: 73.698% (36849/50000)\n",
      "Loss: 0.825 | Acc: 72.490% (7249/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 7\n",
      "Loss: 0.675 | Acc: 76.862% (38431/50000)\n",
      "Loss: 0.977 | Acc: 67.830% (6783/10000)\n",
      "\n",
      "Epoch: 8\n",
      "Loss: 0.619 | Acc: 78.928% (39464/50000)\n",
      "Loss: 0.961 | Acc: 70.840% (7084/10000)\n",
      "\n",
      "Epoch: 9\n",
      "Loss: 0.566 | Acc: 80.848% (40424/50000)\n",
      "Loss: 0.698 | Acc: 76.480% (7648/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 10\n",
      "Loss: 0.538 | Acc: 81.818% (40909/50000)\n",
      "Loss: 1.351 | Acc: 65.920% (6592/10000)\n",
      "\n",
      "Epoch: 11\n",
      "Loss: 0.511 | Acc: 82.608% (41304/50000)\n",
      "Loss: 0.579 | Acc: 81.010% (8101/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 12\n",
      "Loss: 0.486 | Acc: 83.670% (41835/50000)\n",
      "Loss: 0.677 | Acc: 77.580% (7758/10000)\n",
      "\n",
      "Epoch: 13\n",
      "Loss: 0.478 | Acc: 83.984% (41992/50000)\n",
      "Loss: 0.716 | Acc: 76.560% (7656/10000)\n",
      "\n",
      "Epoch: 14\n",
      "Loss: 0.456 | Acc: 84.464% (42232/50000)\n",
      "Loss: 0.590 | Acc: 80.300% (8030/10000)\n",
      "\n",
      "Epoch: 15\n",
      "Loss: 0.435 | Acc: 85.256% (42628/50000)\n",
      "Loss: 1.757 | Acc: 59.650% (5965/10000)\n",
      "\n",
      "Epoch: 16\n",
      "Loss: 0.438 | Acc: 85.056% (42528/50000)\n",
      "Loss: 0.583 | Acc: 80.500% (8050/10000)\n",
      "\n",
      "Epoch: 17\n",
      "Loss: 0.425 | Acc: 85.618% (42809/50000)\n",
      "Loss: 0.548 | Acc: 81.700% (8170/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 18\n",
      "Loss: 0.415 | Acc: 85.922% (42961/50000)\n",
      "Loss: 0.669 | Acc: 78.360% (7836/10000)\n",
      "\n",
      "Epoch: 19\n",
      "Loss: 0.404 | Acc: 86.490% (43245/50000)\n",
      "Loss: 0.514 | Acc: 82.970% (8297/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 20\n",
      "Loss: 0.394 | Acc: 86.674% (43337/50000)\n",
      "Loss: 0.709 | Acc: 77.130% (7713/10000)\n",
      "\n",
      "Epoch: 21\n",
      "Loss: 0.393 | Acc: 86.826% (43413/50000)\n",
      "Loss: 0.577 | Acc: 81.000% (8100/10000)\n",
      "\n",
      "Epoch: 22\n",
      "Loss: 0.385 | Acc: 87.110% (43555/50000)\n",
      "Loss: 0.630 | Acc: 80.850% (8085/10000)\n",
      "\n",
      "Epoch: 23\n",
      "Loss: 0.380 | Acc: 87.274% (43637/50000)\n",
      "Loss: 0.557 | Acc: 82.110% (8211/10000)\n",
      "\n",
      "Epoch: 24\n",
      "Loss: 0.382 | Acc: 87.128% (43564/50000)\n",
      "Loss: 0.513 | Acc: 82.520% (8252/10000)\n",
      "\n",
      "Epoch: 25\n",
      "Loss: 0.364 | Acc: 87.598% (43799/50000)\n",
      "Loss: 0.649 | Acc: 79.090% (7909/10000)\n",
      "\n",
      "Epoch: 26\n",
      "Loss: 0.369 | Acc: 87.788% (43894/50000)\n",
      "Loss: 0.577 | Acc: 80.470% (8047/10000)\n",
      "\n",
      "Epoch: 27\n",
      "Loss: 0.355 | Acc: 88.026% (44013/50000)\n",
      "Loss: 0.529 | Acc: 82.940% (8294/10000)\n",
      "\n",
      "Epoch: 28\n",
      "Loss: 0.359 | Acc: 87.900% (43950/50000)\n",
      "Loss: 1.030 | Acc: 68.420% (6842/10000)\n",
      "\n",
      "Epoch: 29\n",
      "Loss: 0.353 | Acc: 88.010% (44005/50000)\n",
      "Loss: 0.647 | Acc: 79.180% (7918/10000)\n",
      "\n",
      "Epoch: 30\n",
      "Loss: 0.352 | Acc: 88.032% (44016/50000)\n",
      "Loss: 0.443 | Acc: 85.030% (8503/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 31\n",
      "Loss: 0.338 | Acc: 88.512% (44256/50000)\n",
      "Loss: 0.663 | Acc: 80.230% (8023/10000)\n",
      "\n",
      "Epoch: 32\n",
      "Loss: 0.339 | Acc: 88.508% (44254/50000)\n",
      "Loss: 0.682 | Acc: 78.670% (7867/10000)\n",
      "\n",
      "Epoch: 33\n",
      "Loss: 0.338 | Acc: 88.612% (44306/50000)\n",
      "Loss: 0.559 | Acc: 81.810% (8181/10000)\n",
      "\n",
      "Epoch: 34\n",
      "Loss: 0.338 | Acc: 88.540% (44270/50000)\n",
      "Loss: 0.653 | Acc: 79.720% (7972/10000)\n",
      "\n",
      "Epoch: 35\n",
      "Loss: 0.334 | Acc: 88.630% (44315/50000)\n",
      "Loss: 0.737 | Acc: 76.500% (7650/10000)\n",
      "\n",
      "Epoch: 36\n",
      "Loss: 0.333 | Acc: 88.770% (44385/50000)\n",
      "Loss: 0.606 | Acc: 81.130% (8113/10000)\n",
      "\n",
      "Epoch: 37\n",
      "Loss: 0.318 | Acc: 89.368% (44684/50000)\n",
      "Loss: 0.410 | Acc: 86.530% (8653/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 38\n",
      "Loss: 0.322 | Acc: 89.258% (44629/50000)\n",
      "Loss: 0.705 | Acc: 79.210% (7921/10000)\n",
      "\n",
      "Epoch: 39\n",
      "Loss: 0.327 | Acc: 89.054% (44527/50000)\n",
      "Loss: 0.547 | Acc: 82.960% (8296/10000)\n",
      "\n",
      "Epoch: 40\n",
      "Loss: 0.323 | Acc: 89.194% (44597/50000)\n",
      "Loss: 0.476 | Acc: 84.310% (8431/10000)\n",
      "\n",
      "Epoch: 41\n",
      "Loss: 0.319 | Acc: 89.202% (44601/50000)\n",
      "Loss: 0.451 | Acc: 84.960% (8496/10000)\n",
      "\n",
      "Epoch: 42\n",
      "Loss: 0.314 | Acc: 89.366% (44683/50000)\n",
      "Loss: 0.570 | Acc: 82.550% (8255/10000)\n",
      "\n",
      "Epoch: 43\n",
      "Loss: 0.313 | Acc: 89.412% (44706/50000)\n",
      "Loss: 0.475 | Acc: 84.530% (8453/10000)\n",
      "\n",
      "Epoch: 44\n",
      "Loss: 0.311 | Acc: 89.458% (44729/50000)\n",
      "Loss: 0.409 | Acc: 85.860% (8586/10000)\n",
      "\n",
      "Epoch: 45\n",
      "Loss: 0.311 | Acc: 89.354% (44677/50000)\n",
      "Loss: 0.454 | Acc: 85.190% (8519/10000)\n",
      "\n",
      "Epoch: 46\n",
      "Loss: 0.302 | Acc: 89.732% (44866/50000)\n",
      "Loss: 0.536 | Acc: 82.890% (8289/10000)\n",
      "\n",
      "Epoch: 47\n",
      "Loss: 0.307 | Acc: 89.692% (44846/50000)\n",
      "Loss: 0.560 | Acc: 82.240% (8224/10000)\n",
      "\n",
      "Epoch: 48\n",
      "Loss: 0.309 | Acc: 89.730% (44865/50000)\n",
      "Loss: 0.481 | Acc: 84.230% (8423/10000)\n",
      "\n",
      "Epoch: 49\n",
      "Loss: 0.291 | Acc: 90.294% (45147/50000)\n",
      "Loss: 0.710 | Acc: 77.850% (7785/10000)\n",
      "\n",
      "Epoch: 50\n",
      "Loss: 0.301 | Acc: 90.034% (45017/50000)\n",
      "Loss: 0.388 | Acc: 86.910% (8691/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 51\n",
      "Loss: 0.295 | Acc: 90.044% (45022/50000)\n",
      "Loss: 0.739 | Acc: 77.720% (7772/10000)\n",
      "\n",
      "Epoch: 52\n",
      "Loss: 0.294 | Acc: 90.142% (45071/50000)\n",
      "Loss: 0.402 | Acc: 86.700% (8670/10000)\n",
      "\n",
      "Epoch: 53\n",
      "Loss: 0.285 | Acc: 90.524% (45262/50000)\n",
      "Loss: 0.602 | Acc: 81.370% (8137/10000)\n",
      "\n",
      "Epoch: 54\n",
      "Loss: 0.288 | Acc: 90.336% (45168/50000)\n",
      "Loss: 0.903 | Acc: 74.250% (7425/10000)\n",
      "\n",
      "Epoch: 55\n",
      "Loss: 0.292 | Acc: 90.142% (45071/50000)\n",
      "Loss: 0.775 | Acc: 78.180% (7818/10000)\n",
      "\n",
      "Epoch: 56\n",
      "Loss: 0.284 | Acc: 90.534% (45267/50000)\n",
      "Loss: 0.439 | Acc: 85.720% (8572/10000)\n",
      "\n",
      "Epoch: 57\n",
      "Loss: 0.286 | Acc: 90.362% (45181/50000)\n",
      "Loss: 0.579 | Acc: 81.780% (8178/10000)\n",
      "\n",
      "Epoch: 58\n",
      "Loss: 0.280 | Acc: 90.550% (45275/50000)\n",
      "Loss: 0.546 | Acc: 83.430% (8343/10000)\n",
      "\n",
      "Epoch: 59\n",
      "Loss: 0.283 | Acc: 90.382% (45191/50000)\n",
      "Loss: 0.433 | Acc: 85.800% (8580/10000)\n",
      "\n",
      "Epoch: 60\n",
      "Loss: 0.276 | Acc: 90.692% (45346/50000)\n",
      "Loss: 0.491 | Acc: 84.020% (8402/10000)\n",
      "\n",
      "Epoch: 61\n",
      "Loss: 0.276 | Acc: 90.748% (45374/50000)\n",
      "Loss: 0.467 | Acc: 85.190% (8519/10000)\n",
      "\n",
      "Epoch: 62\n",
      "Loss: 0.272 | Acc: 90.896% (45448/50000)\n",
      "Loss: 0.594 | Acc: 82.370% (8237/10000)\n",
      "\n",
      "Epoch: 63\n",
      "Loss: 0.275 | Acc: 90.752% (45376/50000)\n",
      "Loss: 0.428 | Acc: 85.860% (8586/10000)\n",
      "\n",
      "Epoch: 64\n",
      "Loss: 0.264 | Acc: 91.154% (45577/50000)\n",
      "Loss: 0.416 | Acc: 87.050% (8705/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 65\n",
      "Loss: 0.270 | Acc: 90.944% (45472/50000)\n",
      "Loss: 0.375 | Acc: 87.330% (8733/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 66\n",
      "Loss: 0.264 | Acc: 91.094% (45547/50000)\n",
      "Loss: 0.500 | Acc: 84.160% (8416/10000)\n",
      "\n",
      "Epoch: 67\n",
      "Loss: 0.258 | Acc: 91.374% (45687/50000)\n",
      "Loss: 0.585 | Acc: 81.600% (8160/10000)\n",
      "\n",
      "Epoch: 68\n",
      "Loss: 0.255 | Acc: 91.398% (45699/50000)\n",
      "Loss: 0.487 | Acc: 84.070% (8407/10000)\n",
      "\n",
      "Epoch: 69\n",
      "Loss: 0.261 | Acc: 91.186% (45593/50000)\n",
      "Loss: 0.453 | Acc: 85.010% (8501/10000)\n",
      "\n",
      "Epoch: 70\n",
      "Loss: 0.257 | Acc: 91.260% (45630/50000)\n",
      "Loss: 0.401 | Acc: 86.820% (8682/10000)\n",
      "\n",
      "Epoch: 71\n",
      "Loss: 0.253 | Acc: 91.532% (45766/50000)\n",
      "Loss: 0.470 | Acc: 85.360% (8536/10000)\n",
      "\n",
      "Epoch: 72\n",
      "Loss: 0.253 | Acc: 91.446% (45723/50000)\n",
      "Loss: 0.532 | Acc: 83.230% (8323/10000)\n",
      "\n",
      "Epoch: 73\n",
      "Loss: 0.246 | Acc: 91.802% (45901/50000)\n",
      "Loss: 0.497 | Acc: 84.430% (8443/10000)\n",
      "\n",
      "Epoch: 74\n",
      "Loss: 0.246 | Acc: 91.782% (45891/50000)\n",
      "Loss: 0.455 | Acc: 85.600% (8560/10000)\n",
      "\n",
      "Epoch: 75\n",
      "Loss: 0.248 | Acc: 91.892% (45946/50000)\n",
      "Loss: 0.438 | Acc: 85.350% (8535/10000)\n",
      "\n",
      "Epoch: 76\n",
      "Loss: 0.244 | Acc: 91.802% (45901/50000)\n",
      "Loss: 0.397 | Acc: 87.500% (8750/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 77\n",
      "Loss: 0.242 | Acc: 91.850% (45925/50000)\n",
      "Loss: 0.345 | Acc: 88.520% (8852/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 78\n",
      "Loss: 0.234 | Acc: 92.062% (46031/50000)\n",
      "Loss: 0.513 | Acc: 83.300% (8330/10000)\n",
      "\n",
      "Epoch: 79\n",
      "Loss: 0.238 | Acc: 91.906% (45953/50000)\n",
      "Loss: 0.483 | Acc: 84.110% (8411/10000)\n",
      "\n",
      "Epoch: 80\n",
      "Loss: 0.234 | Acc: 91.998% (45999/50000)\n",
      "Loss: 0.444 | Acc: 85.960% (8596/10000)\n",
      "\n",
      "Epoch: 81\n",
      "Loss: 0.227 | Acc: 92.328% (46164/50000)\n",
      "Loss: 0.405 | Acc: 87.020% (8702/10000)\n",
      "\n",
      "Epoch: 82\n",
      "Loss: 0.230 | Acc: 92.234% (46117/50000)\n",
      "Loss: 0.395 | Acc: 87.480% (8748/10000)\n",
      "\n",
      "Epoch: 83\n",
      "Loss: 0.222 | Acc: 92.554% (46277/50000)\n",
      "Loss: 0.415 | Acc: 86.950% (8695/10000)\n",
      "\n",
      "Epoch: 84\n",
      "Loss: 0.224 | Acc: 92.466% (46233/50000)\n",
      "Loss: 0.472 | Acc: 85.450% (8545/10000)\n",
      "\n",
      "Epoch: 85\n",
      "Loss: 0.222 | Acc: 92.566% (46283/50000)\n",
      "Loss: 0.433 | Acc: 86.400% (8640/10000)\n",
      "\n",
      "Epoch: 86\n",
      "Loss: 0.222 | Acc: 92.618% (46309/50000)\n",
      "Loss: 0.383 | Acc: 87.350% (8735/10000)\n",
      "\n",
      "Epoch: 87\n",
      "Loss: 0.213 | Acc: 92.866% (46433/50000)\n",
      "Loss: 0.372 | Acc: 87.860% (8786/10000)\n",
      "\n",
      "Epoch: 88\n",
      "Loss: 0.216 | Acc: 92.700% (46350/50000)\n",
      "Loss: 0.397 | Acc: 87.750% (8775/10000)\n",
      "\n",
      "Epoch: 89\n",
      "Loss: 0.209 | Acc: 93.010% (46505/50000)\n",
      "Loss: 0.342 | Acc: 88.540% (8854/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 90\n",
      "Loss: 0.210 | Acc: 92.976% (46488/50000)\n",
      "Loss: 0.699 | Acc: 80.030% (8003/10000)\n",
      "\n",
      "Epoch: 91\n",
      "Loss: 0.205 | Acc: 93.050% (46525/50000)\n",
      "Loss: 0.421 | Acc: 87.520% (8752/10000)\n",
      "\n",
      "Epoch: 92\n",
      "Loss: 0.211 | Acc: 92.920% (46460/50000)\n",
      "Loss: 0.403 | Acc: 87.100% (8710/10000)\n",
      "\n",
      "Epoch: 93\n",
      "Loss: 0.198 | Acc: 93.352% (46676/50000)\n",
      "Loss: 0.389 | Acc: 87.890% (8789/10000)\n",
      "\n",
      "Epoch: 94\n",
      "Loss: 0.200 | Acc: 93.306% (46653/50000)\n",
      "Loss: 0.319 | Acc: 89.650% (8965/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 95\n",
      "Loss: 0.199 | Acc: 93.160% (46580/50000)\n",
      "Loss: 0.435 | Acc: 86.430% (8643/10000)\n",
      "\n",
      "Epoch: 96\n",
      "Loss: 0.190 | Acc: 93.626% (46813/50000)\n",
      "Loss: 0.400 | Acc: 87.930% (8793/10000)\n",
      "\n",
      "Epoch: 97\n",
      "Loss: 0.194 | Acc: 93.580% (46790/50000)\n",
      "Loss: 0.307 | Acc: 90.300% (9030/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 98\n",
      "Loss: 0.186 | Acc: 93.762% (46881/50000)\n",
      "Loss: 0.323 | Acc: 90.090% (9009/10000)\n",
      "\n",
      "Epoch: 99\n",
      "Loss: 0.183 | Acc: 94.000% (47000/50000)\n",
      "Loss: 0.404 | Acc: 87.030% (8703/10000)\n",
      "\n",
      "Epoch: 100\n",
      "Loss: 0.179 | Acc: 93.896% (46948/50000)\n",
      "Loss: 0.352 | Acc: 88.750% (8875/10000)\n",
      "\n",
      "Epoch: 101\n",
      "Loss: 0.182 | Acc: 93.908% (46954/50000)\n",
      "Loss: 0.412 | Acc: 87.460% (8746/10000)\n",
      "\n",
      "Epoch: 102\n",
      "Loss: 0.180 | Acc: 93.880% (46940/50000)\n",
      "Loss: 0.325 | Acc: 89.750% (8975/10000)\n",
      "\n",
      "Epoch: 103\n",
      "Loss: 0.173 | Acc: 94.214% (47107/50000)\n",
      "Loss: 0.328 | Acc: 89.630% (8963/10000)\n",
      "\n",
      "Epoch: 104\n",
      "Loss: 0.173 | Acc: 94.010% (47005/50000)\n",
      "Loss: 0.330 | Acc: 89.730% (8973/10000)\n",
      "\n",
      "Epoch: 105\n",
      "Loss: 0.167 | Acc: 94.472% (47236/50000)\n",
      "Loss: 0.435 | Acc: 86.040% (8604/10000)\n",
      "\n",
      "Epoch: 106\n",
      "Loss: 0.170 | Acc: 94.350% (47175/50000)\n",
      "Loss: 0.374 | Acc: 88.840% (8884/10000)\n",
      "\n",
      "Epoch: 107\n",
      "Loss: 0.159 | Acc: 94.670% (47335/50000)\n",
      "Loss: 0.400 | Acc: 87.630% (8763/10000)\n",
      "\n",
      "Epoch: 108\n",
      "Loss: 0.160 | Acc: 94.666% (47333/50000)\n",
      "Loss: 0.357 | Acc: 88.800% (8880/10000)\n",
      "\n",
      "Epoch: 109\n",
      "Loss: 0.159 | Acc: 94.692% (47346/50000)\n",
      "Loss: 0.402 | Acc: 87.730% (8773/10000)\n",
      "\n",
      "Epoch: 110\n",
      "Loss: 0.158 | Acc: 94.706% (47353/50000)\n",
      "Loss: 0.329 | Acc: 90.140% (9014/10000)\n",
      "\n",
      "Epoch: 111\n",
      "Loss: 0.150 | Acc: 94.886% (47443/50000)\n",
      "Loss: 0.381 | Acc: 88.520% (8852/10000)\n",
      "\n",
      "Epoch: 112\n",
      "Loss: 0.152 | Acc: 94.958% (47479/50000)\n",
      "Loss: 0.342 | Acc: 89.280% (8928/10000)\n",
      "\n",
      "Epoch: 113\n",
      "Loss: 0.151 | Acc: 94.850% (47425/50000)\n",
      "Loss: 0.391 | Acc: 88.010% (8801/10000)\n",
      "\n",
      "Epoch: 114\n",
      "Loss: 0.146 | Acc: 95.234% (47617/50000)\n",
      "Loss: 0.344 | Acc: 89.620% (8962/10000)\n",
      "\n",
      "Epoch: 115\n",
      "Loss: 0.138 | Acc: 95.448% (47724/50000)\n",
      "Loss: 0.318 | Acc: 90.050% (9005/10000)\n",
      "\n",
      "Epoch: 116\n",
      "Loss: 0.139 | Acc: 95.428% (47714/50000)\n",
      "Loss: 0.436 | Acc: 87.110% (8711/10000)\n",
      "\n",
      "Epoch: 117\n",
      "Loss: 0.132 | Acc: 95.694% (47847/50000)\n",
      "Loss: 0.337 | Acc: 89.340% (8934/10000)\n",
      "\n",
      "Epoch: 118\n",
      "Loss: 0.132 | Acc: 95.584% (47792/50000)\n",
      "Loss: 0.362 | Acc: 88.760% (8876/10000)\n",
      "\n",
      "Epoch: 119\n",
      "Loss: 0.126 | Acc: 95.792% (47896/50000)\n",
      "Loss: 0.446 | Acc: 87.410% (8741/10000)\n",
      "\n",
      "Epoch: 120\n",
      "Loss: 0.129 | Acc: 95.632% (47816/50000)\n",
      "Loss: 0.288 | Acc: 91.050% (9105/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 121\n",
      "Loss: 0.132 | Acc: 95.638% (47819/50000)\n",
      "Loss: 0.279 | Acc: 91.080% (9108/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 122\n",
      "Loss: 0.112 | Acc: 96.334% (48167/50000)\n",
      "Loss: 0.401 | Acc: 88.210% (8821/10000)\n",
      "\n",
      "Epoch: 123\n",
      "Loss: 0.121 | Acc: 95.948% (47974/50000)\n",
      "Loss: 0.360 | Acc: 89.660% (8966/10000)\n",
      "\n",
      "Epoch: 124\n",
      "Loss: 0.113 | Acc: 96.298% (48149/50000)\n",
      "Loss: 0.379 | Acc: 89.160% (8916/10000)\n",
      "\n",
      "Epoch: 125\n",
      "Loss: 0.110 | Acc: 96.322% (48161/50000)\n",
      "Loss: 0.324 | Acc: 90.520% (9052/10000)\n",
      "\n",
      "Epoch: 126\n",
      "Loss: 0.111 | Acc: 96.184% (48092/50000)\n",
      "Loss: 0.388 | Acc: 89.250% (8925/10000)\n",
      "\n",
      "Epoch: 127\n",
      "Loss: 0.107 | Acc: 96.482% (48241/50000)\n",
      "Loss: 0.317 | Acc: 90.230% (9023/10000)\n",
      "\n",
      "Epoch: 128\n",
      "Loss: 0.099 | Acc: 96.818% (48409/50000)\n",
      "Loss: 0.345 | Acc: 90.250% (9025/10000)\n",
      "\n",
      "Epoch: 129\n",
      "Loss: 0.098 | Acc: 96.718% (48359/50000)\n",
      "Loss: 0.269 | Acc: 92.000% (9200/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 130\n",
      "Loss: 0.095 | Acc: 96.882% (48441/50000)\n",
      "Loss: 0.305 | Acc: 90.780% (9078/10000)\n",
      "\n",
      "Epoch: 131\n",
      "Loss: 0.094 | Acc: 96.934% (48467/50000)\n",
      "Loss: 0.357 | Acc: 90.020% (9002/10000)\n",
      "\n",
      "Epoch: 132\n",
      "Loss: 0.091 | Acc: 96.984% (48492/50000)\n",
      "Loss: 0.283 | Acc: 91.460% (9146/10000)\n",
      "\n",
      "Epoch: 133\n",
      "Loss: 0.085 | Acc: 97.172% (48586/50000)\n",
      "Loss: 0.327 | Acc: 90.660% (9066/10000)\n",
      "\n",
      "Epoch: 134\n",
      "Loss: 0.084 | Acc: 97.178% (48589/50000)\n",
      "Loss: 0.378 | Acc: 89.570% (8957/10000)\n",
      "\n",
      "Epoch: 135\n",
      "Loss: 0.082 | Acc: 97.366% (48683/50000)\n",
      "Loss: 0.315 | Acc: 90.940% (9094/10000)\n",
      "\n",
      "Epoch: 136\n",
      "Loss: 0.081 | Acc: 97.386% (48693/50000)\n",
      "Loss: 0.333 | Acc: 90.700% (9070/10000)\n",
      "\n",
      "Epoch: 137\n",
      "Loss: 0.077 | Acc: 97.458% (48729/50000)\n",
      "Loss: 0.324 | Acc: 90.770% (9077/10000)\n",
      "\n",
      "Epoch: 138\n",
      "Loss: 0.074 | Acc: 97.530% (48765/50000)\n",
      "Loss: 0.286 | Acc: 91.630% (9163/10000)\n",
      "\n",
      "Epoch: 139\n",
      "Loss: 0.068 | Acc: 97.868% (48934/50000)\n",
      "Loss: 0.287 | Acc: 91.630% (9163/10000)\n",
      "\n",
      "Epoch: 140\n",
      "Loss: 0.070 | Acc: 97.714% (48857/50000)\n",
      "Loss: 0.292 | Acc: 91.730% (9173/10000)\n",
      "\n",
      "Epoch: 141\n",
      "Loss: 0.066 | Acc: 97.830% (48915/50000)\n",
      "Loss: 0.261 | Acc: 92.620% (9262/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 142\n",
      "Loss: 0.056 | Acc: 98.206% (49103/50000)\n",
      "Loss: 0.246 | Acc: 92.930% (9293/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 143\n",
      "Loss: 0.057 | Acc: 98.174% (49087/50000)\n",
      "Loss: 0.271 | Acc: 92.070% (9207/10000)\n",
      "\n",
      "Epoch: 144\n",
      "Loss: 0.057 | Acc: 98.204% (49102/50000)\n",
      "Loss: 0.262 | Acc: 92.760% (9276/10000)\n",
      "\n",
      "Epoch: 145\n",
      "Loss: 0.052 | Acc: 98.364% (49182/50000)\n",
      "Loss: 0.281 | Acc: 92.140% (9214/10000)\n",
      "\n",
      "Epoch: 146\n",
      "Loss: 0.051 | Acc: 98.350% (49175/50000)\n",
      "Loss: 0.266 | Acc: 92.500% (9250/10000)\n",
      "\n",
      "Epoch: 147\n",
      "Loss: 0.048 | Acc: 98.498% (49249/50000)\n",
      "Loss: 0.262 | Acc: 92.840% (9284/10000)\n",
      "\n",
      "Epoch: 148\n",
      "Loss: 0.043 | Acc: 98.658% (49329/50000)\n",
      "Loss: 0.320 | Acc: 91.420% (9142/10000)\n",
      "\n",
      "Epoch: 149\n",
      "Loss: 0.046 | Acc: 98.548% (49274/50000)\n",
      "Loss: 0.267 | Acc: 92.760% (9276/10000)\n",
      "\n",
      "Epoch: 150\n",
      "Loss: 0.042 | Acc: 98.706% (49353/50000)\n",
      "Loss: 0.290 | Acc: 91.960% (9196/10000)\n",
      "\n",
      "Epoch: 151\n",
      "Loss: 0.036 | Acc: 98.906% (49453/50000)\n",
      "Loss: 0.319 | Acc: 91.840% (9184/10000)\n",
      "\n",
      "Epoch: 152\n",
      "Loss: 0.033 | Acc: 99.020% (49510/50000)\n",
      "Loss: 0.264 | Acc: 92.780% (9278/10000)\n",
      "\n",
      "Epoch: 153\n",
      "Loss: 0.028 | Acc: 99.178% (49589/50000)\n",
      "Loss: 0.269 | Acc: 92.990% (9299/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 154\n",
      "Loss: 0.027 | Acc: 99.236% (49618/50000)\n",
      "Loss: 0.251 | Acc: 93.280% (9328/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 155\n",
      "Loss: 0.027 | Acc: 99.208% (49604/50000)\n",
      "Loss: 0.267 | Acc: 92.950% (9295/10000)\n",
      "\n",
      "Epoch: 156\n",
      "Loss: 0.022 | Acc: 99.378% (49689/50000)\n",
      "Loss: 0.236 | Acc: 93.270% (9327/10000)\n",
      "\n",
      "Epoch: 157\n",
      "Loss: 0.021 | Acc: 99.436% (49718/50000)\n",
      "Loss: 0.226 | Acc: 94.120% (9412/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 158\n",
      "Loss: 0.018 | Acc: 99.512% (49756/50000)\n",
      "Loss: 0.257 | Acc: 93.340% (9334/10000)\n",
      "\n",
      "Epoch: 159\n",
      "Loss: 0.017 | Acc: 99.576% (49788/50000)\n",
      "Loss: 0.226 | Acc: 94.120% (9412/10000)\n",
      "\n",
      "Epoch: 160\n",
      "Loss: 0.015 | Acc: 99.634% (49817/50000)\n",
      "Loss: 0.218 | Acc: 94.380% (9438/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 161\n",
      "Loss: 0.012 | Acc: 99.706% (49853/50000)\n",
      "Loss: 0.221 | Acc: 94.310% (9431/10000)\n",
      "\n",
      "Epoch: 162\n",
      "Loss: 0.010 | Acc: 99.768% (49884/50000)\n",
      "Loss: 0.218 | Acc: 94.480% (9448/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 163\n",
      "Loss: 0.009 | Acc: 99.832% (49916/50000)\n",
      "Loss: 0.202 | Acc: 94.810% (9481/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 164\n",
      "Loss: 0.006 | Acc: 99.902% (49951/50000)\n",
      "Loss: 0.202 | Acc: 94.850% (9485/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 165\n",
      "Loss: 0.006 | Acc: 99.914% (49957/50000)\n",
      "Loss: 0.192 | Acc: 95.140% (9514/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 166\n",
      "Loss: 0.006 | Acc: 99.900% (49950/50000)\n",
      "Loss: 0.200 | Acc: 94.790% (9479/10000)\n",
      "\n",
      "Epoch: 167\n",
      "Loss: 0.006 | Acc: 99.918% (49959/50000)\n",
      "Loss: 0.200 | Acc: 94.930% (9493/10000)\n",
      "\n",
      "Epoch: 168\n",
      "Loss: 0.005 | Acc: 99.934% (49967/50000)\n",
      "Loss: 0.180 | Acc: 95.280% (9528/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 169\n",
      "Loss: 0.004 | Acc: 99.970% (49985/50000)\n",
      "Loss: 0.181 | Acc: 95.300% (9530/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 170\n",
      "Loss: 0.003 | Acc: 99.980% (49990/50000)\n",
      "Loss: 0.182 | Acc: 95.360% (9536/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 171\n",
      "Loss: 0.003 | Acc: 99.984% (49992/50000)\n",
      "Loss: 0.182 | Acc: 95.380% (9538/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 172\n",
      "Loss: 0.003 | Acc: 99.978% (49989/50000)\n",
      "Loss: 0.177 | Acc: 95.390% (9539/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 173\n",
      "Loss: 0.003 | Acc: 99.986% (49993/50000)\n",
      "Loss: 0.175 | Acc: 95.590% (9559/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 174\n",
      "Loss: 0.003 | Acc: 99.992% (49996/50000)\n",
      "Loss: 0.171 | Acc: 95.620% (9562/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 175\n",
      "Loss: 0.003 | Acc: 99.992% (49996/50000)\n",
      "Loss: 0.168 | Acc: 95.570% (9557/10000)\n",
      "\n",
      "Epoch: 176\n",
      "Loss: 0.003 | Acc: 99.996% (49998/50000)\n",
      "Loss: 0.171 | Acc: 95.520% (9552/10000)\n",
      "\n",
      "Epoch: 177\n",
      "Loss: 0.002 | Acc: 100.000% (50000/50000)\n",
      "Loss: 0.167 | Acc: 95.580% (9558/10000)\n",
      "\n",
      "Epoch: 178\n",
      "Loss: 0.002 | Acc: 99.996% (49998/50000)\n",
      "Loss: 0.167 | Acc: 95.590% (9559/10000)\n",
      "\n",
      "Epoch: 179\n",
      "Loss: 0.003 | Acc: 99.996% (49998/50000)\n",
      "Loss: 0.171 | Acc: 95.560% (9556/10000)\n",
      "\n",
      "Epoch: 180\n",
      "Loss: 0.003 | Acc: 99.998% (49999/50000)\n",
      "Loss: 0.163 | Acc: 95.610% (9561/10000)\n",
      "\n",
      "Epoch: 181\n",
      "Loss: 0.002 | Acc: 100.000% (50000/50000)\n",
      "Loss: 0.164 | Acc: 95.680% (9568/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 182\n",
      "Loss: 0.002 | Acc: 99.998% (49999/50000)\n",
      "Loss: 0.163 | Acc: 95.640% (9564/10000)\n",
      "\n",
      "Epoch: 183\n",
      "Loss: 0.002 | Acc: 99.992% (49996/50000)\n",
      "Loss: 0.165 | Acc: 95.630% (9563/10000)\n",
      "\n",
      "Epoch: 184\n",
      "Loss: 0.003 | Acc: 99.994% (49997/50000)\n",
      "Loss: 0.163 | Acc: 95.630% (9563/10000)\n",
      "\n",
      "Epoch: 185\n",
      "Loss: 0.002 | Acc: 99.994% (49997/50000)\n",
      "Loss: 0.163 | Acc: 95.600% (9560/10000)\n",
      "\n",
      "Epoch: 186\n",
      "Loss: 0.002 | Acc: 99.998% (49999/50000)\n",
      "Loss: 0.163 | Acc: 95.620% (9562/10000)\n",
      "\n",
      "Epoch: 187\n",
      "Loss: 0.002 | Acc: 100.000% (50000/50000)\n",
      "Loss: 0.164 | Acc: 95.720% (9572/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 188\n",
      "Loss: 0.002 | Acc: 100.000% (50000/50000)\n",
      "Loss: 0.163 | Acc: 95.590% (9559/10000)\n",
      "\n",
      "Epoch: 189\n",
      "Loss: 0.002 | Acc: 99.994% (49997/50000)\n",
      "Loss: 0.163 | Acc: 95.590% (9559/10000)\n",
      "\n",
      "Epoch: 190\n",
      "Loss: 0.002 | Acc: 100.000% (50000/50000)\n",
      "Loss: 0.163 | Acc: 95.600% (9560/10000)\n",
      "\n",
      "Epoch: 191\n",
      "Loss: 0.002 | Acc: 99.998% (49999/50000)\n",
      "Loss: 0.162 | Acc: 95.740% (9574/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 192\n",
      "Loss: 0.002 | Acc: 99.994% (49997/50000)\n",
      "Loss: 0.163 | Acc: 95.740% (9574/10000)\n",
      "\n",
      "Epoch: 193\n",
      "Loss: 0.002 | Acc: 99.994% (49997/50000)\n",
      "Loss: 0.161 | Acc: 95.690% (9569/10000)\n",
      "\n",
      "Epoch: 194\n",
      "Loss: 0.002 | Acc: 99.994% (49997/50000)\n",
      "Loss: 0.163 | Acc: 95.710% (9571/10000)\n",
      "\n",
      "Epoch: 195\n",
      "Loss: 0.002 | Acc: 100.000% (50000/50000)\n",
      "Loss: 0.163 | Acc: 95.630% (9563/10000)\n",
      "\n",
      "Epoch: 196\n",
      "Loss: 0.002 | Acc: 99.998% (49999/50000)\n",
      "Loss: 0.162 | Acc: 95.650% (9565/10000)\n",
      "\n",
      "Epoch: 197\n",
      "Loss: 0.002 | Acc: 100.000% (50000/50000)\n",
      "Loss: 0.163 | Acc: 95.610% (9561/10000)\n",
      "\n",
      "Epoch: 198\n",
      "Loss: 0.002 | Acc: 99.996% (49998/50000)\n",
      "Loss: 0.164 | Acc: 95.620% (9562/10000)\n",
      "\n",
      "Epoch: 199\n",
      "Loss: 0.002 | Acc: 99.998% (49999/50000)\n",
      "Loss: 0.164 | Acc: 95.520% (9552/10000)\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(start_epoch, 200):\n",
    "    train(epoch)\n",
    "    test(epoch)\n",
    "    scheduler.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "77f56d3e-1db8-4509-a9a8-6e3fb23aec07",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95.74\n"
     ]
    }
   ],
   "source": [
    "print(best_acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c28ae517-57d8-4f89-a336-99375fefd393",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "my_env",
   "language": "python",
   "name": "my_env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}