symbolset-pavel-s-model.ipynb

{"cells":[{"metadata":{"id":"tJimWGHIp1-i","trusted":true},"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\nimport torchvision.transforms as transforms\nimport torch.optim as optim\nimport time ","execution_count":null,"outputs":[]},{"metadata":{"id":"m07oF_jovtHq","trusted":true},"cell_type":"code","source":"batch_size = 64\nepochCount = 20\nlearningRate = 0.0005","execution_count":null,"outputs":[]},{"metadata":{"id":"Tw3u-YSGs_WM","trusted":true},"cell_type":"code","source":"batchrow = int(batch_size / 8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# taken from https://stackoverflow.com/a/56512552\ndef imshow(image, ax=None, title=None, normalize=True):\n  \"\"\"Imshow for Tensor.\"\"\"\n  if ax is None:\n      fig, ax = plt.subplots()\n  image = image.numpy().transpose((1, 2, 0))\n\n  if normalize:\n      mean = np.array([0.485, 0.456, 0.406])\n      std = np.array([0.229, 0.224, 0.225])\n      image = std * image + mean\n      image = np.clip(image, 0, 1)\n\n  ax.imshow(image)\n  ax.spines['top'].set_visible(False)\n  ax.spines['right'].set_visible(False)\n  ax.spines['left'].set_visible(False)\n  ax.spines['bottom'].set_visible(False)\n  ax.tick_params(axis='both', length=0)\n  ax.set_xticklabels('')\n  ax.set_yticklabels('')\n\n  return ax","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import the handwrittenmathsymbols set using the ImageFolder dataset class\ndataset = torchvision.datasets.ImageFolder(root='../input/handwrittenmathsymbols/', transform=transforms.ToTensor())\n\n# dataloaders \"represent a Python iterable over a dataset\"\ndataloader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, num_workers=2, shuffle=True)\n\n# show a random image\nimages, labels = next(iter(dataloader)) #todo label\nimshow(images[0], title=labels[0], normalize=False)\n","execution_count":null,"outputs":[]},{"metadata":{"id":"mULla7jfCHMM","outputId":"669d0af8-3e01-455b-92a0-104caca6f822","trusted":true},"cell_type":"code","source":"mean = 0.\nstd = 0.\n\n# what does this do?\nfor batch_idx, data in enumerate(dataloader, 0):\n    image, label = data\n    batchSize = image.size(0)\n    image = image.view(batchSize, image.size(1), -1)\n    mean += image.mean(2).sum(0)\n    std += image.std(2).sum(0)\n  \nmean /= len(dataset)\nstd /= len(dataset)\nprint(mean, std)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trfms = transforms.Compose(\n    [transforms.ToTensor(),\n     transforms.Normalize((std),(mean))])\n\ndataset = torchvision.datasets.ImageFolder(root='../input/handwrittenmathsymbols/', transform=transforms.ToTensor())\nlen(dataset)","execution_count":null,"outputs":[]},{"metadata":{"id":"9ICz3FlrrRVi","trusted":true},"cell_type":"code","source":"trainset, valset = torch.utils.data.random_split(dataset, [80000, 16429])\n\n#testset = torchvision.datasets.MNIST(root='./data', train=False, download=True, transform=transforms)\n\n\ntrainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=2)\n\nvalloader = torch.utils.data.DataLoader(valset, batch_size=int(batch_size / 5), shuffle=True, num_workers=2)\n\n#testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=2)\n\n\nclasses = ('0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '(', ')', '!', '+', ',', '-', '=', '[', ']', '{', '}', 'alpha', 'ascii_124', 'beta', 'cos', 'Delta', 'div', 'exists',\n'forall', 'forward_slash', 'gamma', 'geq', 'gt', 'in', 'infty', 'int', 'lambda', 'ldots', 'leq', 'lim', 'log', 'lt', 'M', 'mu', 'neq', 'phi', 'pi', 'pm', 'prime', 'rightarrow',\n'sigma', 'sin', 'sqrt', 'sum', 'tan', 'theta', 'times', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x',\n'y', 'z')","execution_count":null,"outputs":[]},{"metadata":{"id":"Nk8yQli6e05I","cellView":"form","trusted":true},"cell_type":"code","source":"#@title Under construction\n#under construction\n\"\"\"\nexp_labels_list = []\nfor i in range(100):\n  _,label = trainset[i]\n  exp_labels_list.append(label)\nprint(exp_labels_list)\ntrainset.targets\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"id":"cVL42jyniwO8","cellView":"form","trusted":true},"cell_type":"code","source":"#@title Under construction\n#under construction, code for dataset splitter with equal distribtuion of classes\n\"\"\"\neqsplitloader = torch.utils.data.DataLoader(trainset, batch_size=1, shuffle=False,num_workers=2)\n\nclinp_list = [] # list of lists, containing images sorted by classes\ncltgt_list = [] # list of lists, containing labels for images\ncl_count = len(classes)\nflag = 0 #constatnt to flag if the image of this class was already sorted\ninp_list = []\ntgt_list = []\n\nfor i in range(cl_count):\n  empty_list = []\n  clinp_list.append(empty_list)\n  cltgt_list.append(empty_list)\n\nfor i, data in enumerate(eqsplitloader, 0):\n  inp, tgt = data\n  inp_list.append(inp)\n  tgt_list.append(tgt)\n\nprint(inp_list[0])\nprint(tgt_list[0])\n\n\n  emptycl_idx = 0 #index of first empty list in list of lists\n  clinp_list[emptycl_idx].append(inp[0]) #sorting the first image and its label\n  cltgt_list[emptycl_idx].append(tgt[0])\n  for k in range(i): #running check\n\n    if tgt[i] == tgt[k]:\n      flag = 1\n      break\n  if flag == 1: #if there is list for class of next item in dataloader\n    for j in range(cl_count): \n          if tgt[i] != cltgt_list[j][0]: #find it\n            continue\n          else:\n            clinp_list[j].append(inp[i]) #store it and its label\n            cltgt_list[j].append(tgt[i])\n            break\n  else: #if loaded image is of new class\n    emptycl_idx += 1 #move index of first empty list\n    clinp_list[emptycl_idx].append(inp[i]) #store the imge and its label\n    cltgt_list[emptycl_idx].append(tgt[i])\n\"\"\"\n  \n\n\n","execution_count":null,"outputs":[]},{"metadata":{"id":"qi9e9kmuYr7j","outputId":"cea3f03b-e551-419c-d57d-0abec0d4f869","trusted":true},"cell_type":"code","source":"print(len(trainset))\nprint(len(valset))\nepoch_count_train = int(len(trainset) / batch_size)\nepoch_count_val = int(5 * len(valset) / batch_size)\nprint(epoch_count_train)\nprint(epoch_count_val)","execution_count":null,"outputs":[]},{"metadata":{"id":"uu6hMYzMy2L_","outputId":"24f83629-5374-4f9b-fd1b-63147f5f6704","trusted":true},"cell_type":"code","source":"def showimg(image):\n  image = image * 0.3015 + 0.1307\n  npimage = image.numpy()\n  plt.imshow(np.transpose(npimage, (1, 2, 0)))\n  plt.show()\n\ndataiter_train = iter(trainloader)\nimages, labels = dataiter_train.next()\n\nshowimg(torchvision.utils.make_grid(images))\nfor k in range(batchrow):\n  print(' '.join('%5s' % classes[labels[j]] for j in range(k * 8, (k + 1) * 8)), end='\\n\\n')\n  k += 1","execution_count":null,"outputs":[]},{"metadata":{"id":"bKIUf3Lt9q96","trusted":true},"cell_type":"code","source":"class skynet1(nn.Module):\n  def __init__(self):\n    super(skynet1, self).__init__()\n    self.conv00 = nn.Conv2d(3, 8, 10)\n    self.conv01 = nn.Conv2d(8, 8, 9)\n    torch.nn.init.xavier_uniform_(self.conv00.weight)\n    torch.nn.init.xavier_uniform_(self.conv01.weight)\n    self.conv1 = nn.Conv2d(3, 8, 3) #3 input channels, 8 output channels(number of filters), 3x3 kernel, stride 1, no padding\n    torch.nn.init.xavier_uniform_(self.conv1.weight)\n    self.pool1 = nn.MaxPool2d(2, 2, 1) #2x2 filter, stride 2, padding 1\n    self.conv2 = nn.Conv2d(8, 16, 3) \n    torch.nn.init.xavier_uniform_(self.conv2.weight)\n    self.pool2 = nn.MaxPool2d(2, 2)\n    self.conv3 = nn.Conv2d(16, 128, 6)\n    torch.nn.init.xavier_uniform_(self.conv3.weight)\n    self.fc1 = nn.Linear(128, 64)\n    torch.nn.init.xavier_uniform_(self.fc1.weight)\n    self.fc2 = nn.Linear(64, 10)\n    torch.nn.init.xavier_uniform_(self.fc2.weight)\n    #self.dropout1 = nn.Dropout(0.1)\n    #self.dropout2 = nn.Dropout(0.5)\n    \n  def forward(self, x):\n    x = F.relu(self.conv00(x))\n    x = F.relu(self.conv01(x))\n    #x = self.dropout1(x)\n    x = F.relu(self.conv1(x))\n    #x = self.dropout1(x)\n    x = self.pool1(x)\n    x = F.relu(self.conv2(x))\n    #x = self.dropout1(x)\n    x = self.pool2(x)\n    x = F.relu(self.conv3(x))\n    #x = self.dropout1(x)\n    x = x.view(-1, 128)\n    x = self.fc1(x)\n    #x = self.dropout2(x)\n    x = F.relu(x)\n    x = self.fc2(x)\n    return x\n\nnet = skynet1()\n","execution_count":null,"outputs":[]},{"metadata":{"id":"n8IYVXC0UY89","outputId":"4f555510-2a53-43b0-f3eb-612f9f465176","trusted":true},"cell_type":"code","source":"params = list(net.parameters())\nprint(len(params))\nfor i in range(10):\n  print(params[i].size())","execution_count":null,"outputs":[]},{"metadata":{"id":"d6dtp2iQLmjR","trusted":true},"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(params, lr = learningRate, betas = (0.9,0.999), eps = 1e-6)","execution_count":null,"outputs":[]},{"metadata":{"id":"VG1gE4SgRytq","outputId":"451c8660-f8f8-4c62-d2de-fe86cf3f84f7","trusted":true},"cell_type":"code","source":"tloss_plt = []\nepoch_tloss_plt = []\ntrain_acc_plot = []\ncorrect_train = 0.0\ntotal_train = 0.0\n\nvloss_plt = []\nepoch_vloss_plt = []\nval_acc_plot = []\ncorrect_val = 0.0\ntotal_val = 0.0\n\n\nfor epoch in range(epochCount):\n\n    running_tloss = 0.0\n    running_vloss = 0.0\n    avg_perbatch_acc_train = 0.0\n    avg_perbatch_acc_val = 0.0\n    start = time.time()\n\n    print('epoch', epoch + 1)\n\n    net.train(mode=True)\n    print('training cycle')\n    for i, data in enumerate(trainloader, 0):\n      inputs, labels = data\n\n      optimizer.zero_grad()\n\n      outputs = net(inputs)\n      print(outputs.shape, labels.shape)\n      loss = criterion(outputs, labels)\n      loss.backward()\n      optimizer.step()\n\n      perbatch_acc_train = 0.0\n      _, predicted_train = torch.max(outputs.data, 1)\n      total_train += labels.size(0)\n      correct_train += (predicted_train == labels).sum().item()\n      perbatch_acc_train = correct_train / total_train\n      avg_perbatch_acc_train += perbatch_acc_train / 2\n\n      running_tloss += loss.item()\n\n      if i % 50 == 49:\n         tloss_plt.append(loss.item())\n         \n      if i % 200 == 199:\n        print('[%d, %5d] train loss: %.6f, avg_train_acc: %.2f %%' % (epoch + 1, i + 1, running_tloss / 200, avg_perbatch_acc_train))\n        train_acc_plot.append(avg_perbatch_acc_train)\n        running_tloss = 0.0\n        avg_perbatch_acc_train = 0.0\n\n    net.train(mode=False)\n    print('evaluation cycle')\n    with torch.no_grad():\n      for i, data in enumerate(valloader, 0):\n        inputs, labels = data\n\n        outputs = net(inputs)\n        loss = criterion(outputs, labels)\n        \n        perbatch_acc_val = 0.0\n        _, predicted_val = torch.max(outputs.data, 1)\n        total_val += labels.size(0)\n        correct_val += (predicted_val == labels).sum().item()\n        perbatch_acc_val = correct_val / total_val\n        avg_perbatch_acc_val += perbatch_acc_val / 2\n\n        running_vloss += loss.item()\n\n        if i % 50 == 49:\n         vloss_plt.append(loss.item())\n         \n        if i % 200 == 199:\n          print('[%d, %5d] val loss: %.6f, avg_val_acc: %.2f %%' % (epoch + 1, i + 1, running_vloss / 200, avg_perbatch_acc_val))\n          val_acc_plot.append(avg_perbatch_acc_val)\n          running_vloss = 0.0\n          avg_perbatch_acc_val = 0.0\n\n    epoch_tloss_plt.append(running_tloss / epoch_count_train)\n    epoch_vloss_plt.append(running_vloss / epoch_count_val)\n    end = time.time()\n    print('time spent on epoch %d: %.2f s' % (epoch + 1,end - start))\nprint('sector clear!')","execution_count":null,"outputs":[]},{"metadata":{"id":"JjqDroVrz3K2","outputId":"ac7fab2c-6d96-433b-c3ca-ae8d58b4a07b","trusted":true},"cell_type":"code","source":"print(len(tloss_plt))\nprint(len(vloss_plt))\nplt.figure(figsize=[10, 10])\nplt.plot(tloss_plt)\nplt.plot(vloss_plt)\nplt.ylabel('loss')","execution_count":null,"outputs":[]},{"metadata":{"id":"hDBnbQfNZphr","outputId":"8947441c-672d-46a4-c5e4-e9c1662bea98","trusted":true},"cell_type":"code","source":"plt.figure(figsize=[10, 10])\nplt.plot(epoch_tloss_plt)\nplt.plot(epoch_vloss_plt)\nplt.xlabel('epoch')\nplt.ylabel('loss')","execution_count":null,"outputs":[]},{"metadata":{"id":"6-nc5JohfQV_","outputId":"f64459be-a597-4cc8-afc1-2f82f0af8741","trusted":true},"cell_type":"code","source":"dataiter = iter(testloader)\nimages, labels = dataiter.next()\n\nshowimg(torchvision.utils.make_grid(images))\nprint('ground truth labels:')\nfor k in range(batchrow):\n  print(' '.join('%5s' % classes[labels[j]] for j in range(k * 8, (k + 1) * 8)),end='\\n\\n')\n  k += 1\n\noutputs = net(images)\n\n_,predicted = torch.max(outputs, 1)\nprint('predicted:')\nfor k in range(batchrow):\n  print(' '.join('%5s' % classes[predicted[j]] for j in range(k * 8, (k + 1) * 8)),end='\\n\\n')\n  k += 1","execution_count":null,"outputs":[]},{"metadata":{"id":"LulKEYBohY5-","outputId":"beb99237-1a18-457b-aae6-8d0f6fa16eef","trusted":true},"cell_type":"code","source":"loss_test_plt = []\ncorrect_test = 0\ntotal_test = 0\ntest_acc_plot = []\navg_test_acc = 0.0\navg_loss_test = 0.0\n\nwith torch.no_grad():\n    for data in testloader:\n        images, labels = data\n        outputs = net(images)\n\n        _, predicted_test = torch.max(outputs.data, 1)\n        total_test += labels.size(0)\n        correct_test += (predicted_test == labels).sum().item()\n        avg_test_acc = 100 * correct_test / total_test\n        test_acc_plot.append(avg_test_acc)\n\nprint('accuracy %.2f %%, error rate %.2f %%' % (avg_test_acc, 100 - avg_test_acc))","execution_count":null,"outputs":[]},{"metadata":{"id":"HEO4P0zjSo8M","outputId":"3a59b214-5652-4b09-c38f-1a1ee07d5650","trusted":true},"cell_type":"code","source":"print(len(train_acc_plot),len(val_acc_plot),len(test_acc_plot))\nplt.figure(figsize=[10, 10])\nplt.plot(test_acc_plot,label='test set accuracy')\nplt.plot(val_acc_plot,label='validation set accuracy')\nplt.plot(train_acc_plot,label='train set accuracy')\nplt.legend(loc='upper right')\nplt.ylabel('accuracy,%')","execution_count":null,"outputs":[]},{"metadata":{"id":"dmCAswyypaNS"},"cell_type":"markdown","source":"version-batchsize-epochnum-lr-dropout-error\n\n1-64-60-0.0005-no-1.19%\n\n2-64-30-0.0005-no-1.06%\n\n3-32-30-0.0005-no-1.02%\n\n4-32-30-0.0005-fc1,fc2-40.78%\n\n5-64-15-0.005-fc1,fc2-40.30%\n\n6-64-15-0.005-fc1-1.66%\n\n7-64-15-0.05-fc1-43.58%\n\n8-128-15-0.0005-fc1-1.39%\n\n9-64-15-0.0005-init-1.07%\n\n10-32-15-0.0005-init-1.10%\n\n11-64-15-0.0005-init,fc1-1.49%\n\n12-64-15-0.0005-init,fc1,fc2(0.1)-5.26%\n\n12-64-15-0.0005-init,conv1,conv2,conv3-1.33%\n\n12-64-50-0.0001-init-1.27%"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}