Skip to content
This repository was archived by the owner on Jan 3, 2023. It is now read-only.

resolve python3 compatibility issue #53

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

resolve python3 compatibility issue #53

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 42 additions & 54 deletions notebooks/neural_networks/introduction_to_mnist.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,7 @@
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": false
},
"metadata": {},
"outputs": [
{
"name": "stdout",
Expand All @@ -47,6 +45,7 @@
"import matplotlib.pyplot as plt\n",
"from pyspark import SparkContext\n",
"from matplotlib.pyplot import imshow\n",
"\n",
"sc=SparkContext.getOrCreate(conf=create_spark_conf().setMaster(\"local[4]\").set(\"spark.driver.memory\",\"2g\"))\n",
"init_engine()"
]
Expand All @@ -60,19 +59,26 @@
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": false
},
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"('Extracting', 'datasets/mnist/train-images-idx3-ubyte.gz')\n",
"('Extracting', 'datasets/mnist/train-labels-idx1-ubyte.gz')\n",
"('Extracting', 'datasets/mnist/t10k-images-idx3-ubyte.gz')\n",
"('Extracting', 'datasets/mnist/t10k-labels-idx1-ubyte.gz')\n",
"Downloading data from http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\n",
"9846784/9912422 [============================>.] - ETA: 0sSuccessfully downloaded train-images-idx3-ubyte.gz 9912422 bytes.\n",
"Extracting datasets/mnist/train-images-idx3-ubyte.gz\n",
"Downloading data from http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz\n",
" 8192/28881 [=======>......................] - ETA: 0sSuccessfully downloaded train-labels-idx1-ubyte.gz 28881 bytes.\n",
"Extracting datasets/mnist/train-labels-idx1-ubyte.gz\n",
"Downloading data from http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz\n",
"1556480/1648877 [===========================>..] - ETA: 0sSuccessfully downloaded t10k-images-idx3-ubyte.gz 1648877 bytes.\n",
"Extracting datasets/mnist/t10k-images-idx3-ubyte.gz\n",
"Downloading data from http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz\n",
"8192/4542 [======================================================] - 0s\n",
"Successfully downloaded t10k-labels-idx1-ubyte.gz 4542 bytes.\n",
"Extracting datasets/mnist/t10k-labels-idx1-ubyte.gz\n",
"(60000, 28, 28, 1)\n",
"(60000,)\n",
"(10000, 28, 28, 1)\n",
Expand All @@ -85,10 +91,10 @@
"(train_images, train_labels) = mnist.read_data_sets(mnist_path, \"train\")\n",
"(test_images, test_labels) = mnist.read_data_sets(mnist_path, \"test\")\n",
"\n",
"print train_images.shape\n",
"print train_labels.shape\n",
"print test_images.shape\n",
"print test_labels.shape"
"print (train_images.shape)\n",
"print (train_labels.shape)\n",
"print (test_images.shape)\n",
"print (test_labels.shape)"
]
},
{
Expand All @@ -100,10 +106,8 @@
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": false
},
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
Expand All @@ -115,9 +119,9 @@
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAABECAYAAACRbs5KAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAE09JREFUeJztnX9QVOUax7+LLFgiIkKCqVCZGXJ1g0ozRmWi1PSqTReV\nEbwwYz+sNJ2xlFHJYhxIp1BJC8zhjsjVCK1srtxqQI20mDRtmAo1MqDAH4MiRDuwZ8/3/uHsuayg\nLXDOQtvzmXn+2XP2PM95zznfffZ9n/c9JpIQBEEQ/vx49XYAgiAIgj6IoAuCIHgIIuiCIAgeggi6\nIAiChyCCLgiC4CGIoAuCIHgIIuiCIAgeggi6IAiChyCCLgiC4CF4u9OZyWSSaamCIAhdhKTJlf0k\nQxcEQfAQRNAFQRA8BBF0QRAED0EEXRAEwUMQQe8m0dHRiI6ORl5eHux2O/Ly8hAVFdXbYQl/UbZs\n2YItW7aAJCoqKhAWFtbbIQldpKSkBKWlpT07CEm3GQB2x/r168d+/foxMDBQs7S0NGZmZjIzM5Mf\nfPABhw0bxn//+98kSavVSqvVyldeeaVb/v7ILBYLL1++zMuXL1NRFM0aGhoM8dcde+SRR3j+/Hne\nc889vOeee9zqe+3atbTb7XQwZcqUXm8Pd9nAgQM5cOBAhoaGcvHixUxNTaWvr6+hPsPDw9nQ0MCG\nhgba7XYqisJp06a5/dxHjx7NsWPHcsmSJSRJu93eqe3fv5/79++nj4+PIXGYzWaazWZOmTKFR48e\n7fV7whXLysqi1WplTk5Op9td1Vi3li26ysiRI+Hj44NJkyYhJiYGAQEBAIAnn3yy0/1/+eUXbN26\nFU888QSam5vx7bffAgCOHDmie2wPPvgg9u3bh0GDBgEASKK5uRltbW0YMmQIJk6cCAD45ptv0NbW\n1qVjT548GUOGDMEHH3zQ4zgfeOABfP311z0+TldJTk7GqlWroKqq9tlf4SUq4eHhWLVqFR566CEA\nQGRkpLYtNDQUy5YtM8z3pUuX8PnnnwMAZs+ebZifzhg7diySk5MBAPHx8fDy8sKwYcOgquoNr7sj\nxnfeeQfLly9HU1OTrjE5ns1Dhw7h/PnzCAkJwfnz53X1oReZmZkAgGeffRY2mw0lJSU9Ol6fEnSL\nxQIAKC0t1S7KzXCIxtq1a/Hbb7+hoKAA9fX1uHLlCgDg9OnTusV26623IioqCrt370ZoaKjTtrNn\nz2Ljxo3Yu3cvjh49qsWUkZHRJR9Tp07F3Xff3WNB9/Lywh133IGwsDCYTC6Vr+pGWFgY+vfv7xZf\nEyZMAAAkJiZiypQpGDt2LABg5cqVAIC6ujrExMRg9+7dKC8vNySGMWPGYPny5Vi4cCFuueUWrb1r\na2vR3NyMe++9F/PmzcP27dsBAJWVlbrH0NLSgurqat2P6woZGRl4/PHHu/XdRYsWYefOndozYwQh\nISF9WtAdCaDZbMYXX3yBwsLCHh1P+tAFQRA8hD6VodfU1AAAGhoabpihl5eXo7GxEbGxsVqXRn5+\nvuGx5eTkICEhodNtUVFR8PPzw5EjRzB16lQAwLhx47rsY9GiRfjyyy97EiaAa3/xn3rqKezevduQ\njPBGxMXFYenSpQCuZaKzZs0CAFy4cEF3X/Pnz8eWLVsAAEFBQTCZTDh8+DCCg4OxadMmbT+TyYTg\n4GAsWLBAN9+DBg3C66+/rsUxcOBAbdvZs2cBANOmTYPZbEZlZSWCgoIQFBSkm//rCQgIwPjx4w07\n/s347LPPnDL0ixcvYufOnfDy8nLqdps0aRKmTJni9vjc/Q+1PZMnT8aaNWuQkJCAy5cvd9iekJCg\ndc1VVVVp/yx7RF8cFJ07dy7fffddPv/8806DKSdOnOCAAQMIgGPHjmVubi5zc3MNHayIjo5mdHQ0\nL1++rMVRWlrKFStWcMWKFbTb7aytreX48eM5Z84cbSBw7969Xfb1008/MT8/v8cxFxcXU1VVrlu3\nzi0DOjExMYyJiWFtbS1tNhttNhsXLVpkiC9vb29OnDiRTU1N2mB0aWkpY2NjaTab6efnx4MHD/Lg\nwYNUFIV2u50rV67UNYbk5GSnwXCHnT59miNGjOCIESMIgKNGjdK2OdrIiDYZNmwYq6qqWFVVpQ2K\nrl27lmFhYYZfe29vb+2cR4wYwZCQkE738/f3Z01NjVN7FRUVGTJgHBQUxKCgIKqqSlVVOXHiRMPb\noTOrrKyk3W6/4XWvqKjQ9OKJJ5646bFc1ti+KOiOG8BkMjE3N1cT0oSEBLdekM6qWT7++GP6+flx\n5syZnDlzJlNTUxkcHKx9xxFrc3Mzo6KiXPY1btw4trS06CLox44dc+uNvGPHDu7YsUM795KSEsN8\ntRfT4uJiFhcX09/fX9uemJjoJBrV1dVO10cP+89//uPk48cff+SePXs4cuRIp/3+/ve/u0XQAXDd\nunVct26dJuiKovCFF15wy/V3xeLj49nc3OzUbps3bzbE1/WC3lvt8M0331BRFMbFxXXYZrFY2NTU\n5LK2/ekF3WGbNm1yyoy9vLzccjFGjx7NgoICzfeFCxd46tQp/uMf/7jp9xz7K4rCgoICl/2tXr2a\nqqr2WNCHDh3K+vp6qqqqZYpGWlBQkHbONpuNly5dYmxsrCG+0tPTtbbdunUr/f39ncQcAH/44Qcn\n0ZgzZ47ucQwbNozr16/n+vXrOWnSJN52222d7rd48WK3CXr7+6+vCfqCBQtYUlLS4R/N9ddOLwsI\nCGBAQACvXLlCVVWZlZXl9nNOT0+nzWZjRUVFh4RiwIAB3LNnDxVF4dGjR3n06FGazeabHs9VjZVB\nUUEQBE+hr2foAwYMYGlpKUtLS2m32/nYY48Z/uvq6+vLAwcOUFEUNjY2srGxkdOmTeOQIUM4fPjw\nm363fYZeVlbmss+8vDyqqsrVq1f3KPb8/HyqqsrKykoGBAQY2k7h4eE8ceKEU4aelpamu5+0tDSm\npaXRbrfTarXyww8/5C233OK0T//+/Tl79my2tLRo8bz66quG3ys3s507d7o9Qyf/P6GnNzP0hQsX\nsqKighUVFbRarU6Z+fHjx3n8+PEO11BvO3DggNszdMdYwvnz52m1WjudVJeTk0NFUVhTU9OV6+qS\nxvapKpfOaGlpwVNPPQXg2mSdHTt24NChQzh+/Di2bdsGAI4fC9247777tJH7OXPmADBmklJndHUy\nkL+/P6ZPn47ExEQAwGOPPQYASE9PR2Njo+7xtWf69OlO1TwlJSVa5YleBAQE4LnnngNw7Tp/8skn\nmDt3rtM+o0aNQkFBAaKjowEARUVFAICNGzfqGsvNWLZsGQYMGOD02d/+9jcAwLFjx3SpXnKFm03o\nMYLw8HAkJSUBuFbl5CAmJqZDHE1NTVi9ejUOHjwIALBarW6L0x1ERkZqc0iCgoKQnZ3dQTdWrlyp\nTcTasGGD7jH0eUEHrpX0ANdmIebl5SEpKQlJSUnaA7Rr1y7U19fr5u/NN9+EyWTCkSNHuizkXl7X\nerHal2x1hcDAwA6fjR8/HiaTCXFxcRg+fDgAwMfHBwsXLoSXlxesVqs2caa1tRXe3t44ceJEt/y7\nyty5c7VZbl988QUA4J///CeuXr2qqx8fHx+nkr9ly5bhtttuQ0pKijbjMDIyEn5+flqWsnv3bgDX\nkgGjuPXWWwEAEREReOWVV7QE4PrrX1dXh5SUFNjtdsNi6S0iIyNx4MABjBw50qX9y8rKkJuba3BU\nHRkyZIihx/f29kZiYqJWrglcu/4PPfQQUlNT8eabb2rPdXx8PEwmE3bt2oWcnBz9g+nrXS7XW2Rk\nJD/99FOncsbt27fz9ttv1+Uv06xZs/j7779TURQuX768y99v3+WSnZ3t8ve2b99Ou93OhoYGnjx5\n0snsdjtVVWVbW5vWBXTs2DFmZWVx4cKFHD58uLZ+xYULF9jW1mbo38rw8HCn9s/Ly2NeXp4hvgIC\nAlhfX8/6+nqtDPH6wbWamhrW1tZSURTW19cbeu5ms5kPPvgga2trNZ/Nzc2sra1lYWEhm5qanEoq\n6+vr+dJLL9HHx8ewtUuuv//cNSgaGRnJc+fOdbpeS/uun/Y2Y8YMw9vAYY4ul8bGRkP9tK+scpzn\n6dOntc+++uorp/ulO/eox1S5dGYBAQFMSkpyasTPPvtMl2PHx8dTURTW1dUxNDTU5e/5+voyIyND\nu6Cffvop/fz8uuR71apV/Oijjzq1lJSUm5YhPv3003z66aepqip//PFHQ2/gt99+W6s3t9lshi8A\nNmHCBE6YMIGXLl3SHpaNGzcyIiKCERERDAkJ4eHDh6koiqH9pT4+Ppw9e7bTj8m6dev48MMPEwAD\nAwN56tQpnjp1qsOPzvz58zl//nzDF+pqL6SFhYWG+gLAsLAwrlmzhmvWrOH999/PyMjIDpaVlaW1\ngzsFfcWKFYYL+vz582mz2Wi1WllfX8/Y2FjGxsbSYrE4Vfa0T/RsNhtra2t51113deW6SpWLIAjC\nX4o/Y4busNbWVra2ttJut7O1tZVTp07t8TEdGfq5c+dc/o6vry/T09O1iSzV1dVuX770vffe43vv\nvUdVVfn6668b4sNisdBisbCqqkrLzouKitx6np3Z5MmTtcx06dKlhvgwm83MyMhwmmD28ccfa5VE\nwcHB/Prrr7VMzGq18tVXX+W+ffucMvX//ve/WgZnsVh0j/P6LinHv5jevD6DBg3qlQz9ySefpKqq\nbGlpMWzWbGlpKauqqpiSktJhW0REBMvKyjpk6IqicNeuXV3y47FdLuPGjeNrr73G4uJip765kydP\n6jLpyCHoW7ZscWl/i8XCgoICKorCffv2ue1mvd7aC7pRM0QvXrzIixcvamJeVlbW5W4lI2zatGna\nw6L3rFDHWvyZmZlUFIVXr17lkiVLOHjwYA4ePJgAeP/99/Orr76ioiisrKxkZWWlNrnK39+f06dP\nZ35+PvPz83n16lXtoe5K0uCqbdu2rcNsTKNmZLpq8+bN6xVBnzNnDlVV5e+//87Ro0cb4uPFF1+8\n4QS+yZMn88qVK1QUhfPmzeO8efO0H9iuTqryKEF39NFmZ2fz119/7TDQ0tbWxoMHD+p289ntdlZX\nV990P8daLo41Xrr6i6u3uUPQ29eb22w2ty/F8EexGSHoS5Ys4ZIlS6goCpuamrhgwQIGBgZyxowZ\nnDFjBgsLC7Up7WlpaU5ruXRmCQkJWnY/atQo3dth6dKlhgu62WzmzJkz/7COPCUlhSkpKU6DxO4U\ndAD8/vvvqaoqt2/f7la/gwYNYnZ2Nu12O8+cOdPj43mEoIeEhHDFihVOCw+1t/LycpaXl3P27Nm6\nXQhHht7a2sqtW7dqf41HjBjB+Ph4HjhwgNXV1VoM586d4549e3ptASCHOQSdpCELY+Xl5dGB49zd\nsfiTK2Zkht6+wqalpYUnTpxgZWVlh0HPtWvXsl+/fr3eFgB45swZp2oTkl0agLuROSZHFRcXU1GU\nG/5wBQYGMjExkVeuXNEyVEc1kFHLQtzINm/ezKtXr7J///5u9ZuamqpVtPzRZERXzFWN7ZN16EOH\nDkVERATeeustjBkzpsP28vJybNq0CR999BGA7td834x+/frhueee096S1NTUhLvvvlvbfuzYMQDX\n3oqSlpamu//uQlKrhdULi8WCuLg4rZ3b2tqwbds2Q5bF7Q533nmnYcd2vBghODgYvr6+2jK1jskx\nn3/+OT788EP8/PPPfabW/LvvvtPaRM9n46233gLw/7cxvfzyy2hubu6w36OPPoqoqChHEgcAOHz4\nMN5++20cOnRIt3hchWSX3x7WXRzvcl28eDFIIjc3F7/88otbfAPyggtBEATPoa90uQQGBvL999/n\n+++/z7Nnz3Y6KaGsrIxz5841dA2I4cOH88svv+y0flRRFF64cMHlAVN3Wvs+9Bu9aLa7NnXqVNps\nNq0tjK5z76pFRkZqXUF6d7k4XvqclJTErKwspqamcujQoW6bKNQdmzFjRof7V48ulxvV2Hdmdrud\ndXV1rKurY05Ojtu7PBy2efNmqqr6h+uN62VnzpzhmTNnqCgK//Wvf+l23D9NH/qECRNYVFTEmpqa\nTkW8ubmZGzZs4IYNG7SXWxhtoaGhXL9+fQdBf+ONNwwZyNLD2veh/9UEHYD2EPX2WEZfsLCwMFZU\nVOgu6I7xpPaLjl1vp0+f5smTJ7l161ZtYlFvtkVdXR2tVivHjBnjFn+pqala/7mePyJ/GkHPzMzs\nIOIVFRXMyMhgenq64SsGeoolJyczOTnZkAw9JCSER44c6dOC7njxRUlJSZ+ovfZk8/X15TPPPMNL\nly5RUa69eaioqIjPPPPMDd9Y1Fu2d+9efvvtt31mAL+75qrGmtoPXBiNyWRynzPhL4W/vz8KCwsR\nFxeH/fv3AwBSUlIMXaBLENwFSZdejiqDooIgCJ5Cb3e5iInpZf7+/szOznaa9t7bMYmJ6WHS5SII\nguAhuNrl4lZBFwRBEIxD+tAFQRA8BBF0QRAED0EEXRAEwUMQQRcEQfAQRNAFQRA8BBF0QRAED0EE\nXRAEwUMQQRcEQfAQRNAFQRA8BBF0QRAED0EEXRAEwUMQQRcEQfAQRNAFQRA8BBF0QRAED0EEXRAE\nwUMQQRcEQfAQRNAFQRA8BBF0QRAED0EEXRAEwUMQQRcEQfAQRNAFQRA8BBF0QRAED0EEXRAEwUP4\nHxebd0nIMiF9AAAAAElFTkSuQmCC\n",
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAABECAYAAACRbs5KAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAE09JREFUeJztnX9QVOUax7+LLFgiIkKCqVCZGXJ1g0ozRmWi1PSqTReVEbwwYz+sNJ2xlFHJYhxIp1BJC8zhjsjVCK1srtxqQI20mDRtmAo1MqDAH4MiRDuwZ8/3/uHsuaygLXDOQtvzmXn+2XP2PM95zznfffZ9n/c9JpIQBEEQ/vx49XYAgiAIgj6IoAuCIHgIIuiCIAgeggi6IAiChyCCLgiC4CGIoAuCIHgIIuiCIAgeggi6IAiChyCCLgiC4CF4u9OZyWSSaamCIAhdhKTJlf0kQxcEQfAQRNAFQRA8BBF0QRAED0EEXRAEwUMQQe8m0dHRiI6ORl5eHux2O/Ly8hAVFdXbYQl/UbZs2YItW7aAJCoqKhAWFtbbIQldpKSkBKWlpT07CEm3GQB2x/r168d+/foxMDBQs7S0NGZmZjIzM5MffPABhw0bxn//+98kSavVSqvVyldeeaVb/v7ILBYLL1++zMuXL1NRFM0aGhoM8dcde+SRR3j+/Hnec889vOeee9zqe+3atbTb7XQwZcqUXm8Pd9nAgQM5cOBAhoaGcvHixUxNTaWvr6+hPsPDw9nQ0MCGhgba7XYqisJp06a5/dxHjx7NsWPHcsmSJSRJu93eqe3fv5/79++nj4+PIXGYzWaazWZOmTKFR48e7fV7whXLysqi1WplTk5Op9td1Vi3li26ysiRI+Hj44NJkyYhJiYGAQEBAIAnn3yy0/1/+eUXbN26FU888QSam5vx7bffAgCOHDmie2wPPvgg9u3bh0GDBgEASKK5uRltbW0YMmQIJk6cCAD45ptv0NbW1qVjT548GUOGDMEHH3zQ4zgfeOABfP311z0+TldJTk7GqlWroKqq9tlf4SUq4eHhWLVqFR566CEAQGRkpLYtNDQUy5YtM8z3pUuX8PnnnwMAZs+ebZifzhg7diySk5MBAPHx8fDy8sKwYcOgquoNr7sjxnfeeQfLly9HU1OTrjE5ns1Dhw7h/PnzCAkJwfnz53X1oReZmZkAgGeffRY2mw0lJSU9Ol6fEnSLxQIAKC0t1S7KzXCIxtq1a/Hbb7+hoKAA9fX1uHLlCgDg9OnTusV26623IioqCrt370ZoaKjTtrNnz2Ljxo3Yu3cvjh49qsWUkZHRJR9Tp07F3Xff3WNB9/Lywh133IGwsDCYTC6Vr+pGWFgY+vfv7xZfEyZMAAAkJiZiypQpGDt2LABg5cqVAIC6ujrExMRg9+7dKC8vNySGMWPGYPny5Vi4cCFuueUWrb1ra2vR3NyMe++9F/PmzcP27dsBAJWVlbrH0NLSgurqat2P6woZGRl4/PHHu/XdRYsWYefOndozYwQhISF9WtAdCaDZbMYXX3yBwsLCHh1P+tAFQRA8hD6VodfU1AAAGhoabpihl5eXo7GxEbGxsVqXRn5+vuGx5eTkICEhodNtUVFR8PPzw5EjRzB16lQAwLhx47rsY9GiRfjyyy97EiaAa3/xn3rqKezevduQjPBGxMXFYenSpQCuZaKzZs0CAFy4cEF3X/Pnz8eWLVsAAEFBQTCZTDh8+DCCg4OxadMmbT+TyYTg4GAsWLBAN9+DBg3C66+/rsUxcOBAbdvZs2cBANOmTYPZbEZlZSWCgoIQFBSkm//rCQgIwPjx4w07/s347LPPnDL0ixcvYufOnfDy8nLqdps0aRKmTJni9vjc/Q+1PZMnT8aaNWuQkJCAy5cvd9iekJCgdc1VVVVp/yx7RF8cFJ07dy7fffddPv/8806DKSdOnOCAAQMIgGPHjmVubi5zc3MNHayIjo5mdHQ0L1++rMVRWlrKFStWcMWKFbTb7aytreX48eM5Z84cbSBw7969Xfb1008/MT8/v8cxFxcXU1VVrlu3zi0DOjExMYyJiWFtbS1tNhttNhsXLVpkiC9vb29OnDiRTU1N2mB0aWkpY2NjaTab6efnx4MHD/LgwYNUFIV2u50rV67UNYbk5GSnwXCHnT59miNGjOCIESMIgKNGjdK2OdrIiDYZNmwYq6qqWFVVpQ2Krl27lmFhYYZfe29vb+2cR4wYwZCQkE738/f3Z01NjVN7FRUVGTJgHBQUxKCgIKqqSlVVOXHiRMPboTOrrKyk3W6/4XWvqKjQ9OKJJ5646bFc1ti+KOiOG8BkMjE3N1cT0oSEBLdekM6qWT7++GP6+flx5syZnDlzJlNTUxkcHKx9xxFrc3Mzo6KiXPY1btw4trS06CLox44dc+uNvGPHDu7YsUM795KSEsN8tRfT4uJiFhcX09/fX9uemJjoJBrV1dVO10cP+89//uPk48cff+SePXs4cuRIp/3+/ve/u0XQAXDdunVct26dJuiKovCFF15wy/V3xeLj49nc3OzUbps3bzbE1/WC3lvt8M0331BRFMbFxXXYZrFY2NTU5LK2/ekF3WGbNm1yyoy9vLzccjFGjx7NgoICzfeFCxd46tQp/uMf/7jp9xz7K4rCgoICl/2tXr2aqqr2WNCHDh3K+vp6qqqqZYpGWlBQkHbONpuNly5dYmxsrCG+0tPTtbbdunUr/f39ncQcAH/44Qcn0ZgzZ47ucQwbNozr16/n+vXrOWnSJN52222d7rd48WK3CXr7+6+vCfqCBQtYUlLS4R/N9ddOLwsICGBAQACvXLlCVVWZlZXl9nNOT0+nzWZjRUVFh4RiwIAB3LNnDxVF4dGjR3n06FGazeabHs9VjZVBUUEQBE+hr2foAwYMYGlpKUtLS2m32/nYY48Z/uvq6+vLAwcOUFEUNjY2srGxkdOmTeOQIUM4fPjwm363fYZeVlbmss+8vDyqqsrVq1f3KPb8/HyqqsrKykoGBAQY2k7h4eE8ceKEU4aelpamu5+0tDSmpaXRbrfTarXyww8/5C233OK0T//+/Tl79my2tLRo8bz66quG3ys3s507d7o9Qyf/P6GnNzP0hQsXsqKighUVFbRarU6Z+fHjx3n8+PEO11BvO3DggNszdMdYwvnz52m1WjudVJeTk0NFUVhTU9OV6+qSxvapKpfOaGlpwVNPPQXg2mSdHTt24NChQzh+/Di2bdsGAI4fC9247777tJH7OXPmADBmklJndHUykL+/P6ZPn47ExEQAwGOPPQYASE9PR2Njo+7xtWf69OlO1TwlJSVa5YleBAQE4LnnngNw7Tp/8sknmDt3rtM+o0aNQkFBAaKjowEARUVFAICNGzfqGsvNWLZsGQYMGOD02d/+9jcAwLFjx3SpXnKFm03oMYLw8HAkJSUBuFbl5CAmJqZDHE1NTVi9ejUOHjwIALBarW6L0x1ERkZqc0iCgoKQnZ3dQTdWrlypTcTasGGD7jH0eUEHrpX0ANdmIebl5SEpKQlJSUnaA7Rr1y7U19fr5u/NN9+EyWTCkSNHuizkXl7XerHal2x1hcDAwA6fjR8/HiaTCXFxcRg+fDgAwMfHBwsXLoSXlxesVqs2caa1tRXe3t44ceJEt/y7yty5c7VZbl988QUA4J///CeuXr2qqx8fHx+nkr9ly5bhtttuQ0pKijbjMDIyEn5+flqWsnv3bgDXkgGjuPXWWwEAEREReOWVV7QE4PrrX1dXh5SUFNjtdsNi6S0iIyNx4MABjBw50qX9y8rKkJuba3BUHRkyZIihx/f29kZiYqJWrglcu/4PPfQQUlNT8eabb2rPdXx8PEwmE3bt2oWcnBz9g+nrXS7XW2RkJD/99FOncsbt27fz9ttv1+Uv06xZs/j7779TURQuX768y99v3+WSnZ3t8ve2b99Ou93OhoYGnjx50snsdjtVVWVbW5vWBXTs2DFmZWVx4cKFHD58uLZ+xYULF9jW1mbo38rw8HCn9s/Ly2NeXp4hvgICAlhfX8/6+nqtDPH6wbWamhrW1tZSURTW19cbeu5ms5kPPvgga2trNZ/Nzc2sra1lYWEhm5qanEoq6+vr+dJLL9HHx8ewtUuuv//cNSgaGRnJc+fOdbpeS/uun/Y2Y8YMw9vAYY4ul8bGRkP9tK+scpzn6dOntc+++uorp/ulO/eox1S5dGYBAQFMSkpyasTPPvtMl2PHx8dTURTW1dUxNDTU5e/5+voyIyNDu6Cffvop/fz8uuR71apV/Oijjzq1lJSUm5YhPv3003z66aepqip//PFHQ2/gt99+W6s3t9lshi8ANmHCBE6YMIGXLl3SHpaNGzcyIiKCERERDAkJ4eHDh6koiqH9pT4+Ppw9e7bTj8m6dev48MMPEwADAwN56tQpnjp1qsOPzvz58zl//nzDF+pqL6SFhYWG+gLAsLAwrlmzhmvWrOH999/PyMjIDpaVlaW1gzsFfcWKFYYL+vz582mz2Wi1WllfX8/Y2FjGxsbSYrE4Vfa0T/RsNhtra2t51113deW6SpWLIAjCX4o/Y4busNbWVra2ttJut7O1tZVTp07t8TEdGfq5c+dc/o6vry/T09O1iSzV1dVuX770vffe43vvvUdVVfn6668b4sNisdBisbCqqkrLzouKitx6np3Z5MmTtcx06dKlhvgwm83MyMhwmmD28ccfa5VEwcHB/Prrr7VMzGq18tVXX+W+ffucMvX//ve/WgZnsVh0j/P6LinHv5jevD6DBg3qlQz9ySefpKqqbGlpMWzWbGlpKauqqpiSktJhW0REBMvKyjpk6IqicNeuXV3y47FdLuPGjeNrr73G4uJip765kydP6jLpyCHoW7ZscWl/i8XCgoICKorCffv2ue1mvd7aC7pRM0QvXrzIixcvamJeVlbW5W4lI2zatGnaw6L3rFDHWvyZmZlUFIVXr17lkiVLOHjwYA4ePJgAeP/99/Orr76ioiisrKxkZWWlNrnK39+f06dPZ35+PvPz83n16lXtoe5K0uCqbdu2rcNsTKNmZLpq8+bN6xVBnzNnDlVV5e+//87Ro0cb4uPFF1+84QS+yZMn88qVK1QUhfPmzeO8efO0H9iuTqryKEF39NFmZ2fz119/7TDQ0tbWxoMHD+p289ntdlZXV990P8daLo41Xrr6i6u3uUPQ29eb22w2ty/F8EexGSHoS5Ys4ZIlS6goCpuamrhgwQIGBgZyxowZnDFjBgsLC7Up7WlpaU5ruXRmCQkJWnY/atQo3dth6dKlhgu62WzmzJkz/7COPCUlhSkpKU6DxO4UdAD8/vvvqaoqt2/f7la/gwYNYnZ2Nu12O8+cOdPj43mEoIeEhHDFihVOCw+1t/LycpaXl3P27Nm6XQhHht7a2sqtW7dqf41HjBjB+Ph4HjhwgNXV1VoM586d4549e3ptASCHOQSdpCELY+Xl5dGB49zdsfiTK2Zkht6+wqalpYUnTpxgZWVlh0HPtWvXsl+/fr3eFgB45swZp2oTkl0agLuROSZHFRcXU1GUG/5wBQYGMjExkVeuXNEyVEc1kFHLQtzINm/ezKtXr7J///5u9ZuamqpVtPzRZERXzFWN7ZN16EOHDkVERATeeustjBkzpsP28vJybNq0CR999BGA7td834x+/frhueee096S1NTUhLvvvlvbfuzYMQDX3oqSlpamu//uQlKrhdULi8WCuLg4rZ3b2tqwbds2Q5bF7Q533nmnYcd2vBghODgYvr6+2jK1jskxn3/+OT788EP8/PPPfabW/LvvvtPaRM9n46233gLw/7cxvfzyy2hubu6w36OPPoqoqChHEgcAOHz4MN5++20cOnRIt3hchWSX3x7WXRzvcl28eDFIIjc3F7/88otbfAPyggtBEATPoa90uQQGBvL999/n+++/z7Nnz3Y6KaGsrIxz5841dA2I4cOH88svv+y0flRRFF64cMHlAVN3Wvs+9Bu9aLa7NnXqVNpsNq0tjK5z76pFRkZqXUF6d7k4XvqclJTErKwspqamcujQoW6bKNQdmzFjRof7V48ulxvV2HdmdruddXV1rKurY05Ojtu7PBy2efNmqqr6h+uN62VnzpzhmTNnqCgK//Wvf+l23D9NH/qECRNYVFTEmpqaTkW8ubmZGzZs4IYNG7SXWxhtoaGhXL9+fQdBf+ONNwwZyNLD2veh/9UEHYD2EPX2WEZfsLCwMFZUVOgu6I7xpPaLjl1vp0+f5smTJ7l161ZtYlFvtkVdXR2tVivHjBnjFn+pqala/7mePyJ/GkHPzMzsIOIVFRXMyMhgenq64SsGeoolJyczOTnZkAw9JCSER44c6dOC7njxRUlJSZ+ovfZk8/X15TPPPMNLly5RUa69eaioqIjPPPPMDd9Y1Fu2d+9efvvtt31mAL+75qrGmtoPXBiNyWRynzPhL4W/vz8KCwsRFxeH/fv3AwBSUlIMXaBLENwFSZdejiqDooIgCJ5Cb3e5iInpZf7+/szOznaa9t7bMYmJ6WHS5SIIguAhuNrl4lZBFwRBEIxD+tAFQRA8BBF0QRAED0EEXRAEwUMQQRcEQfAQRNAFQRA8BBF0QRAED0EEXRAEwUMQQRcEQfAQRNAFQRA8BBF0QRAED0EEXRAEwUMQQRcEQfAQRNAFQRA8BBF0QRAED0EEXRAEwUMQQRcEQfAQRNAFQRA8BBF0QRAED0EEXRAEwUMQQRcEQfAQRNAFQRA8BBF0QRAED0EEXRAEwUP4Hxebd0nIMiF9AAAAAElFTkSuQmCC\n",
"text/plain": [
"<matplotlib.figure.Figure at 0x7f8cb14495d0>"
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {},
Expand All @@ -126,8 +130,8 @@
],
"source": [
"imshow(np.column_stack(train_images[0:10].reshape(10, 28,28)),cmap='gray'); plt.axis('off')\n",
"print \"groud true labels: \"\n",
"print train_labels[0:10]"
"print (\"groud true labels: \")\n",
"print (train_labels[0:10])"
]
},
{
Expand All @@ -140,9 +144,7 @@
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": true
},
"metadata": {},
"outputs": [],
"source": [
"def get_mnist(sc, mnist_path):\n",
Expand All @@ -156,37 +158,35 @@
" rdd_test_images = sc.parallelize(test_images)\n",
" rdd_test_labels = sc.parallelize(test_labels)\n",
" \n",
" rdd_train_sample = rdd_train_images.zip(rdd_train_labels).map(lambda (features, label):\n",
" Sample.from_ndarray((features - training_mean)/training_std, label + 1))\n",
" rdd_test_sample = rdd_test_images.zip(rdd_test_labels).map(lambda (features, label):\n",
" Sample.from_ndarray((features - training_mean)/training_std, label + 1))\n",
" rdd_train_sample = rdd_train_images.zip(rdd_train_labels).map(lambda kv:\n",
" Sample.from_ndarray((kv[0] - training_mean)/training_std, kv[1] + 1))\n",
" rdd_test_sample = rdd_test_images.zip(rdd_test_labels).map(lambda kv:\n",
" Sample.from_ndarray((kv[0] - training_mean)/training_std, kv[1] + 1))\n",
" return (rdd_train_sample, rdd_test_sample)\n"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"collapsed": false
},
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"('Extracting', 'datasets/mnist/train-images-idx3-ubyte.gz')\n",
"('Extracting', 'datasets/mnist/train-labels-idx1-ubyte.gz')\n",
"('Extracting', 'datasets/mnist/t10k-images-idx3-ubyte.gz')\n",
"('Extracting', 'datasets/mnist/t10k-labels-idx1-ubyte.gz')\n",
"Extracting datasets/mnist/train-images-idx3-ubyte.gz\n",
"Extracting datasets/mnist/train-labels-idx1-ubyte.gz\n",
"Extracting datasets/mnist/t10k-images-idx3-ubyte.gz\n",
"Extracting datasets/mnist/t10k-labels-idx1-ubyte.gz\n",
"60000\n",
"10000\n"
]
}
],
"source": [
"(train_data, test_data) = get_mnist(sc, mnist_path)\n",
"print train_data.count()\n",
"print test_data.count()"
"print (train_data.count())\n",
"print (test_data.count())"
]
},
{
Expand All @@ -201,21 +201,9 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"display_name": "Python 3",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.13"
"name": "python3"
}
},
"nbformat": 4,
Expand Down