diff --git a/notebooks/bigdl-tutorials-scala/autoencoder.ipynb b/notebooks/bigdl-tutorials-scala/autoencoder.ipynb new file mode 100644 index 0000000..0757e4f --- /dev/null +++ b/notebooks/bigdl-tutorials-scala/autoencoder.ipynb @@ -0,0 +1,4160 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Using an auto encoder on MNIST handwritten digits.¢" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import org.apache.log4j.{Level, Logger}\n", + "import org.apache.spark.SparkContext\n", + "\n", + "import com.intel.analytics.bigdl.utils._\n", + "import com.intel.analytics.bigdl.utils.{Engine, LoggerFilter, T, Table}\n", + "import com.intel.analytics.bigdl.dataset.DataSet\n", + "import com.intel.analytics.bigdl.dataset.image.{BytesToGreyImg, GreyImgNormalizer, GreyImgToBatch, GreyImgToSample}\n", + "import com.intel.analytics.bigdl.models.lenet.Utils._\n", + "import com.intel.analytics.bigdl.models.autoencoder._\n", + "import com.intel.analytics.bigdl.nn._\n", + "import com.intel.analytics.bigdl.optim._\n", + "import com.intel.analytics.bigdl.optim.{Adam, Top1Accuracy, Trigger}\n", + "import com.intel.analytics.bigdl.visualization.{TrainSummary, ValidationSummary}\n", + "import com.intel.analytics.bigdl.tensor.Tensor\n", + "import com.intel.analytics.bigdl.numeric.NumericFloat\n", + "\n", + "Engine.init" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Load MNIST dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import java.nio.ByteBuffer\n", + "import java.nio.file.{Files, Path, Paths}\n", + "\n", + "import com.intel.analytics.bigdl.dataset.ByteRecord\n", + "import com.intel.analytics.bigdl.utils.File\n", + "import scopt.OptionParser\n", + "\n", + "def load(featureFile: String, labelFile: String): Array[ByteRecord] = {\n", + " val featureBuffer = ByteBuffer.wrap(Files.readAllBytes(Paths.get(featureFile)))\n", + " val labelBuffer = ByteBuffer.wrap(Files.readAllBytes(Paths.get(labelFile)))\n", + " \n", + " val labelMagicNumber = labelBuffer.getInt()\n", + " require(labelMagicNumber == 2049)\n", + " val featureMagicNumber = featureBuffer.getInt()\n", + " require(featureMagicNumber == 2051)\n", + "\n", + " val labelCount = labelBuffer.getInt()\n", + " val featureCount = featureBuffer.getInt()\n", + " require(labelCount == featureCount)\n", + "\n", + " val rowNum = featureBuffer.getInt()\n", + " val colNum = featureBuffer.getInt()\n", + "\n", + " val result = new Array[ByteRecord](featureCount)\n", + " var i = 0\n", + " while (i < featureCount) {\n", + " val img = new Array[Byte]((rowNum * colNum))\n", + " var y = 0\n", + " while (y < rowNum) {\n", + " var x = 0\n", + " while (x < colNum) {\n", + " img(x + y * colNum) = featureBuffer.get()\n", + " x += 1\n", + " }\n", + " y += 1\n", + " }\n", + " result(i) = ByteRecord(img, labelBuffer.get().toFloat + 1.0f)\n", + " i += 1\n", + " }\n", + "\n", + " result\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then we need to set paths of data. Please edit paths if they are changed." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "val trainData = \"../../datasets/mnist/train-images-idx3-ubyte\"\n", + "val trainLabel = \"../../datasets/mnist/train-labels-idx1-ubyte\"\n", + "val validationData = \"../../datasets/mnist/t10k-images-idx3-ubyte\"\n", + "val validationLabel = \"../../datasets/mnist/t10k-labels-idx1-ubyte\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Model Setup" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "//Parameters\n", + "val batchSize = 128\n", + "val maxEpochs = 2\n", + "val displayStep = 1\n", + "val examples2Show = 10\n", + "\n", + "//Network Parameters\n", + "val nInput = 784 //MNIST data input (img shape: 28*28)\n", + "val nHidden = 32 // hidden layer num of features" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then the data set should be created and the model needs to be established." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "val trainMean = 0.13066047740239436\n", + "val trainStd = 0.30810779333114624\n", + "val trainSet = \n", + " DataSet.array(load(trainData, trainLabel), sc) -> BytesToGreyImg(28, 28) -> GreyImgNormalizer(trainMean, trainStd) -> GreyImgToBatch(batchSize) -> toAutoencoderBatch()\n", + "val validationSet = \n", + " DataSet.array(load(validationData, validationLabel), sc) -> BytesToGreyImg(28, 28) -> GreyImgNormalizer(testMean, testStd) -> GreyImgToBatch(batchSize) -> toAutoencoderBatch()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Sequential[d3fa8b9b]{\n", + " [input -> (1) -> (2) -> (3) -> (4) -> (5) -> output]\n", + " (1): Reshape[a4e0b228](784)\n", + " (2): Linear[d8a294c5](784 -> 32)\n", + " (3): ReLU[678cd7a1](0.0, 0.0)\n", + " (4): Linear[f62a17fc](32 -> 784)\n", + " (5): Sigmoid[66c5634b]\n", + "}" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "var model = Sequential()\n", + "model.add(Reshape(Array(28*28)))\n", + "// encoder\n", + "model.add(Linear(nInput, nHidden))\n", + "model.add(ReLU[Float]())\n", + "// decoder\n", + "model.add(Linear(nHidden, nInput))\n", + "model.add(Sigmoid[Float]())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Optimizer Setup" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "com.intel.analytics.bigdl.optim.DistriOptimizer@2d7621e1" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "val optimizer = Optimizer(model = model, dataset = trainSet, criterion = new MSECriterion[Float]())\n", + "optimizer.setOptimMethod(new Adam())\n", + "optimizer.setEndWhen(Trigger.maxEpoch(maxEpochs))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following is to create training and validation summary." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "saving logs to autoencoder-20170925-041354" + ] + } + ], + "source": [ + "import java.text.SimpleDateFormat\n", + "import java.util.Calendar\n", + "val today = Calendar.getInstance\n", + "val formatDate = new SimpleDateFormat(\"yyyyMMdd-hhmmss\")\n", + "val name = \"autoencoder-\" + formatDate.format(today.getTime()).toString()\n", + "val trainSummary = TrainSummary(logDir=\"/tmp/bigdl_summaries\", appName=name)\n", + "trainSummary.setSummaryTrigger(\"Parameters\", Trigger.severalIteration(50))\n", + "val valSummary = ValidationSummary(logDir=\"/tmp/bigdl_summaries\", appName=name)\n", + "optimizer.setTrainSummary(trainSummary)\n", + "optimizer.setValidationSummary(valSummary)\n", + "printf(\"saving logs to %s\", name)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Optimization Done." + ] + } + ], + "source": [ + "// Boot training process\n", + "val trainedModel = optimizer.optimize()\n", + "print(\"Optimization Done.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import vegas._\n", + "import vegas.render.HTMLRenderer._\n", + "import vegas.DSL._" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "val loss = trainSummary.readScalar(\"Loss\")\n", + "val lossXY = loss.map(_ ._1).zip(loss.map(_ ._2)).toSeq\n", + "Vegas(description = \"The Loss curve.\", width = 700.0, height = 300.0).\n", + " withXY(lossXY).\n", + " encodeX(\"x\", Quantitative, bin = Bin(maxbins = 500.0), title = \"Iteration\").\n", + " encodeY(\"y\", Quantitative, title = \"Loss\").\n", + " mark(Line).\n", + " show" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, the Spark should be stopped." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "sc.stop()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Apache Toree - Scala", + "language": "scala", + "name": "apache_toree_scala" + }, + "language_info": { + "file_extension": ".scala", + "name": "scala", + "version": "2.11.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/bigdl-tutorials-scala/birnn.ipynb b/notebooks/bigdl-tutorials-scala/birnn.ipynb new file mode 100644 index 0000000..63b6535 --- /dev/null +++ b/notebooks/bigdl-tutorials-scala/birnn.ipynb @@ -0,0 +1,19308 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Handwritten Digit Classfication using Bidirectional Recurrent Neural Network" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this example, we are going to use the MNIST dataset to train a multi-layer feed foward neural network. MNIST is a simple computer vision dataset of handwritten digits. It has 60,000 training examles and 10,000 test examples. \"It is a good database for people who want to try learning techniques and pattern recognition methods on real-world data while spending minimal efforts on preprocessing and formatting.\" For more details, please checkout the website [MNIST](http://yann.lecun.com/exdb/mnist/)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import org.apache.log4j.{Level, Logger}\n", + "import org.apache.spark.SparkContext\n", + "\n", + "import com.intel.analytics.bigdl.utils._\n", + "import com.intel.analytics.bigdl.utils.{Engine, LoggerFilter, T, Table}\n", + "import com.intel.analytics.bigdl.dataset.DataSet\n", + "import com.intel.analytics.bigdl.dataset.image.{BytesToGreyImg, GreyImgNormalizer, GreyImgToBatch, GreyImgToSample}\n", + "import com.intel.analytics.bigdl.models.lenet.Utils._\n", + "import com.intel.analytics.bigdl.nn._\n", + "import com.intel.analytics.bigdl.optim._\n", + "import com.intel.analytics.bigdl.optim.{Adam, Top1Accuracy, Trigger}\n", + "import com.intel.analytics.bigdl.visualization.{TrainSummary, ValidationSummary}\n", + "import com.intel.analytics.bigdl.tensor.Tensor\n", + "import com.intel.analytics.bigdl.numeric.NumericFloat\n", + "\n", + "Engine.init" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Load MNIST dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import java.nio.ByteBuffer\n", + "import java.nio.file.{Files, Path, Paths}\n", + "\n", + "import com.intel.analytics.bigdl.dataset.ByteRecord\n", + "import com.intel.analytics.bigdl.utils.File\n", + "import scopt.OptionParser\n", + "\n", + "def load(featureFile: String, labelFile: String): Array[ByteRecord] = {\n", + " val featureBuffer = ByteBuffer.wrap(Files.readAllBytes(Paths.get(featureFile)))\n", + " val labelBuffer = ByteBuffer.wrap(Files.readAllBytes(Paths.get(labelFile)))\n", + " \n", + " val labelMagicNumber = labelBuffer.getInt()\n", + " require(labelMagicNumber == 2049)\n", + " val featureMagicNumber = featureBuffer.getInt()\n", + " require(featureMagicNumber == 2051)\n", + "\n", + " val labelCount = labelBuffer.getInt()\n", + " val featureCount = featureBuffer.getInt()\n", + " require(labelCount == featureCount)\n", + "\n", + " val rowNum = featureBuffer.getInt()\n", + " val colNum = featureBuffer.getInt()\n", + "\n", + " val result = new Array[ByteRecord](featureCount)\n", + " var i = 0\n", + " while (i < featureCount) {\n", + " val img = new Array[Byte]((rowNum * colNum))\n", + " var y = 0\n", + " while (y < rowNum) {\n", + " var x = 0\n", + " while (x < colNum) {\n", + " img(x + y * colNum) = featureBuffer.get()\n", + " x += 1\n", + " }\n", + " y += 1\n", + " }\n", + " result(i) = ByteRecord(img, labelBuffer.get().toFloat + 1.0f)\n", + " i += 1\n", + " }\n", + "\n", + " result\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then we need to set paths of data. Please edit paths if they are changed." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "val trainData = \"../../datasets/mnist/train-images-idx3-ubyte\"\n", + "val trainLabel = \"../../datasets/mnist/train-labels-idx1-ubyte\"\n", + "val validationData = \"../../datasets/mnist/t10k-images-idx3-ubyte\"\n", + "val validationLabel = \"../../datasets/mnist/t10k-labels-idx1-ubyte\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Recurent Neural Network Model Setup" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Bidirectional RNNs are based on the idea that the output at time t may not only depend on the previous elements in the sequence, but also future elements. They combine an RNN that moves foward through time beginning from the end of the sequence with another RNN that moves backward through time from the end of the sequence." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "//Parameters\n", + "val batchSize = 64\n", + "val maxEpochs = 5\n", + "\n", + "//Network Parameters\n", + "val nInput = 28 //MNIST data input (img shape: 28*28)\n", + "val nHidden = 128 // hidden layer num of features\n", + "val nClasses = 10 //MNIST total classes (0-9 digits)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then the data set should be created and the model needs to be established." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "val trainSet = \n", + " DataSet.array(load(trainData, trainLabel), sc) -> BytesToGreyImg(28, 28) -> GreyImgNormalizer(trainMean, trainStd) -> GreyImgToBatch(batchSize)\n", + "val validationSet = \n", + " DataSet.array(load(validationData, validationLabel), sc) -> BytesToGreyImg(28, 28) -> GreyImgNormalizer(testMean, testStd) -> GreyImgToBatch(batchSize)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "val recurrent = BiRecurrent(JoinTable(3, 3)).add(LSTM(nInput, nHidden))\n", + "val rnnModel = Sequential().add(InferReshape(Array(-1, nInput), true)).add(recurrent).add(Select(2, -1)).add(Linear(2*nHidden, nClasses))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Optimizer Setup" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "com.intel.analytics.bigdl.optim.DistriOptimizer@783bcb1f" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "val optimizer = Optimizer(model = rnnModel, dataset = trainSet, criterion = CrossEntropyCriterion[Float]())\n", + "optimizer.setValidation(trigger = Trigger.everyEpoch, dataset = validationSet, vMethods = Array(new Top1Accuracy))\n", + "optimizer.setOptimMethod(new Adam())\n", + "optimizer.setEndWhen(Trigger.maxEpoch(maxEpochs))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following is to create training and validation summary." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "saving logs to rnn-20170922-012336" + ] + } + ], + "source": [ + "import java.text.SimpleDateFormat\n", + "import java.util.Calendar\n", + "val today = Calendar.getInstance\n", + "val formatDate = new SimpleDateFormat(\"yyyyMMdd-hhmmss\")\n", + "val name = \"rnn-\" + formatDate.format(today.getTime()).toString()\n", + "val trainSummary = TrainSummary(logDir=\"/tmp/bigdl_summaries\", appName=name)\n", + "trainSummary.setSummaryTrigger(\"Parameters\", Trigger.severalIteration(50))\n", + "val valSummary = ValidationSummary(logDir=\"/tmp/bigdl_summaries\", appName=name)\n", + "optimizer.setTrainSummary(trainSummary)\n", + "optimizer.setValidationSummary(valSummary)\n", + "printf(\"saving logs to %s\", name)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "can't find locality partition for partition 0 Partition locations are (ArrayBuffer(172.168.2.109)) Candidate partition locations are\n", + "(0,List()).\n", + "Optimization Done." + ] + } + ], + "source": [ + "// Boot training process\n", + "val trainedModel = optimizer.optimize()\n", + "print(\"Optimization Done.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Top1Accuracy is Accuracy(correct: 9818, count: 10000, accuracy: 0.9818)\n" + ] + } + ], + "source": [ + "val rddData = sc.parallelize(load(validationData, validationLabel), batchSize)\n", + "val transformer = BytesToGreyImg(28, 28) -> GreyImgNormalizer(testMean, testStd) -> GreyImgToSample()\n", + "val evaluationSet = transformer(rddData)\n", + " \n", + "val result = trainedModel.evaluate(evaluationSet, Array(new Top1Accuracy[Float]), Some(batchSize))\n", + "\n", + "result.foreach(r => println(s\"${r._2} is ${r._1}\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5.0,2.0,5.0,10.0,5.0,2.0,5.0,10.0\n", + "8.0,3.0,2.0,1.0,5.0,2.0,5.0,10.0\n" + ] + } + ], + "source": [ + "val predictions = trainedModel.predict(evaluationSet)\n", + "val preLables = predictions.take(8).map(_.toTensor.max(1)._2.valueAt(1)).mkString(\",\")\n", + "val lables = evaluationSet.take(8).map(_.label.valueAt(1)).mkString(\",\")\n", + "println(preLables)\n", + "println(lables)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Draw the performance curve" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import vegas._\n", + "import vegas.render.HTMLRenderer._\n", + "import vegas.DSL._" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "val loss = trainSummary.readScalar(\"Loss\")\n", + "val top1 = valSummary.readScalar(\"Top1Accuracy\")\n", + "\n", + "val lossXY = loss.map(_ ._1).zip(loss.map(_ ._2)).toSeq\n", + "\n", + "Vegas(description = \"The Loss curve.\", width = 700.0, height = 300.0).\n", + " withXY(lossXY).\n", + " encodeX(\"x\", Quantitative, bin = Bin(maxbins = 500.0), title = \"Iteration\").\n", + " encodeY(\"y\", Quantitative, title = \"Loss\").\n", + " mark(Line).\n", + " show\n", + "\n", + "val top1XY = top1.map(_ ._1).zip(top1.map(_ ._2)).toSeq\n", + "Vegas(description = \"The Top1Accuracy curve.\", width = 700.0, height = 300.0).\n", + " withXY(top1XY).\n", + " encodeX(\"x\", Quantitative, bin = Bin(maxbins = 500.0), title = \"Iteration\").\n", + " encodeY(\"y\", Quantitative, bin = Bin(base = 0.9), title = \"Top1Accuracy\").\n", + " mark(Line).\n", + " show" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, the Spark should be stopped." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "sc.stop()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Apache Toree - Scala", + "language": "scala", + "name": "apache_toree_scala" + }, + "language_info": { + "file_extension": ".scala", + "name": "scala", + "version": "2.11.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/bigdl-tutorials-scala/cnn.ipynb b/notebooks/bigdl-tutorials-scala/cnn.ipynb new file mode 100644 index 0000000..798a432 --- /dev/null +++ b/notebooks/bigdl-tutorials-scala/cnn.ipynb @@ -0,0 +1,3059 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Handwritten Digit Classfication using Convolutional Neural Network" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this example, we are going to use the MNIST dataset to train a multi-layer feed foward neural network. MNIST is a simple computer vision dataset of handwritten digits. It has 60,000 training examles and 10,000 test examples. \"It is a good database for people who want to try learning techniques and pattern recognition methods on real-world data while spending minimal efforts on preprocessing and formatting.\" For more details, please checkout the website [MNIST](http://yann.lecun.com/exdb/mnist/)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import org.apache.log4j.{Level, Logger}\n", + "import org.apache.spark.SparkContext\n", + "\n", + "import com.intel.analytics.bigdl.utils._\n", + "import com.intel.analytics.bigdl.utils.{Engine, LoggerFilter, T, Table}\n", + "import com.intel.analytics.bigdl.dataset.DataSet\n", + "import com.intel.analytics.bigdl.dataset.image.{BytesToGreyImg, GreyImgNormalizer, GreyImgToBatch, GreyImgToSample}\n", + "import com.intel.analytics.bigdl.models.lenet.Utils._\n", + "import com.intel.analytics.bigdl.nn._\n", + "import com.intel.analytics.bigdl.optim._\n", + "import com.intel.analytics.bigdl.optim.{Adam, Top1Accuracy, Trigger}\n", + "import com.intel.analytics.bigdl.visualization.{TrainSummary, ValidationSummary}\n", + "import com.intel.analytics.bigdl.tensor.Tensor\n", + "import com.intel.analytics.bigdl.numeric.NumericFloat\n", + "\n", + "Engine.init" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Train the network" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* prepare training and validation samples" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import java.nio.ByteBuffer\n", + "import java.nio.file.{Files, Path, Paths}\n", + "\n", + "import com.intel.analytics.bigdl.dataset.ByteRecord\n", + "import com.intel.analytics.bigdl.utils.File\n", + "import scopt.OptionParser\n", + "\n", + "def load(featureFile: String, labelFile: String): Array[ByteRecord] = {\n", + " val featureBuffer = ByteBuffer.wrap(Files.readAllBytes(Paths.get(featureFile)))\n", + " val labelBuffer = ByteBuffer.wrap(Files.readAllBytes(Paths.get(labelFile)))\n", + " \n", + " val labelMagicNumber = labelBuffer.getInt()\n", + " require(labelMagicNumber == 2049)\n", + " val featureMagicNumber = featureBuffer.getInt()\n", + " require(featureMagicNumber == 2051)\n", + "\n", + " val labelCount = labelBuffer.getInt()\n", + " val featureCount = featureBuffer.getInt()\n", + " require(labelCount == featureCount)\n", + "\n", + " val rowNum = featureBuffer.getInt()\n", + " val colNum = featureBuffer.getInt()\n", + "\n", + " val result = new Array[ByteRecord](featureCount)\n", + " var i = 0\n", + " while (i < featureCount) {\n", + " val img = new Array[Byte]((rowNum * colNum))\n", + " var y = 0\n", + " while (y < rowNum) {\n", + " var x = 0\n", + " while (x < colNum) {\n", + " img(x + y * colNum) = featureBuffer.get()\n", + " x += 1\n", + " }\n", + " y += 1\n", + " }\n", + " result(i) = ByteRecord(img, labelBuffer.get().toFloat + 1.0f)\n", + " i += 1\n", + " }\n", + "\n", + " result\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then we need to set paths of data. Please edit paths if they are changed." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "val trainData = \"../../datasets/mnist/train-images-idx3-ubyte\"\n", + "val trainLabel = \"../../datasets/mnist/train-labels-idx1-ubyte\"\n", + "val validationData = \"../../datasets/mnist/t10k-images-idx3-ubyte\"\n", + "val validationLabel = \"../../datasets/mnist/t10k-labels-idx1-ubyte\"" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "//Parameters\n", + "val batchSize = 2048\n", + "val maxEpochs = 20\n", + "val learningRate = 0.2\n", + "val learningRateDecay = 0.0002\n", + "\n", + "//Network Parameters\n", + "val nClasses = 10 //MNIST total classes (0-9 digits)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "val trainSet = \n", + " DataSet.array(load(trainData, trainLabel), sc) -> BytesToGreyImg(28, 28) -> GreyImgNormalizer(trainMean, trainStd) -> GreyImgToBatch(batchSize)\n", + "val validationSet = \n", + " DataSet.array(load(validationData, validationLabel), sc) -> BytesToGreyImg(28, 28) -> GreyImgNormalizer(testMean, testStd) -> GreyImgToBatch(batchSize)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* create the LeNet-5 model" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Sequential[e8160a75]{\n", + " [input -> (1) -> (2) -> (3) -> (4) -> (5) -> (6) -> (7) -> (8) -> (9) -> (10) -> (11) -> (12) -> output]\n", + " (1): Reshape[df46fd98](1x28x28)\n", + " (2): SpatialConvolution[conv1](1 -> 6, 5 x 5, 1, 1, 0, 0)\n", + " (3): Tanh[3902fc5b]\n", + " (4): SpatialMaxPooling[pool1](2, 2, 2, 2, 0, 0)\n", + " (5): Tanh[ee0cc375]\n", + " (6): SpatialConvolution[conv2](6 -> 12, 5 x 5, 1, 1, 0, 0)\n", + " (7): SpatialMaxPooling[pool2](2, 2, 2, 2, 0, 0)\n", + " (8): Reshape[35b4f8df](192)\n", + " (9): Linear[fc1](192 -> 100)\n", + " (10): Tanh[df74b1]\n", + " (11): Linear[score](100 -> 10)\n", + " (12): LogSoftMax[35d95790]\n", + "}" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// Create a LeNet model\n", + "val model = Sequential()\n", + "model.add(Reshape(Array(1, 28, 28)))\n", + "model.add(SpatialConvolution(1, 6, 5, 5).setName(\"conv1\"))\n", + "model.add(Tanh())\n", + "model.add(SpatialMaxPooling(2, 2, 2, 2).setName(\"pool1\"))\n", + "model.add(Tanh())\n", + "model.add(SpatialConvolution(6, 12, 5, 5).setName(\"conv2\"))\n", + "model.add(SpatialMaxPooling(2, 2, 2, 2).setName(\"pool2\"))\n", + "model.add(Reshape(Array(12 * 4 * 4)))\n", + "model.add(Linear(12 * 4 * 4, 100).setName(\"fc1\"))\n", + "model.add(Tanh())\n", + "model.add(Linear(100, nClasses).setName(\"score\"))\n", + "model.add(LogSoftMax())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* configure optimizer" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "com.intel.analytics.bigdl.optim.DistriOptimizer@588f72cc" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "val optimizer = Optimizer(model = model, dataset = trainSet, criterion = CrossEntropyCriterion[Float]())\n", + "optimizer.setValidation(trigger = Trigger.everyEpoch, dataset = validationSet, vMethods = Array(new Top1Accuracy))\n", + "optimizer.setOptimMethod(new SGD(learningRate = learningRate, learningRateDecay = learningRateDecay))\n", + "optimizer.setEndWhen(Trigger.maxEpoch(maxEpochs))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following is to create training and validation summary." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "saving logs to lenet-20170925-025447" + ] + } + ], + "source": [ + "import java.text.SimpleDateFormat\n", + "import java.util.Calendar\n", + "val today = Calendar.getInstance\n", + "val formatDate = new SimpleDateFormat(\"yyyyMMdd-hhmmss\")\n", + "val name = \"lenet-\" + formatDate.format(today.getTime()).toString()\n", + "val trainSummary = TrainSummary(logDir=\"/tmp/bigdl_summaries\", appName=name)\n", + "trainSummary.setSummaryTrigger(\"Parameters\", Trigger.severalIteration(50))\n", + "val valSummary = ValidationSummary(logDir=\"/tmp/bigdl_summaries\", appName=name)\n", + "optimizer.setTrainSummary(trainSummary)\n", + "optimizer.setValidationSummary(valSummary)\n", + "printf(\"saving logs to %s\", name)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* Train the network. Wait some time till it finished.. Voila! You've got a trained model" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "can't find locality partition for partition 0 Partition locations are (ArrayBuffer(172.168.2.109)) Candidate partition locations are\n", + "(0,List()).\n", + "Optimization Done." + ] + } + ], + "source": [ + "// Boot training process\n", + "val trainedModel = optimizer.optimize()\n", + "print(\"Optimization Done.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Inspect the prediction results" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Top1Accuracy is Accuracy(correct: 9760, count: 10000, accuracy: 0.976)\n" + ] + } + ], + "source": [ + "val rddData = sc.parallelize(load(validationData, validationLabel), batchSize)\n", + "val transformer = BytesToGreyImg(28, 28) -> GreyImgNormalizer(testMean, testStd) -> GreyImgToSample()\n", + "val evaluationSet = transformer(rddData)\n", + " \n", + "val result = trainedModel.evaluate(evaluationSet, Array(new Top1Accuracy[Float]), Some(batchSize))\n", + "\n", + "result.foreach(r => println(s\"${r._2} is ${r._1}\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "8.0,3.0,2.0,1.0,5.0,2.0,5.0,10.0\n", + "8.0,3.0,2.0,1.0,5.0,2.0,5.0,10.0\n" + ] + } + ], + "source": [ + "val predictions = trainedModel.predict(evaluationSet)\n", + "val preLables = predictions.take(8).map(_.toTensor.max(1)._2.valueAt(1)).mkString(\",\")\n", + "val lables = evaluationSet.take(8).map(_.label.valueAt(1)).mkString(\",\")\n", + "println(preLables)\n", + "println(lables)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Draw the performance curve" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import vegas._\n", + "import vegas.render.HTMLRenderer._\n", + "import vegas.DSL._" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " " + ] + }, + "metadata": {}, + "output_type": "display_data", + "source": "user" + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " " + ] + }, + "metadata": {}, + "output_type": "display_data", + "source": "user" + } + ], + "source": [ + "val loss = trainSummary.readScalar(\"Loss\")\n", + "val top1 = valSummary.readScalar(\"Top1Accuracy\")\n", + "\n", + "val lossXY = loss.map(_ ._1).zip(loss.map(_ ._2)).toSeq\n", + "\n", + "Vegas(description = \"The Loss curve.\", width = 700.0, height = 300.0).\n", + " withXY(lossXY).\n", + " encodeX(\"x\", Quantitative, bin = Bin(maxbins = 500.0), title = \"Iteration\").\n", + " encodeY(\"y\", Quantitative, title = \"Loss\").\n", + " mark(Line).\n", + " show\n", + "\n", + "val top1XY = top1.map(_ ._1).zip(top1.map(_ ._2)).toSeq\n", + "Vegas(description = \"The Top1Accuracy curve.\", width = 700.0, height = 300.0).\n", + " withXY(top1XY).\n", + " encodeX(\"x\", Quantitative, bin = Bin(maxbins = 500.0), title = \"Iteration\").\n", + " encodeY(\"y\", Quantitative, bin = Bin(base = 0.9), title = \"Top1Accuracy\").\n", + " mark(Line).\n", + " show" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, the Spark should be stopped." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "sc.stop()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Apache Toree - Scala", + "language": "scala", + "name": "apache_toree_scala" + }, + "language_info": { + "file_extension": ".scala", + "name": "scala", + "version": "2.11.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/bigdl-tutorials-scala/deep_feed_forward_neural_network.ipynb b/notebooks/bigdl-tutorials-scala/deep_feed_forward_neural_network.ipynb new file mode 100644 index 0000000..b04d14a --- /dev/null +++ b/notebooks/bigdl-tutorials-scala/deep_feed_forward_neural_network.ipynb @@ -0,0 +1,2417 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Handwritten Digit Classfication using Deep Feed Forward Neural Network" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this example, we are going to use the MNIST dataset to train a multi-layer feed foward neural network. MNIST is a simple computer vision dataset of handwritten digits. It has 60,000 training examles and 10,000 test examples. \"It is a good database for people who want to try learning techniques and pattern recognition methods on real-world data while spending minimal efforts on preprocessing and formatting.\" For more details, please checkout the website [MNIST](http://yann.lecun.com/exdb/mnist/)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import org.apache.log4j.{Level, Logger}\n", + "import org.apache.spark.SparkContext\n", + "\n", + "import com.intel.analytics.bigdl.utils._\n", + "import com.intel.analytics.bigdl.utils.{Engine, LoggerFilter, T, Table}\n", + "import com.intel.analytics.bigdl.dataset.DataSet\n", + "import com.intel.analytics.bigdl.dataset.image.{BytesToGreyImg, GreyImgNormalizer, GreyImgToBatch, GreyImgToSample}\n", + "import com.intel.analytics.bigdl.models.lenet.Utils._\n", + "import com.intel.analytics.bigdl.nn._\n", + "import com.intel.analytics.bigdl.optim._\n", + "import com.intel.analytics.bigdl.optim.{Adam, Top1Accuracy, Trigger}\n", + "import com.intel.analytics.bigdl.visualization.{TrainSummary, ValidationSummary}\n", + "import com.intel.analytics.bigdl.tensor.Tensor\n", + "import com.intel.analytics.bigdl.numeric.NumericFloat\n", + "\n", + "Engine.init" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Load MNIST dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import java.nio.ByteBuffer\n", + "import java.nio.file.{Files, Path, Paths}\n", + "\n", + "import com.intel.analytics.bigdl.dataset.ByteRecord\n", + "import com.intel.analytics.bigdl.utils.File\n", + "import scopt.OptionParser\n", + "\n", + "def load(featureFile: String, labelFile: String): Array[ByteRecord] = {\n", + " val featureBuffer = ByteBuffer.wrap(Files.readAllBytes(Paths.get(featureFile)))\n", + " val labelBuffer = ByteBuffer.wrap(Files.readAllBytes(Paths.get(labelFile)))\n", + " \n", + " val labelMagicNumber = labelBuffer.getInt()\n", + " require(labelMagicNumber == 2049)\n", + " val featureMagicNumber = featureBuffer.getInt()\n", + " require(featureMagicNumber == 2051)\n", + "\n", + " val labelCount = labelBuffer.getInt()\n", + " val featureCount = featureBuffer.getInt()\n", + " require(labelCount == featureCount)\n", + "\n", + " val rowNum = featureBuffer.getInt()\n", + " val colNum = featureBuffer.getInt()\n", + "\n", + " val result = new Array[ByteRecord](featureCount)\n", + " var i = 0\n", + " while (i < featureCount) {\n", + " val img = new Array[Byte]((rowNum * colNum))\n", + " var y = 0\n", + " while (y < rowNum) {\n", + " var x = 0\n", + " while (x < colNum) {\n", + " img(x + y * colNum) = featureBuffer.get()\n", + " x += 1\n", + " }\n", + " y += 1\n", + " }\n", + " result(i) = ByteRecord(img, labelBuffer.get().toFloat + 1.0f)\n", + " i += 1\n", + " }\n", + "\n", + " result\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then we need to set paths of data. Please edit paths if they are changed." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "val trainData = \"../../datasets/mnist/train-images-idx3-ubyte\"\n", + "val trainLabel = \"../../datasets/mnist/train-labels-idx1-ubyte\"\n", + "val validationData = \"../../datasets/mnist/t10k-images-idx3-ubyte\"\n", + "val validationLabel = \"../../datasets/mnist/t10k-labels-idx1-ubyte\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Deep Feed Forward Neural Network Model Setup" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This time we will use the deep feed forward neural network to classify handwritten digits. You can checkout this blog to get a detailed understanding of the deep feed forward neural network in particular." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "//Parameters\n", + "val learningRate = 0.2\n", + "val batchSize = 2048\n", + "val maxEpochs = 15\n", + "val displayStep = 1\n", + "\n", + "//Network Parameters\n", + "val nInput = 784 //MNIST data input (img shape: 28*28)\n", + "val nHidden1 = 256 // 1st hidden layer num of features\n", + "val nHidden2 = 256 // 2nd hidden layer num of features\n", + "val nClasses = 10 //MNIST total classes (0-9 digits)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then the data set should be created and the model needs to be established." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "val trainSet = \n", + " DataSet.array(load(trainData, trainLabel), sc) -> BytesToGreyImg(28, 28) -> GreyImgNormalizer(trainMean, trainStd) -> GreyImgToBatch(batchSize)\n", + "val validationSet = \n", + " DataSet.array(load(validationData, validationLabel), sc) -> BytesToGreyImg(28, 28) -> GreyImgNormalizer(testMean, testStd) -> GreyImgToBatch(batchSize)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Sequential[6bd56665]{\n", + " [input -> (1) -> (2) -> (3) -> (4) -> (5) -> (6) -> (7) -> output]\n", + " (1): Reshape[db28b021](784)\n", + " (2): Linear[mlp_fc1](784 -> 256)\n", + " (3): ReLU[8cf55aa1](0.0, 0.0)\n", + " (4): Linear[mlp_fc2](256 -> 256)\n", + " (5): ReLU[53e45a6d](0.0, 0.0)\n", + " (6): Linear[mlp_fc3](256 -> 10)\n", + " (7): LogSoftMax[d5d9df87]\n", + "}" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "val model = Sequential()\n", + "model.add(Reshape(Array(28*28)))\n", + "model.add(Linear(nInput, nHidden1).setName(\"mlp_fc1\"))\n", + "model.add(ReLU())\n", + "// Hidden layer with ReLu activation\n", + "model.add(Linear(nHidden1, nHidden2).setName(\"mlp_fc2\"))\n", + "model.add(ReLU())\n", + "// output layer\n", + "model.add(Linear(nHidden2, nClasses).setName(\"mlp_fc3\"))\n", + "model.add(LogSoftMax())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Optimizer Setup" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "com.intel.analytics.bigdl.optim.DistriOptimizer@466152cb" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "val optimizer = Optimizer(model = model, dataset = trainSet, criterion = ClassNLLCriterion[Float]())\n", + "optimizer.setValidation(trigger = Trigger.everyEpoch, dataset = validationSet, vMethods = Array(new Top1Accuracy))\n", + "optimizer.setOptimMethod(new SGD(learningRate=learningRate))\n", + "optimizer.setEndWhen(Trigger.maxEpoch(maxEpochs))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following is to create training and validation summary." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "saving logs to multilayer_perceptron-20170922-012115" + ] + } + ], + "source": [ + "import java.text.SimpleDateFormat\n", + "import java.util.Calendar\n", + "val today = Calendar.getInstance\n", + "val formatDate = new SimpleDateFormat(\"yyyyMMdd-hhmmss\")\n", + "val name = \"multilayer_perceptron-\" + formatDate.format(today.getTime()).toString()\n", + "val trainSummary = TrainSummary(logDir=\"/tmp/bigdl_summaries\", appName=name)\n", + "trainSummary.setSummaryTrigger(\"Parameters\", Trigger.severalIteration(50))\n", + "val valSummary = ValidationSummary(logDir=\"/tmp/bigdl_summaries\", appName=name)\n", + "optimizer.setTrainSummary(trainSummary)\n", + "optimizer.setValidationSummary(valSummary)\n", + "printf(\"saving logs to %s\", name)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "can't find locality partition for partition 0 Partition locations are (ArrayBuffer(172.168.2.109)) Candidate partition locations are\n", + "(0,List()).\n", + "Optimization Done." + ] + } + ], + "source": [ + "// Boot training process\n", + "val trainedModel = optimizer.optimize()\n", + "print(\"Optimization Done.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Top1Accuracy is Accuracy(correct: 9670, count: 10000, accuracy: 0.967)\n" + ] + } + ], + "source": [ + "val rddData = sc.parallelize(load(validationData, validationLabel), batchSize)\n", + "val transformer = BytesToGreyImg(28, 28) -> GreyImgNormalizer(testMean, testStd) -> GreyImgToSample()\n", + "val evaluationSet = transformer(rddData)\n", + " \n", + "val result = trainedModel.evaluate(evaluationSet, Array(new Top1Accuracy[Float]), Some(batchSize))\n", + "\n", + "result.foreach(r => println(s\"${r._2} is ${r._1}\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "8.0,3.0,2.0,1.0,5.0,2.0,5.0,10.0\n", + "8.0,3.0,2.0,1.0,5.0,2.0,5.0,10.0\n" + ] + } + ], + "source": [ + "val predictions = trainedModel.predict(evaluationSet)\n", + "val preLables = predictions.take(8).map(_.toTensor.max(1)._2.valueAt(1)).mkString(\",\")\n", + "val lables = evaluationSet.take(8).map(_.label.valueAt(1)).mkString(\",\")\n", + "println(preLables)\n", + "println(lables)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Draw the performance curve" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import vegas._\n", + "import vegas.render.HTMLRenderer._\n", + "import vegas.DSL._" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "val loss = trainSummary.readScalar(\"Loss\")\n", + "val top1 = valSummary.readScalar(\"Top1Accuracy\")\n", + "\n", + "val lossXY = loss.map(_ ._1).zip(loss.map(_ ._2)).toSeq\n", + "\n", + "Vegas(description = \"The Loss curve.\", width = 700.0, height = 300.0).\n", + " withXY(lossXY).\n", + " encodeX(\"x\", Quantitative, bin = Bin(maxbins = 500.0), title = \"Iteration\").\n", + " encodeY(\"y\", Quantitative, title = \"Loss\").\n", + " mark(Line).\n", + " show\n", + "\n", + "val top1XY = top1.map(_ ._1).zip(top1.map(_ ._2)).toSeq\n", + "Vegas(description = \"The Top1Accuracy curve.\", width = 700.0, height = 300.0).\n", + " withXY(top1XY).\n", + " encodeX(\"x\", Quantitative, bin = Bin(maxbins = 500.0), title = \"Iteration\").\n", + " encodeY(\"y\", Quantitative, bin = Bin(base = 0.9), title = \"Top1Accuracy\").\n", + " mark(Line).\n", + " show" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, the Spark should be stopped." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "sc.stop()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Apache Toree - Scala", + "language": "scala", + "name": "apache_toree_scala" + }, + "language_info": { + "file_extension": ".scala", + "name": "scala", + "version": "2.11.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/bigdl-tutorials-scala/forward_and_backward.ipynb b/notebooks/bigdl-tutorials-scala/forward_and_backward.ipynb new file mode 100644 index 0000000..58f67f5 --- /dev/null +++ b/notebooks/bigdl-tutorials-scala/forward_and_backward.ipynb @@ -0,0 +1,192 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Forward and backward" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this section, we will unveil the basic mechanism of the computational process of BigDL using a simple example. In this example, we show that how to obtain the gradients with a single forward and backward pass for updating." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We first need to import the necessary modules and initialize the engine." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import org.apache.log4j.{Level, Logger}\n", + "import org.apache.spark.SparkContext\n", + "\n", + "import com.intel.analytics.bigdl.nn._\n", + "import com.intel.analytics.bigdl.utils.{Engine, LoggerFilter, T, Table}\n", + "import com.intel.analytics.bigdl.nn.{AbsCriterion}\n", + "import com.intel.analytics.bigdl.tensor._\n", + "import com.intel.analytics.bigdl.numeric.NumericFloat\n", + "\n", + "Engine.init" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then we create a simple linear regression which can be formulized as *y = Wx + b*， where *W = [w1,w2]* are weight parameters and *b* is the bias." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Weight and Bias:\n", + "0.3978135\t-0.2532979\t\n", + "[com.intel.analytics.bigdl.tensor.DenseTensor of size 1x2]\n", + "0.66053367\n", + "[com.intel.analytics.bigdl.tensor.DenseTensor of size 1]\n", + "GradWeight and gradBias:\n", + "0.0\t0.0\t\n", + "[com.intel.analytics.bigdl.tensor.DenseTensor of size 1x2]\n", + "0.0\n", + "[com.intel.analytics.bigdl.tensor.DenseTensor of size 1]\n" + ] + } + ], + "source": [ + "// the input data size is 2*1, the output size is 1*1\n", + "val linear = Linear(2, 1)\n", + "// print the randomly initialized parameters\n", + "val (param1, param2) = linear.parameters()\n", + "println(\"Weight and Bias:\")\n", + "param1.foreach(println)\n", + "println(\"GradWeight and gradBias:\")\n", + "param2.foreach(println)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1.564943" + ] + } + ], + "source": [ + "val input = Tensor(T(T(1f, -2f)))\n", + "// forward to output\n", + "val output = linear.updateOutput(input)\n", + "print(output.valueAt(1, 1))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then we backpropagate the error of the predicted output to the input." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loss: 1.564943\n", + "Weight and Bias:\n", + "0.3978135\t-0.2532979\t\n", + "[com.intel.analytics.bigdl.tensor.DenseTensor of size 1x2]\n", + "0.66053367\n", + "[com.intel.analytics.bigdl.tensor.DenseTensor of size 1]\n", + "GradWeight and gradBias:\n", + "0.0\t0.0\t\n", + "[com.intel.analytics.bigdl.tensor.DenseTensor of size 1x2]\n", + "0.0\n", + "[com.intel.analytics.bigdl.tensor.DenseTensor of size 1]\n" + ] + } + ], + "source": [ + "// mean absolute error\n", + "val mae = AbsCriterion()\n", + "val target = Tensor(1).fill(0)\n", + "\n", + "val loss = mae.updateOutput(output, target)\n", + "printf(\"loss: %s\\n\", loss.toString)\n", + " \n", + "val gradOutput = mae.updateGradInput(output, target)\n", + "linear.updateGradInput(input, gradOutput)\n", + "\n", + "val (param1, param2) = linear.parameters()\n", + "println(\"Weight and Bias:\")\n", + "param1.foreach(println)\n", + "println(\"GradWeight and gradBias:\")\n", + "param2.foreach(println)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, the Spark should be stopped." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "sc.stop()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "From above we can see that the backward pass has computed the gradient of the weights in respect to the loss. Therefore we can update the weights with the gradients using algorithms such as *stochastic gradient descent*. However in practice you **should** use *optimizer.optimize()* to circumvent the details." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Apache Toree - Scala", + "language": "scala", + "name": "apache_toree_scala" + }, + "language_info": { + "file_extension": ".scala", + "name": "scala", + "version": "2.11.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/bigdl-tutorials-scala/introduction_to_mnist.ipynb b/notebooks/bigdl-tutorials-scala/introduction_to_mnist.ipynb new file mode 100644 index 0000000..2d1e4f1 --- /dev/null +++ b/notebooks/bigdl-tutorials-scala/introduction_to_mnist.ipynb @@ -0,0 +1,201 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Introduction to the MNIST database" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the following tutorials, we are going to use the MNIST database of handwritten digits. MNIST is a simple computer vision dataset of handwritten digits. It has 60,000 training examles and 10,000 test examples. \"It is a good database for people who want to try learning techniques and pattern recognition methods on real-world data while spending minimal efforts on preprocessing and formatting.\" For more details of this database, please checkout the website [MNIST](http://yann.lecun.com/exdb/mnist/)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In BigDL, we need to write a function to download and read the MNIST data when using Scala." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import java.nio.ByteBuffer\n", + "import java.nio.file.{Files, Path, Paths}\n", + "\n", + "import com.intel.analytics.bigdl.dataset.ByteRecord\n", + "import com.intel.analytics.bigdl.utils.File\n", + "import scopt.OptionParser\n", + "\n", + "def load(featureFile: String, labelFile: String): Array[ByteRecord] = {\n", + " val featureBuffer = ByteBuffer.wrap(Files.readAllBytes(Paths.get(featureFile)))\n", + " val labelBuffer = ByteBuffer.wrap(Files.readAllBytes(Paths.get(labelFile)))\n", + " \n", + " val labelMagicNumber = labelBuffer.getInt()\n", + " require(labelMagicNumber == 2049)\n", + " val featureMagicNumber = featureBuffer.getInt()\n", + " require(featureMagicNumber == 2051)\n", + "\n", + " val labelCount = labelBuffer.getInt()\n", + " val featureCount = featureBuffer.getInt()\n", + " require(labelCount == featureCount)\n", + "\n", + " val rowNum = featureBuffer.getInt()\n", + " val colNum = featureBuffer.getInt()\n", + "\n", + " val result = new Array[ByteRecord](featureCount)\n", + " var i = 0\n", + " while (i < featureCount) {\n", + " val img = new Array[Byte]((rowNum * colNum))\n", + " var y = 0\n", + " while (y < rowNum) {\n", + " var x = 0\n", + " while (x < colNum) {\n", + " img(x + y * colNum) = featureBuffer.get()\n", + " x += 1\n", + " }\n", + " y += 1\n", + " }\n", + " result(i) = ByteRecord(img, labelBuffer.get().toFloat + 1.0f)\n", + " i += 1\n", + " }\n", + "\n", + " result\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First, we need to import the necessary packages and initialize the engine." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import org.apache.log4j.{Level, Logger}\n", + "import org.apache.spark.SparkContext\n", + "\n", + "import com.intel.analytics.bigdl.utils._\n", + "import com.intel.analytics.bigdl.dataset.DataSet\n", + "import com.intel.analytics.bigdl.dataset.image.{BytesToGreyImg, GreyImgNormalizer, GreyImgToBatch, GreyImgToSample}\n", + "import com.intel.analytics.bigdl.nn.{ClassNLLCriterion, Module}\n", + "import com.intel.analytics.bigdl.models.lenet.Utils._\n", + "import com.intel.analytics.bigdl.nn.{ClassNLLCriterion, Linear, LogSoftMax, Sequential, Reshape}\n", + "import com.intel.analytics.bigdl.numeric.NumericFloat\n", + "import com.intel.analytics.bigdl.optim.{SGD, Top1Accuracy}\n", + "import com.intel.analytics.bigdl.utils.{Engine, LoggerFilter, T, Table}\n", + "import com.intel.analytics.bigdl.tensor.Tensor\n", + "\n", + "Engine.init" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, the paths of training data and validation data should be set." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "val trainData = \"../../datasets/mnist/train-images-idx3-ubyte\"\n", + "val trainLabel = \"../../datasets/mnist/train-labels-idx1-ubyte\"\n", + "val validationData = \"../../datasets/mnist/t10k-images-idx3-ubyte\"\n", + "val validationLabel = \"../../datasets/mnist/t10k-labels-idx1-ubyte\"" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "Then, we need to define some parameters for loading the MINST data." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "//Parameters\n", + "val batchSize = 2048\n", + "val learningRate = 0.2\n", + "val maxEpochs = 15\n", + "\n", + "//Network Parameters\n", + "val nInput = 784 //MNIST data input (img shape: 28*28)\n", + "val nClasses = 10 //MNIST total classes (0-9 digits)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, we can use predefined function to load and serialize MNIST data. If you want to output the data, some modifications on the funtion should be applied." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "val trainSet = \n", + " DataSet.array(load(trainData, trainLabel), sc) -> BytesToGreyImg(28, 28) -> GreyImgNormalizer(trainMean, trainStd) -> GreyImgToBatch(batchSize)\n", + "val validationSet = \n", + " DataSet.array(load(validationData, validationLabel), sc) -> BytesToGreyImg(28, 28) -> GreyImgNormalizer(testMean, testStd) -> GreyImgToBatch(batchSize)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "sc.stop()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Apache Toree - Scala", + "language": "scala", + "name": "apache_toree_scala" + }, + "language_info": { + "file_extension": ".scala", + "name": "scala", + "version": "2.11.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/bigdl-tutorials-scala/linear_regression.ipynb b/notebooks/bigdl-tutorials-scala/linear_regression.ipynb new file mode 100644 index 0000000..4d31e47 --- /dev/null +++ b/notebooks/bigdl-tutorials-scala/linear_regression.ipynb @@ -0,0 +1,250 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Linear Regression" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this tutorial, we will introduce how to use BigDL to train to a simple linear regression model. The first thing we need to do it to import necessary packages and inilialize the engine." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import org.apache.log4j.{Level, Logger}\n", + "import org.apache.spark.SparkContext\n", + "\n", + "import com.intel.analytics.bigdl._\n", + "import com.intel.analytics.bigdl.utils.{Engine, LoggerFilter, T, Table}\n", + "import com.intel.analytics.bigdl.dataset.{DataSet, Sample}\n", + "import com.intel.analytics.bigdl.nn.{Sequential, Linear, MSECriterion}\n", + "import com.intel.analytics.bigdl.optim._\n", + "import com.intel.analytics.bigdl.models.lenet.Utils._\n", + "import com.intel.analytics.bigdl.optim.{SGD, Top1Accuracy}\n", + "import com.intel.analytics.bigdl.tensor._\n", + "import com.intel.analytics.bigdl.numeric.NumericFloat\n", + "\n", + "Engine.init" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then we randomly create datasets for training." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "val featuresDim = 2\n", + "val dataLen = 100\n", + "\n", + "def GetRandSample() = {\n", + " val features = Tensor(featuresDim).rand(0, 1)\n", + " val label = (0.4 + features.sum * 2).toFloat\n", + " val sample = Sample[Float](features, label)\n", + " sample\n", + "}\n", + "\n", + "val rddTrain = sc.parallelize(0 until dataLen).map(_ => GetRandSample())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "Then we specify the necessary parameters and construct a linear regression model using BigDL. Please notice that batch_size should be devided by the number of cores you use. In this example, it was set as 8 since there are 4 cores when running the example." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "// Parameters\n", + "val learningRate = 0.2\n", + "val trainingEpochs = 5\n", + "val batchSize = 4\n", + "val nInput = featuresDim\n", + "val nOutput = 1 \n", + "\n", + "def LinearRegression(nInput: Int, nOutput: Int) = {\n", + " // Initialize a sequential container\n", + " val model = Sequential()\n", + " // Add a linear layer\n", + " model.add(Linear(nInput, nOutput))\n", + " model\n", + "}\n", + "\n", + "val model = LinearRegression(nInput, nOutput)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here we construct the optimizer to optimize the linear regression problem. You can specific your own learning rate in $SGD()$ method, also, you can replace the $SGD()$ with other optimizer such like $Adam()$. Click [here](https://github.com/intel-analytics/BigDL/tree/master/spark/dl/src/main/scala/com/intel/analytics/bigdl/optim) to see more optimizer." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "com.intel.analytics.bigdl.optim.DistriOptimizer@2ec4ba5b" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "val optimizer = Optimizer(model = model, sampleRDD = rddTrain, criterion = MSECriterion[Float](), batchSize = batchSize)\n", + "optimizer.setOptimMethod(new SGD(learningRate=learningRate))\n", + "optimizer.setEndWhen(Trigger.maxEpoch(trainingEpochs))" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "// Start to train\n", + "val trainedModel = optimizer.optimize()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Predict result:\n", + "3.7649865,2.7541423,1.9586959,1.5578532,3.7649865\n" + ] + } + ], + "source": [ + "val predictResult = trainedModel.predict(rddTrain)\n", + "val p = predictResult.take(5).map(_.toTensor.valueAt(1)).mkString(\",\")\n", + "println(\"Predict result:\")\n", + "println(p)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To test the trained model, we construct a dataset for testing and print the result of _Mean Square Error_." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5.747768\n" + ] + } + ], + "source": [ + "val r = new scala.util.Random(100)\n", + "val totalLength = 10\n", + "val features = Tensor(totalLength, featuresDim).rand(0, 1)\n", + "var label = (0.4 + features.sum).toFloat\n", + "val prediction = sc.parallelize(0 until totalLength).map(r => Sample[Float](features(r + 1), label))\n", + "val predictResult = trainedModel.predict(prediction)\n", + "val p = predictResult.take(6).map(_.toTensor.valueAt(1))\n", + "val groundLabel = Tensor(T(\n", + " | T(-0.47596836f),\n", + " | T(-0.37598032f),\n", + " | T(-0.00492062f),\n", + " | T(-0.5906958f),\n", + " | T(-0.12307882f),\n", + " | T(-0.77907401f)))\n", + "\n", + "var mse = 0f\n", + "for (i <- 1 to 6) {\n", + " mse += (p(i - 1) - groundLabel(i).valueAt(1)) * (p(i - 1) - groundLabel(i).valueAt(1))\n", + "}\n", + "mse /= 6f\n", + "println(mse)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, we stop the Spark." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "sc.stop()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Apache Toree - Scala", + "language": "scala", + "name": "apache_toree_scala" + }, + "language_info": { + "file_extension": ".scala", + "name": "scala", + "version": "2.11.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/bigdl-tutorials-scala/logistic_regression.ipynb b/notebooks/bigdl-tutorials-scala/logistic_regression.ipynb new file mode 100644 index 0000000..c2a7075 --- /dev/null +++ b/notebooks/bigdl-tutorials-scala/logistic_regression.ipynb @@ -0,0 +1,321 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Logistic Regression" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this tutorial we will introduce how to build a logistic regression model using BigDL. We use *MNIST* data for experiments in this tutorial. For more information about MNIST, please refer to this [site](http://yann.lecun.com/exdb/mnist/). The first thing we need to do it to import necessary packages and inilialize the engine." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This part aims at preparing for loading MNIST data" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import java.nio.ByteBuffer\n", + "import java.nio.file.{Files, Path, Paths}\n", + "\n", + "import com.intel.analytics.bigdl.dataset.ByteRecord\n", + "import com.intel.analytics.bigdl.utils.File\n", + "import scopt.OptionParser\n", + "\n", + "def load(featureFile: String, labelFile: String): Array[ByteRecord] = {\n", + " val featureBuffer = ByteBuffer.wrap(Files.readAllBytes(Paths.get(featureFile)))\n", + " val labelBuffer = ByteBuffer.wrap(Files.readAllBytes(Paths.get(labelFile)))\n", + " \n", + " val labelMagicNumber = labelBuffer.getInt()\n", + " require(labelMagicNumber == 2049)\n", + " val featureMagicNumber = featureBuffer.getInt()\n", + " require(featureMagicNumber == 2051)\n", + "\n", + " val labelCount = labelBuffer.getInt()\n", + " val featureCount = featureBuffer.getInt()\n", + " require(labelCount == featureCount)\n", + "\n", + " val rowNum = featureBuffer.getInt()\n", + " val colNum = featureBuffer.getInt()\n", + "\n", + " val result = new Array[ByteRecord](featureCount)\n", + " var i = 0\n", + " while (i < featureCount) {\n", + " val img = new Array[Byte]((rowNum * colNum))\n", + " var y = 0\n", + " while (y < rowNum) {\n", + " var x = 0\n", + " while (x < colNum) {\n", + " img(x + y * colNum) = featureBuffer.get()\n", + " x += 1\n", + " }\n", + " y += 1\n", + " }\n", + " result(i) = ByteRecord(img, labelBuffer.get().toFloat + 1.0f)\n", + " i += 1\n", + " }\n", + "\n", + " result\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import org.apache.log4j.{Level, Logger}\n", + "import org.apache.spark.SparkContext\n", + "\n", + "import com.intel.analytics.bigdl._\n", + "import com.intel.analytics.bigdl.utils._\n", + "import com.intel.analytics.bigdl.dataset.DataSet\n", + "import com.intel.analytics.bigdl.dataset.image.{BytesToGreyImg, GreyImgNormalizer, GreyImgToBatch, GreyImgToSample}\n", + "import com.intel.analytics.bigdl.nn.{ClassNLLCriterion, Module}\n", + "import com.intel.analytics.bigdl.numeric.NumericFloat\n", + "import com.intel.analytics.bigdl.optim._\n", + "import com.intel.analytics.bigdl.utils.{Engine, LoggerFilter, T, Table}\n", + "import com.intel.analytics.bigdl.models.lenet.Utils._\n", + "import com.intel.analytics.bigdl.nn.{ClassNLLCriterion, Linear, LogSoftMax, Sequential, Reshape}\n", + "import com.intel.analytics.bigdl.optim.SGD\n", + "import com.intel.analytics.bigdl.optim.Top1Accuracy\n", + "import com.intel.analytics.bigdl.tensor._" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then we get and store MNIST for training and testing. You should edit the paths below according to your system settings." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "val trainData = \"../../datasets/mnist/train-images-idx3-ubyte\"\n", + "val trainLabel = \"../../datasets/mnist/train-labels-idx1-ubyte\"\n", + "val validationData = \"../../datasets/mnist/t10k-images-idx3-ubyte\"\n", + "val validationLabel = \"../../datasets/mnist/t10k-labels-idx1-ubyte\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "//Parameters\n", + "val batchSize = 2048\n", + "val learningRate = 0.2\n", + "val maxEpochs = 15\n", + "\n", + "//Network Parameters\n", + "val nInput = 784 //MNIST data input (img shape: 28*28)\n", + "val nClasses = 10 //MNIST total classes (0-9 digits)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "Engine.init" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "collapsed": true, + "scrolled": true + }, + "outputs": [], + "source": [ + "val trainSet = \n", + " DataSet.array(load(trainData, trainLabel), sc) -> BytesToGreyImg(28, 28) -> GreyImgNormalizer(trainMean, trainStd) -> GreyImgToBatch(batchSize)\n", + "val validationSet = \n", + " DataSet.array(load(validationData, validationLabel), sc) -> BytesToGreyImg(28, 28) -> GreyImgNormalizer(testMean, testStd) -> GreyImgToBatch(batchSize)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Sequential[9857d67b]{\n", + " [input -> (1) -> (2) -> (3) -> output]\n", + " (1): Reshape[3dec9b34](784)\n", + " (2): Linear[c5ea5213](784 -> 10)\n", + " (3): LogSoftMax[5b4a673]\n", + "}" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "val model = Sequential().add(Reshape(Array(28 * 28))).add(Linear(nInput, nClasses)).add(LogSoftMax())\n", + "model" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "com.intel.analytics.bigdl.optim.DistriOptimizer@7ed045fb" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "val optimizer = Optimizer(model = model, dataset = trainSet, criterion = ClassNLLCriterion[Float]())\n", + "optimizer.setValidation(trigger = Trigger.everyEpoch, dataset = validationSet, vMethods = Array(new Top1Accuracy[Float], new Top5Accuracy[Float], new Loss[Float]))\n", + "optimizer.setOptimMethod(new SGD(learningRate=learningRate))\n", + "optimizer.setEndWhen(Trigger.maxEpoch(maxEpochs))" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "can't find locality partition for partition 0 Partition locations are (ArrayBuffer(172.168.2.109)) Candidate partition locations are\n", + "(0,List()).\n" + ] + }, + { + "data": { + "text/plain": [ + "Sequential[9857d67b]{\n", + " [input -> (1) -> (2) -> (3) -> output]\n", + " (1): Reshape[3dec9b34](784)\n", + " (2): Linear[c5ea5213](784 -> 10)\n", + " (3): LogSoftMax[5b4a673]\n", + "}" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "val trainedModel = optimizer.optimize()\n", + "trainedModel" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Top1Accuracy is Accuracy(correct: 9223, count: 10000, accuracy: 0.9223)\n" + ] + } + ], + "source": [ + "val rddData = sc.parallelize(load(validationData, validationLabel), batchSize)\n", + "val transformer = BytesToGreyImg(28, 28) -> GreyImgNormalizer(testMean, testStd) -> GreyImgToSample()\n", + "val evaluationSet = transformer(rddData)\n", + " \n", + "val result = model.evaluate(evaluationSet, Array(new Top1Accuracy[Float]), Some(batchSize))\n", + "\n", + "result.foreach(r => println(s\"${r._2} is ${r._1}\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "8.0,3.0,2.0,1.0,7.0,2.0,5.0,10.0,7.0,1.0,1.0,7.0,10.0,1.0,4.0,6.0,10.0,8.0,4.0,6.0\n", + "8.0,3.0,2.0,1.0,5.0,2.0,5.0,10.0,6.0,10.0,1.0,7.0,10.0,1.0,2.0,6.0,10.0,8.0,4.0,5.0\n" + ] + } + ], + "source": [ + "val predictions = model.predict(evaluationSet)\n", + "val preLabels = predictions.take(20).map(_.toTensor.max(1)._2.valueAt(1)).mkString(\",\")\n", + "val labels = evaluationSet.take(20).map(_.label.valueAt(1)).mkString(\",\")\n", + "println(preLabels)\n", + "println(labels)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "sc.stop()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Apache Toree - Scala", + "language": "scala", + "name": "apache_toree_scala" + }, + "language_info": { + "file_extension": ".scala", + "name": "scala", + "version": "2.11.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/bigdl-tutorials-scala/lstm.ipynb b/notebooks/bigdl-tutorials-scala/lstm.ipynb new file mode 100644 index 0000000..e197869 --- /dev/null +++ b/notebooks/bigdl-tutorials-scala/lstm.ipynb @@ -0,0 +1,19301 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Handwritten Digit Classfication using LSTM" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this example, we are going to use the MNIST dataset to train a multi-layer feed foward neural network. MNIST is a simple computer vision dataset of handwritten digits. It has 60,000 training examles and 10,000 test examples. \"It is a good database for people who want to try learning techniques and pattern recognition methods on real-world data while spending minimal efforts on preprocessing and formatting.\" For more details, please checkout the website [MNIST](http://yann.lecun.com/exdb/mnist/)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import org.apache.log4j.{Level, Logger}\n", + "import org.apache.spark.SparkContext\n", + "\n", + "import com.intel.analytics.bigdl.utils._\n", + "import com.intel.analytics.bigdl.utils.{Engine, LoggerFilter, T, Table}\n", + "import com.intel.analytics.bigdl.dataset.DataSet\n", + "import com.intel.analytics.bigdl.dataset.image.{BytesToGreyImg, GreyImgNormalizer, GreyImgToBatch, GreyImgToSample}\n", + "import com.intel.analytics.bigdl.models.lenet.Utils._\n", + "import com.intel.analytics.bigdl.nn._\n", + "import com.intel.analytics.bigdl.optim._\n", + "import com.intel.analytics.bigdl.optim.{Adam, Top1Accuracy, Trigger}\n", + "import com.intel.analytics.bigdl.visualization.{TrainSummary, ValidationSummary}\n", + "import com.intel.analytics.bigdl.tensor.Tensor\n", + "import com.intel.analytics.bigdl.numeric.NumericFloat\n", + "\n", + "Engine.init" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Load MNIST dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import java.nio.ByteBuffer\n", + "import java.nio.file.{Files, Path, Paths}\n", + "\n", + "import com.intel.analytics.bigdl.dataset.ByteRecord\n", + "import com.intel.analytics.bigdl.utils.File\n", + "import scopt.OptionParser\n", + "\n", + "def load(featureFile: String, labelFile: String): Array[ByteRecord] = {\n", + " val featureBuffer = ByteBuffer.wrap(Files.readAllBytes(Paths.get(featureFile)))\n", + " val labelBuffer = ByteBuffer.wrap(Files.readAllBytes(Paths.get(labelFile)))\n", + " \n", + " val labelMagicNumber = labelBuffer.getInt()\n", + " require(labelMagicNumber == 2049)\n", + " val featureMagicNumber = featureBuffer.getInt()\n", + " require(featureMagicNumber == 2051)\n", + "\n", + " val labelCount = labelBuffer.getInt()\n", + " val featureCount = featureBuffer.getInt()\n", + " require(labelCount == featureCount)\n", + "\n", + " val rowNum = featureBuffer.getInt()\n", + " val colNum = featureBuffer.getInt()\n", + "\n", + " val result = new Array[ByteRecord](featureCount)\n", + " var i = 0\n", + " while (i < featureCount) {\n", + " val img = new Array[Byte]((rowNum * colNum))\n", + " var y = 0\n", + " while (y < rowNum) {\n", + " var x = 0\n", + " while (x < colNum) {\n", + " img(x + y * colNum) = featureBuffer.get()\n", + " x += 1\n", + " }\n", + " y += 1\n", + " }\n", + " result(i) = ByteRecord(img, labelBuffer.get().toFloat + 1.0f)\n", + " i += 1\n", + " }\n", + "\n", + " result\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then we need to set paths of data. Please edit paths if they are changed." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "val trainData = \"../../datasets/mnist/train-images-idx3-ubyte\"\n", + "val trainLabel = \"../../datasets/mnist/train-labels-idx1-ubyte\"\n", + "val validationData = \"../../datasets/mnist/t10k-images-idx3-ubyte\"\n", + "val validationLabel = \"../../datasets/mnist/t10k-labels-idx1-ubyte\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. LSTM Model Setup" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This time we will use another recurrent neural network called LSTM to classify handwritten digits. You can checkout this blog to get a detailed understanding of recurrent neural networks and LSTMs in particular." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "//Parameters\n", + "val batchSize = 64\n", + "val maxEpochs = 5\n", + "\n", + "//Network Parameters\n", + "val nInput = 28 //MNIST data input (img shape: 28*28)\n", + "val nHidden = 128 // hidden layer num of features\n", + "val nClasses = 10 //MNIST total classes (0-9 digits)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then the data set should be created and the model needs to be established." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "val trainSet = \n", + " DataSet.array(load(trainData, trainLabel), sc) -> BytesToGreyImg(28, 28) -> GreyImgNormalizer(trainMean, trainStd) -> GreyImgToBatch(batchSize)\n", + "val validationSet = \n", + " DataSet.array(load(validationData, validationLabel), sc) -> BytesToGreyImg(28, 28) -> GreyImgNormalizer(testMean, testStd) -> GreyImgToBatch(batchSize)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "val recurrent = Recurrent().add(LSTM(nInput, nHidden))\n", + "val rnnModel = Sequential().add(InferReshape(Array(-1, nInput), true)).add(recurrent).add(Select(2, -1)).add(Linear(nHidden, nClasses))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Optimizer Setup" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "com.intel.analytics.bigdl.optim.DistriOptimizer@442350b2" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "val optimizer = Optimizer(model = rnnModel, dataset = trainSet, criterion = CrossEntropyCriterion[Float]())\n", + "optimizer.setValidation(trigger = Trigger.everyEpoch, dataset = validationSet, vMethods = Array(new Top1Accuracy))\n", + "optimizer.setOptimMethod(new Adam())\n", + "optimizer.setEndWhen(Trigger.maxEpoch(maxEpochs))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following is to create training and validation summary." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "saving logs to rnn-20170930-085704" + ] + } + ], + "source": [ + "import java.text.SimpleDateFormat\n", + "import java.util.Calendar\n", + "val today = Calendar.getInstance\n", + "val formatDate = new SimpleDateFormat(\"yyyyMMdd-hhmmss\")\n", + "val name = \"rnn-\" + formatDate.format(today.getTime()).toString()\n", + "val trainSummary = TrainSummary(logDir=\"/tmp/bigdl_summaries\", appName=name)\n", + "trainSummary.setSummaryTrigger(\"Parameters\", Trigger.severalIteration(50))\n", + "val valSummary = ValidationSummary(logDir=\"/tmp/bigdl_summaries\", appName=name)\n", + "optimizer.setTrainSummary(trainSummary)\n", + "optimizer.setValidationSummary(valSummary)\n", + "printf(\"saving logs to %s\", name)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "can't find locality partition for partition 0 Partition locations are (ArrayBuffer(172.168.2.109)) Candidate partition locations are\n", + "(0,List()).\n", + "Optimization Done." + ] + } + ], + "source": [ + "// Boot training process\n", + "val trainedModel = optimizer.optimize()\n", + "print(\"Optimization Done.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Top1Accuracy is Accuracy(correct: 9827, count: 10000, accuracy: 0.9827)\n" + ] + } + ], + "source": [ + "val rddData = sc.parallelize(load(validationData, validationLabel), batchSize)\n", + "val transformer = BytesToGreyImg(28, 28) -> GreyImgNormalizer(testMean, testStd) -> GreyImgToSample()\n", + "val evaluationSet = transformer(rddData)\n", + " \n", + "val result = trainedModel.evaluate(evaluationSet, Array(new Top1Accuracy[Float]), Some(batchSize))\n", + "\n", + "result.foreach(r => println(s\"${r._2} is ${r._1}\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5.0,2.0,5.0,10.0,5.0,2.0,5.0,10.0\n", + "8.0,3.0,2.0,1.0,5.0,2.0,5.0,10.0\n" + ] + } + ], + "source": [ + "val predictions = trainedModel.predict(evaluationSet)\n", + "val preLables = predictions.take(8).map(_.toTensor.max(1)._2.valueAt(1)).mkString(\",\")\n", + "val lables = evaluationSet.take(8).map(_.label.valueAt(1)).mkString(\",\")\n", + "println(preLables)\n", + "println(lables)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import vegas._\n", + "import vegas.render.HTMLRenderer._\n", + "import vegas.DSL._" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "val loss = trainSummary.readScalar(\"Loss\")\n", + "val top1 = valSummary.readScalar(\"Top1Accuracy\")\n", + "\n", + "val lossXY = loss.map(_ ._1).zip(loss.map(_ ._2)).toSeq\n", + "\n", + "Vegas(description = \"The Loss curve.\", width = 700.0, height = 300.0).\n", + " withXY(lossXY).\n", + " encodeX(\"x\", Quantitative, bin = Bin(maxbins = 500.0), title = \"Iteration\").\n", + " encodeY(\"y\", Quantitative, title = \"Loss\").\n", + " mark(Line).\n", + " show\n", + "\n", + "val top1XY = top1.map(_ ._1).zip(top1.map(_ ._2)).toSeq\n", + "Vegas(description = \"The Top1Accuracy curve.\", width = 700.0, height = 300.0).\n", + " withXY(top1XY).\n", + " encodeX(\"x\", Quantitative, bin = Bin(maxbins = 500.0), title = \"Iteration\").\n", + " encodeY(\"y\", Quantitative, bin = Bin(base = 0.9), title = \"Top1Accuracy\").\n", + " mark(Line).\n", + " show" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, the Spark should be stopped." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "sc.stop()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Apache Toree - Scala", + "language": "scala", + "name": "apache_toree_scala" + }, + "language_info": { + "file_extension": ".scala", + "name": "scala", + "version": "2.11.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/bigdl-tutorials-scala/rnn.ipynb b/notebooks/bigdl-tutorials-scala/rnn.ipynb new file mode 100644 index 0000000..ca3ca6e --- /dev/null +++ b/notebooks/bigdl-tutorials-scala/rnn.ipynb @@ -0,0 +1,19407 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Handwritten Digit Classfication using Recurrent Neural Network" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this example, we are going to use the MNIST dataset to train a multi-layer feed foward neural network. MNIST is a simple computer vision dataset of handwritten digits. It has 60,000 training examles and 10,000 test examples. \"It is a good database for people who want to try learning techniques and pattern recognition methods on real-world data while spending minimal efforts on preprocessing and formatting.\" For more details, please checkout the website [MNIST](http://yann.lecun.com/exdb/mnist/)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import org.apache.log4j.{Level, Logger}\n", + "import org.apache.spark.SparkContext\n", + "\n", + "import com.intel.analytics.bigdl.utils._\n", + "import com.intel.analytics.bigdl.utils.{Engine, LoggerFilter, T, Table}\n", + "import com.intel.analytics.bigdl.dataset.DataSet\n", + "import com.intel.analytics.bigdl.dataset.image.{BytesToGreyImg, GreyImgNormalizer, GreyImgToBatch, GreyImgToSample}\n", + "import com.intel.analytics.bigdl.models.lenet.Utils._\n", + "import com.intel.analytics.bigdl.nn._\n", + "import com.intel.analytics.bigdl.optim._\n", + "import com.intel.analytics.bigdl.optim.{Adam, Top1Accuracy, Trigger}\n", + "import com.intel.analytics.bigdl.visualization.{TrainSummary, ValidationSummary}\n", + "import com.intel.analytics.bigdl.tensor.Tensor\n", + "import com.intel.analytics.bigdl.numeric.NumericFloat\n", + "\n", + "Engine.init" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Load MNIST dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import java.nio.ByteBuffer\n", + "import java.nio.file.{Files, Path, Paths}\n", + "\n", + "import com.intel.analytics.bigdl.dataset.ByteRecord\n", + "import com.intel.analytics.bigdl.utils.File\n", + "import scopt.OptionParser\n", + "\n", + "def load(featureFile: String, labelFile: String): Array[ByteRecord] = {\n", + " val featureBuffer = ByteBuffer.wrap(Files.readAllBytes(Paths.get(featureFile)))\n", + " val labelBuffer = ByteBuffer.wrap(Files.readAllBytes(Paths.get(labelFile)))\n", + " \n", + " val labelMagicNumber = labelBuffer.getInt()\n", + " require(labelMagicNumber == 2049)\n", + " val featureMagicNumber = featureBuffer.getInt()\n", + " require(featureMagicNumber == 2051)\n", + "\n", + " val labelCount = labelBuffer.getInt()\n", + " val featureCount = featureBuffer.getInt()\n", + " require(labelCount == featureCount)\n", + "\n", + " val rowNum = featureBuffer.getInt()\n", + " val colNum = featureBuffer.getInt()\n", + "\n", + " val result = new Array[ByteRecord](featureCount)\n", + " var i = 0\n", + " while (i < featureCount) {\n", + " val img = new Array[Byte]((rowNum * colNum))\n", + " var y = 0\n", + " while (y < rowNum) {\n", + " var x = 0\n", + " while (x < colNum) {\n", + " img(x + y * colNum) = featureBuffer.get()\n", + " x += 1\n", + " }\n", + " y += 1\n", + " }\n", + " result(i) = ByteRecord(img, labelBuffer.get().toFloat + 1.0f)\n", + " i += 1\n", + " }\n", + "\n", + " result\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then we need to set paths of data. Please edit paths if they are changed." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "val trainData = \"../../datasets/mnist/train-images-idx3-ubyte\"\n", + "val trainLabel = \"../../datasets/mnist/train-labels-idx1-ubyte\"\n", + "val validationData = \"../../datasets/mnist/t10k-images-idx3-ubyte\"\n", + "val validationLabel = \"../../datasets/mnist/t10k-labels-idx1-ubyte\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Recurent Neural Network Model Setup" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This time we will use a recurrent neural network (aka RNN) to classify handwritten digits. You can checkout this blog to get a detailed understanding of recurrent neural networks and LSTMs in particular." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "//Parameters\n", + "val batchSize = 64\n", + "val maxEpochs = 5\n", + "\n", + "//Network Parameters\n", + "val nInput = 28 //MNIST data input (img shape: 28*28)\n", + "val nHidden = 128 // hidden layer num of features\n", + "val nClasses = 10 //MNIST total classes (0-9 digits)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then the data set should be created and the model needs to be established." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "val trainSet = \n", + " DataSet.array(load(trainData, trainLabel), sc) -> BytesToGreyImg(28, 28) -> GreyImgNormalizer(trainMean, trainStd) -> GreyImgToBatch(batchSize)\n", + "val validationSet = \n", + " DataSet.array(load(validationData, validationLabel), sc) -> BytesToGreyImg(28, 28) -> GreyImgNormalizer(testMean, testStd) -> GreyImgToBatch(batchSize)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "val recurrent = Recurrent().add(RnnCell(nInput, nHidden, Tanh()))\n", + "val rnnModel = Sequential().add(InferReshape(Array(-1, nInput), true)).add(recurrent).add(Select(2, -1)).add(Linear(nHidden, nClasses))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Optimizer Setup" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "com.intel.analytics.bigdl.optim.DistriOptimizer@cecdb7c" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "val optimizer = Optimizer(model = rnnModel, dataset = trainSet, criterion = CrossEntropyCriterion[Float]())\n", + "optimizer.setValidation(trigger = Trigger.everyEpoch, dataset = validationSet, vMethods = Array(new Top1Accuracy))\n", + "optimizer.setOptimMethod(new Adam())\n", + "optimizer.setEndWhen(Trigger.maxEpoch(maxEpochs))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following is to create training and validation summary." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "saving logs to rnn-20170925-024042" + ] + } + ], + "source": [ + "import java.text.SimpleDateFormat\n", + "import java.util.Calendar\n", + "val today = Calendar.getInstance\n", + "val formatDate = new SimpleDateFormat(\"yyyyMMdd-hhmmss\")\n", + "val name = \"rnn-\" + formatDate.format(today.getTime()).toString()\n", + "val trainSummary = TrainSummary(logDir=\"/tmp/bigdl_summaries\", appName=name)\n", + "trainSummary.setSummaryTrigger(\"Parameters\", Trigger.severalIteration(50))\n", + "val valSummary = ValidationSummary(logDir=\"/tmp/bigdl_summaries\", appName=name)\n", + "optimizer.setTrainSummary(trainSummary)\n", + "optimizer.setValidationSummary(valSummary)\n", + "printf(\"saving logs to %s\", name)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "can't find locality partition for partition 0 Partition locations are (ArrayBuffer(172.168.2.109)) Candidate partition locations are\n", + "(0,List()).\n", + "Optimization Done." + ] + } + ], + "source": [ + "// Boot training process\n", + "val trainedModel = optimizer.optimize()\n", + "print(\"Optimization Done.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Top1Accuracy is Accuracy(correct: 9509, count: 10000, accuracy: 0.9509)\n" + ] + } + ], + "source": [ + "val rddData = sc.parallelize(load(validationData, validationLabel), batchSize)\n", + "val transformer = BytesToGreyImg(28, 28) -> GreyImgNormalizer(testMean, testStd) -> GreyImgToSample()\n", + "val evaluationSet = transformer(rddData)\n", + " \n", + "val result = trainedModel.evaluate(evaluationSet, Array(new Top1Accuracy[Float]), Some(batchSize))\n", + "\n", + "result.foreach(r => println(s\"${r._2} is ${r._1}\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5.0,2.0,5.0,10.0,5.0,2.0,5.0,10.0\n", + "8.0,3.0,2.0,1.0,5.0,2.0,5.0,10.0\n" + ] + } + ], + "source": [ + "val predictions = trainedModel.predict(evaluationSet)\n", + "val preLabels = predictions.take(8).map(_.toTensor.max(1)._2.valueAt(1)).mkString(\",\")\n", + "val labels = evaluationSet.take(8).map(_.label.valueAt(1)).mkString(\",\")\n", + "println(preLabels)\n", + "println(labels)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "val input = T.array(evaluationSet.take(8).map(_.feature))\n", + "val layer = JoinTable(2, 2)\n", + "val output = layer.forward(input)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "224\n", + "28\n" + ] + }, + { + "data": { + "text/plain": [ + "Name: Compile Error\n", + "Message: :80: error: not found: value ImageIO\n", + " ImageIO.write(img, \"PNG\", new File(\"outimg.png\"));\n", + " ^\n", + "StackTrace: " + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import java.awt.Color\n", + "import java.awt.image.BufferedImage\n", + "import java.io.File\n", + "import javax.imageio\n", + "\n", + "val imageSize = output.size()\n", + "val dim1 = imageSize(1)\n", + "val dim2 = imageSize(0)\n", + "println(dim1)\n", + "println(dim2)\n", + "val img = new BufferedImage(dim1, dim2, BufferedImage.TYPE_BYTE_GRAY)\n", + "for (i <- 0 until dim1)\n", + " for (j <- 0 until dim2) {\n", + " val value = (output.valueAt(j+1, i+1) + 0.5).toInt\n", + " img.setRGB(i, j, new Color(value, value, value).getRGB)\n", + " }\n", + "ImageIO.write(img, \"PNG\", new File(\"outimg.png\")); " + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%html\n", + "\n", + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Draw the performance curve" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "import vegas._\n", + "import vegas.render.HTMLRenderer._\n", + "import vegas.DSL._" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "val loss = trainSummary.readScalar(\"Loss\")\n", + "val lossXY = loss.map(_ ._1).zip(loss.map(_ ._2)).toSeq\n", + "Vegas(description = \"The Loss curve.\", width = 700.0, height = 300.0).\n", + " withXY(lossXY).\n", + " encodeX(\"x\", Quantitative, bin = Bin(maxbins = 500.0), title = \"Iteration\").\n", + " encodeY(\"y\", Quantitative, title = \"Loss\").\n", + " mark(Line).\n", + " show" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "val top1 = valSummary.readScalar(\"Top1Accuracy\")\n", + "val top1XY = top1.map(_ ._1).zip(top1.map(_ ._2)).toSeq\n", + "Vegas(description = \"The Top1Accuracy curve.\", width = 700.0, height = 300.0).\n", + " withXY(top1XY).\n", + " encodeX(\"x\", Quantitative, bin = Bin(maxbins = 500.0), title = \"Iteration\").\n", + " encodeY(\"y\", Quantitative, bin = Bin(base = 0.9), title = \"Top1Accuracy\").\n", + " mark(Line).\n", + " show" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, the Spark should be stopped." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "sc.stop()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Apache Toree - Scala", + "language": "scala", + "name": "apache_toree_scala" + }, + "language_info": { + "file_extension": ".scala", + "name": "scala", + "version": "2.11.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/bin/README.md b/notebooks/bin/README.md new file mode 100644 index 0000000..f69ff13 --- /dev/null +++ b/notebooks/bin/README.md @@ -0,0 +1,46 @@ +# Guide for Starting BigDL +This guide aims at establishing a complete environment for users, who want to use **BigDL**, from an empty dev-box. +Note that this guide only valids for linux system. + +## 1. Preparing BigDL Libraries +The first step is to download **BigDL** libraries. Before running the script `prepare.bigdl.sh`, users should specify the version of BigDL. +The default version is `0.2.0`. It you want to use other versions, you can modify the variable `BigDL_Version` +with the expected version number. + +The default downloading location is under `/opt/work`. If you want to download to other locations, you can change the corresponding parts +in the script. + +After the above specifications, users can run the following command to prepare BigDL libraries. +```bash +./prepare.bigdl.sh +``` + +## 2. Preparing Vegas +[Vegas](https://github.com/vegas-viz/Vegas) is a plotting library for **Scala**. If you do not need this visualization tool when +using the notebook, you can skip this step. + +The script `prepare.vegas.sh` needs users to specify the location of **Spark** by setting the variable `SPARK_HOME`. Then it will download +the dependent jars into the directory `JARS_HOME=$SPARK_HOME/jars`. To start this preparation, users only need the following command +```bash +./prepare.vegas.sh +``` +The current version of **Vegas** is 2.11 and the script only supports this version. + +## 3. Starting Toree Notebooks +After the above preparations, users can start the Jupyter notebooks with the core Toree. We have prepared the script +`start.bigdl.toree.sh` for users to start, but users should specify the following variables before using this script. +* `BIGDL_VERSION` : The version number of BigDL libraries. +* `BIGDL_HOME` : The location of BigDL libraries. +* `SPARK_HOME` : The location of **Spark**. +* `SPARK_MASTER` : The master server for **Spark**. +* `NOTEBOOK_DIR` : The directory of creating, writing and storing notebooks. +* `PORT` : The port number of starting **Toree** +* `SPARK_OPTS` : The options for starting **Spark**. +Then users only need the following command to start **Toree** +```bash +./start.bigdl.toree.sh +``` +If users want to use JupyterLab, please first install it according to [this instruction](https://github.com/jupyterlab/jupyterlab) +and then run the script `start.bigdl.toree.jupyterlab.sh` + +Have fun with **Toree** using **BigDL** ! diff --git a/notebooks/bin/prepare.bigdl.sh b/notebooks/bin/prepare.bigdl.sh new file mode 100755 index 0000000..9a1315c --- /dev/null +++ b/notebooks/bin/prepare.bigdl.sh @@ -0,0 +1,24 @@ +#!/bin/bash +BigDL_Version=0.2.0 + +mkdir -p /opt/work +cd /opt/work + +source /etc/profile + +if [ ! -d "BigDL-$BigDL_Version" ]; then + wget https://repo1.maven.org/maven2/com/intel/analytics/bigdl/dist-spark-2.1.1-scala-2.11.8-linux64/$BigDL_Version/dist-spark-2.1.1-scala-2.11.8-linux64-$BigDL_Version-dist.zip + unzip -d BigDL-$BigDL_Version dist-spark-2.1.1-scala-2.11.8-linux64-$BigDL_Version-dist.zip + rm dist-spark-2.1.1-scala-2.11.8-linux64-$BigDL_Version-dist.zip +fi + +if [ ! -d BigDL-Tutorials ]; then + git config --list + git clone https://github.com/intel-analytics/BigDL-Tutorials.git + git checkout master +else + cd BigDL-Tutorials + git reset --hard HEAD + git pull + cd .. +fi diff --git a/notebooks/bin/prepare.vegas.sh b/notebooks/bin/prepare.vegas.sh new file mode 100755 index 0000000..d070cda --- /dev/null +++ b/notebooks/bin/prepare.vegas.sh @@ -0,0 +1,168 @@ +#!/bin/bash +source /etc/profile + +export SPARK_HOME=/opt/work/spark-2.1.0-bin-hadoop2.7 +export JARS_HOME=${SPARK_HOME}/jars + +if [ ! -e ${JARS_HOME}/circe-parser_2.11-0.7.0.jar ] +then + wget https://repo1.maven.org/maven2/io/circe/circe-parser_2.11/0.7.0/circe-parser_2.11-0.7.0.jar -P ${JARS_HOME} +else + echo "circe-parser_2.11-0.7.0.jar already exists." +fi + +if [ ! -e ${JARS_HOME}/cats-macros_2.11-0.9.0.jar ] +then + wget https://repo1.maven.org/maven2/org/typelevel/cats-macros_2.11/0.9.0/cats-macros_2.11-0.9.0.jar -P ${JARS_HOME} +else + echo "cats-macros_2.11-0.9.0.jar already exists." +fi + +if [ ! -e ${JARS_HOME}/macro-compat_2.11-1.1.1.jar ] +then + wget https://repo1.maven.org/maven2/org/typelevel/macro-compat_2.11/1.1.1/macro-compat_2.11-1.1.1.jar -P ${JARS_HOME} +else + echo "macro-compat_2.11-1.1.1.jar already exists." +fi + +if [ ! -e ${JARS_HOME}/monocle-macro_2.11-1.1.0.jar ] +then + wget https://repo1.maven.org/maven2/com/github/julien-truffaut/monocle-macro_2.11/1.1.0/monocle-macro_2.11-1.1.0.jar -P ${JARS_HOME} +else + echo "monocle-macro_2.11-1.1.0.jar already exists." +fi + +if [ ! -e ${JARS_HOME}/monocle-core_2.11-1.1.0.jar ] +then + wget https://repo1.maven.org/maven2/com/github/julien-truffaut/monocle-core_2.11/1.1.0/monocle-core_2.11-1.1.0.jar -P ${JARS_HOME} +else + echo "monocle-core_2.11-1.1.0.jar already exists." +fi + +if [ ! -e ${JARS_HOME}/circe-numbers_2.11-0.7.0.jar ] +then + wget https://repo1.maven.org/maven2/io/circe/circe-numbers_2.11/0.7.0/circe-numbers_2.11-0.7.0.jar -P ${JARS_HOME} +else + echo "circe-numbers_2.11-0.7.0.jar already exists." +fi + +if [ ! -e ${JARS_HOME}/vegas_2.11-0.3.11.jar ] +then + wget https://repo1.maven.org/maven2/org/vegas-viz/vegas_2.11/0.3.11/vegas_2.11-0.3.11.jar -P ${JARS_HOME} +else + echo "vegas_2.11-0.3.11.jar already exists." +fi + +if [ ! -e ${JARS_HOME}/shapeless_2.11-2.3.2.jar ] +then + wget https://repo1.maven.org/maven2/com/chuusai/shapeless_2.11/2.3.2/shapeless_2.11-2.3.2.jar -P ${JARS_HOME} +else + echo "shapeless_2.11-2.3.2.jar already exists." +fi + +if [ ! -e ${JARS_HOME}/vega-lite-1.2.0.jar ] +then + wget https://repo1.maven.org/maven2/org/webjars/bower/vega-lite/1.2.0/vega-lite-1.2.0.jar -P ${JARS_HOME} +else + echo "vega-lite-1.2.0.jar already exists." +fi + +if [ ! -e ${JARS_HOME}/circe-core_2.11-0.7.0.jar ] +then + wget https://repo1.maven.org/maven2/io/circe/circe-core_2.11/0.7.0/circe-core_2.11-0.7.0.jar -P ${JARS_HOME} +else + echo "circe-core_2.11-0.7.0.jar already exists." +fi + +if [ ! -e ${JARS_HOME}/cats-kernel_2.11-0.9.0.jar ] +then + wget https://repo1.maven.org/maven2/org/typelevel/cats-kernel_2.11/0.9.0/cats-kernel_2.11-0.9.0.jar -P ${JARS_HOME} +else + echo "cats-kernel_2.11-0.9.0.jar already exists." +fi + +if [ ! -e ${JARS_HOME}/circe-jawn_2.11-0.7.0.jar ] +then + wget https://repo1.maven.org/maven2/io/circe/circe-jawn_2.11/0.7.0/circe-jawn_2.11-0.7.0.jar -P ${JARS_HOME} +else + echo "circe-jawn_2.11-0.7.0.jar already exists." +fi + +if [ ! -e ${JARS_HOME}/cats-core_2.11-0.9.0.jar ] +then + wget https://repo1.maven.org/maven2/org/typelevel/cats-core_2.11/0.9.0/cats-core_2.11-0.9.0.jar -P ${JARS_HOME} +else + echo "cats-core_2.11-0.9.0.jar already exists." +fi + +if [ ! -e ${JARS_HOME}/scala-parser-combinators_2.11-1.0.2.jar ] +then + wget https://repo1.maven.org/maven2/org/scala-lang/modules/scala-parser-combinators_2.11/1.0.2/scala-parser-combinators_2.11-1.0.2.jar -P ${JARS_HOME} +else + echo "scala-parser-combinators_2.11-1.0.2.jar already exists." +fi + +if [ ! -e ${JARS_HOME}/circe-generic_2.11-0.7.0.jar ] +then + wget https://repo1.maven.org/maven2/io/circe/circe-generic_2.11/0.7.0/circe-generic_2.11-0.7.0.jar -P ${JARS_HOME} +else + echo "circe-generic_2.11-0.7.0.jar already exists." +fi + +if [ ! -e ${JARS_HOME}/vegas-macros_2.11-0.3.11.jar ] +then + wget https://repo1.maven.org/maven2/org/vegas-viz/vegas-macros_2.11/0.3.11/vegas-macros_2.11-0.3.11.jar -P ${JARS_HOME} +else + echo "vegas-macros_2.11-0.3.11.jar already exists." +fi + +if [ ! -e ${JARS_HOME}/simulacrum_2.11-0.10.0.jar ] +then + wget https://repo1.maven.org/maven2/com/github/mpilquist/simulacrum_2.11/0.10.0/simulacrum_2.11-0.10.0.jar -P ${JARS_HOME} +else + echo "simulacrum_2.11-0.10.0.jar already exists." +fi + +if [ ! -e ${JARS_HOME}/jawn-parser_2.11-0.10.4.jar ] +then + wget https://repo1.maven.org/maven2/org/spire-math/jawn-parser_2.11/0.10.4/jawn-parser_2.11-0.10.4.jar -P ${JARS_HOME} +else + echo "jawn-parser_2.11-0.10.4.jar already exists." +fi + +if [ ! -e ${JARS_HOME}/machinist_2.11-0.6.1.jar ] +then + wget https://repo1.maven.org/maven2/org/typelevel/machinist_2.11/0.6.1/machinist_2.11-0.6.1.jar -P ${JARS_HOME} +else + echo "machinist_2.11-0.6.1.jar already exists." +fi + +if [ ! -e ${JARS_HOME}/scalafx_2.11-8.0.92-R10.jar ] +then + wget https://repo1.maven.org/maven2/org/scalafx/scalafx_2.11/8.0.92-R10/scalafx_2.11-8.0.92-R10.jar -P ${JARS_HOME} +else + echo "scalafx_2.11-8.0.92-R10.jar already exists." +fi + +if [ ! -e ${JARS_HOME}/scala-xml_2.11-1.0.2.jar ] +then + wget https://repo1.maven.org/maven2/org/scala-lang/modules/scala-xml_2.11/1.0.2/scala-xml_2.11-1.0.2.jar -P ${JARS_HOME} +else + echo "scala-xml_2.11-1.0.2.jar already exists." +fi + +if [ ! -e ${JARS_HOME}/scalaz-core_2.11-7.1.1.jar ] +then + wget https://repo1.maven.org/maven2/org/scalaz/scalaz-core_2.11/7.1.1/scalaz-core_2.11-7.1.1.jar -P ${JARS_HOME} +else + echo "scalaz-core_2.11-7.1.1.jar already exists." +fi + +if [ ! -e ${JARS_HOME}/vega-3.0.0-rc4.jar ] +then + wget https://repo1.maven.org/maven2/org/webjars/bower/vega/3.0.0-rc4/vega-3.0.0-rc4.jar -P ${JARS_HOME} +else + echo "vega-3.0.0-rc4.jar already exists." +fi + +echo "Vegas Preparation Done!" diff --git a/notebooks/bin/start.bigdl.pyspark.sh b/notebooks/bin/start.bigdl.pyspark.sh new file mode 100755 index 0000000..9d61b3f --- /dev/null +++ b/notebooks/bin/start.bigdl.pyspark.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +#setup paths +source /etc/profile + +export BIGDL_VERSION=0.2.0 +export BIGDL_HOME=/opt/work/BigDL-${BIGDL_VERSION} +export SPARK_HOME=/opt/work/spark-2.1.0-bin-hadoop2.7 +export SPARK_MASTER="local[4]" +export NOTEBOOK_DIR="../../notebooks" +export PORT="12345" + +export PYSPARK_DRIVER_PYTHON=jupyter +export PYSPARK_DRIVER_PYTHON_OPTS="notebook --notebook-dir=${NOTEBOOK_DIR} --ip=* --port=${PORT} --no-browser --NotebookApp.token='' --allow-root" +export BIGDL_JAR=${BIGDL_HOME}/lib/bigdl-SPARK_2.1-${BIGDL_VERSION}-jar-with-dependencies.jar +export BIGDL_PY_ZIP=${BIGDL_HOME}/lib/bigdl-${BIGDL_VERSION}-python-api.zip +export BIGDL_CONF=${BIGDL_HOME}/conf/spark-bigdl.conf + +rm -r BigDL-Tutorials/notebooks/neural_networks/metastore_db/ + +${SPARK_HOME}/bin/pyspark --master $SPARK_MASTER --driver-memory 4g --properties-file ${BIGDL_CONF} --py-files ${BIGDL_PY_ZIP} --jars ${BIGDL_JAR} --conf spark.driver.extraClassPath=${BIGDL_JAR} --conf spark.executor.extraClassPath=${BIGDL_JAR} --conf spark.sql.catalogImplementation='' diff --git a/notebooks/bin/start.bigdl.toree.jupyterlab.sh b/notebooks/bin/start.bigdl.toree.jupyterlab.sh new file mode 100755 index 0000000..191e3b5 --- /dev/null +++ b/notebooks/bin/start.bigdl.toree.jupyterlab.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +#setup paths +source /etc/profile + +export BIGDL_VERSION=0.2.0 +export BIGDL_HOME=/opt/work/BigDL-${BIGDL_VERSION} +export SPARK_HOME=/opt/work/spark-2.1.0-bin-hadoop2.7 + +export BIGDL_JAR=${BIGDL_HOME}/lib/bigdl-SPARK_2.1-${BIGDL_VERSION}-jar-with-dependencies.jar +export BIGDL_PY_ZIP=${BIGDL_HOME}/lib/bigdl-${BIGDL_VERSION}-python-api.zip +export BIGDL_CONF=${BIGDL_HOME}/conf/spark-bigdl.conf + +export NOTEBOOK_DIR="../../notebooks" +export PORT=12345 +export DISPLAY=:0.0 +rm -r BigDL-Tutorials/notebooks/neural_networks/metastore_db/ + +export SPARK_OPTS="--master local[4] --driver-memory 4g --properties-file ${BIGDL_CONF} --jars ${BIGDL_JAR} --conf spark.driver.extraClassPath=${BIGDL_JAR} --conf spark.executor.extraClassPath=${BIGDL_JAR} --driver-java-options='-Dhttp.proxyHost=child-prc.intel.com -Dhttp.proxyPort=913 -Dhttps.proxyHost=child-prc.intel.com -Dhttps.proxyPort=913'" + +jupyter toree install --interpreters=Scala,PySpark --spark_home=${SPARK_HOME} --spark_opts='${SPARK_OPTS}' +jupyter lab --notebook-dir=${NOTEBOOK_DIR} --ip=* --port=${PORT} --no-browser --NotebookApp.token='' --allow-root + diff --git a/notebooks/bin/start.bigdl.toree.sh b/notebooks/bin/start.bigdl.toree.sh new file mode 100755 index 0000000..72a7264 --- /dev/null +++ b/notebooks/bin/start.bigdl.toree.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +#setup paths +source /etc/profile + +export BIGDL_VERSION=0.2.0 +export BIGDL_HOME=/opt/work/BigDL-${BIGDL_VERSION} +export SPARK_HOME=/opt/work/spark-2.1.0-bin-hadoop2.7 + +export BIGDL_JAR=${BIGDL_HOME}/lib/bigdl-SPARK_2.1-${BIGDL_VERSION}-jar-with-dependencies.jar +export BIGDL_PY_ZIP=${BIGDL_HOME}/lib/bigdl-${BIGDL_VERSION}-python-api.zip +export BIGDL_CONF=${BIGDL_HOME}/conf/spark-bigdl.conf + +export NOTEBOOK_DIR="../../notebooks" +export PORT=12345 + +rm -r ${NOTEBOOK_DIR}/bigdl-tutorials-python/metastore_db/ +rm -r ${NOTEBOOK_DIR}/spark-warehouse/ +rm -r ${NOTEBOOK_DIR}/bigdl-tutorials-python/spark-warehouse/ +rm -r ${NOTEBOOK_DIR}/bigdl-tutorials-scala/spark-warehouse/ + +export SPARK_OPTS="--master local[4] --driver-memory 4g --properties-file ${BIGDL_CONF} --jars ${BIGDL_JAR} --conf spark.driver.extraClassPath=${BIGDL_JAR} --conf spark.executor.extraClassPath=${BIGDL_JAR} --driver-java-options='-Dhttp.proxyHost=child-prc.intel.com -Dhttp.proxyPort=913 -Dhttps.proxyHost=child-prc.intel.com -Dhttps.proxyPort=913'" + +jupyter toree install --interpreters=Scala,PySpark --spark_home=${SPARK_HOME} --spark_opts='${SPARK_OPTS}' +jupyter notebook --notebook-dir=${NOTEBOOK_DIR} --ip=* --port=${PORT} --no-browser --NotebookApp.token='' --allow-root + diff --git a/notebooks/bin/start.jupyter.sh b/notebooks/bin/start.jupyter.sh new file mode 100755 index 0000000..a7e5ef0 --- /dev/null +++ b/notebooks/bin/start.jupyter.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +export NOTEBOOK_DIR="../../notebooks" +export PORT=12345 + +jupyter notebook --notebook-dir=${NOTEBOOK_DIR} --ip=* --port=${PORT} --no-browser --allow-root --NotebookApp.token='' diff --git a/notebooks/bin/start.pyspark.sh b/notebooks/bin/start.pyspark.sh new file mode 100755 index 0000000..00e68fa --- /dev/null +++ b/notebooks/bin/start.pyspark.sh @@ -0,0 +1,8 @@ +#!/bin/bash +export JAVA_HOME=/opt/jdk +export PYSPARK_DRIVER_PYTHON=jupyter +export NOTEBOOK_DIR="../../notebooks" +export PORT=12345 +export PYSPARK_DRIVER_PYTHON_OPTS="notebook --notebook-dir=${NOTEBOOK_DIR} --ip=* --port=${PORT} --no-browser --allow-root --NotebookApp.token=''" + +/opt/work/spark-2.1.0-bin-hadoop2.7/bin/pyspark --master spark://Gondolin-Node-021:7077 --driver-memory 4g --executor-memory 200G #--num-executors 100 diff --git a/notebooks/bin/start.spark.toree.sh b/notebooks/bin/start.spark.toree.sh new file mode 100755 index 0000000..c5f7c66 --- /dev/null +++ b/notebooks/bin/start.spark.toree.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +#setup pathes +source /etc/profile + +export SPARK_HOME=/opt/work/spark-2.2.0-bin-hadoop2.7 +export SPARK_MASTER="spark://Gondolin-Node-054:7077" +export NOTEBOOK_DIR="../../notebooks" +export NOTEBOOK_PORT=12345 + +rm -r BigDL-Tutorials/notebooks/neural_networks/metastore_db/ + +jupyter toree install --spark_home=${SPARK_HOME} --spark_opts="--master ${SPARK_MASTER} --conf spark.ui.port=5050 --conf spark.executor.memory=200G" +jupyter notebook --notebook-dir=${NOTEBOOK_DIR} --ip=* --port=${NOTEBOOK_PORT} --no-browser --NotebookApp.token='' --allow-root