mxnet_custom_loss_function.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import mxnet as mx\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import log_loss\n",
    "from sklearn.metrics import roc_auc_score\n",
    "def mx_log_loss(actual, predicted):\n",
    "    return log_loss(actual, predicted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import logging\n",
    "logging.getLogger().setLevel(logging.DEBUG)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_boston\n",
    "boston = load_boston()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Regression (Boston House Prices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "((506, 13), (506,))\n"
     ]
    }
   ],
   "source": [
    "x = pd.DataFrame(boston.data, columns=boston.feature_names)\n",
    "y = boston.target\n",
    "print((x.shape, y.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "flg_train = np.random.choice([False, True], len(y), replace=True, p=[0.3, 0.7])\n",
    "flg_valid = np.logical_not(flg_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "batch_size = 100\n",
    "train_iter = mx.io.NDArrayIter(x[flg_train].values, y[flg_train],\n",
    "                               batch_size, shuffle=True,\n",
    "                               label_name='label')\n",
    "valid_iter = mx.io.NDArrayIter(x[flg_valid].values, y[flg_valid],\n",
    "                               batch_size,\n",
    "                               label_name='label')\n",
    "test_iter  = mx.io.NDArrayIter(x[flg_valid].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data  = mx.sym.Variable('data')\n",
    "label = mx.sym.Variable('label')\n",
    "fc    = mx.sym.FullyConnected(data, num_hidden=3)\n",
    "act   = mx.sym.Activation(fc, act_type='relu')\n",
    "fco   = mx.sym.FullyConnected(act, num_hidden=1)\n",
    "net1  = mx.sym.LinearRegressionOutput(fco, label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1. Build-in function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Start training with [cpu(0)]\n",
      "INFO:root:Epoch[0] Resetting Data Iterator\n",
      "INFO:root:Epoch[0] Time cost=0.017\n",
      "INFO:root:Epoch[0] Validation-mse=1395687.562500\n",
      "INFO:root:Epoch[1] Resetting Data Iterator\n",
      "INFO:root:Epoch[1] Time cost=0.016\n",
      "INFO:root:Epoch[1] Validation-mse=600322.187500\n",
      "INFO:root:Epoch[2] Resetting Data Iterator\n",
      "INFO:root:Epoch[2] Time cost=0.014\n",
      "INFO:root:Epoch[2] Validation-mse=258119.859375\n",
      "INFO:root:Epoch[3] Resetting Data Iterator\n",
      "INFO:root:Epoch[3] Time cost=0.037\n",
      "INFO:root:Epoch[3] Validation-mse=110928.750000\n",
      "INFO:root:Epoch[4] Resetting Data Iterator\n",
      "INFO:root:Epoch[4] Time cost=0.008\n",
      "INFO:root:Epoch[4] Validation-mse=47643.732422\n",
      "INFO:root:Epoch[5] Resetting Data Iterator\n",
      "INFO:root:Epoch[5] Time cost=0.007\n",
      "INFO:root:Epoch[5] Validation-mse=20451.433594\n",
      "INFO:root:Epoch[6] Resetting Data Iterator\n",
      "INFO:root:Epoch[6] Time cost=0.006\n",
      "INFO:root:Epoch[6] Validation-mse=8778.733398\n",
      "INFO:root:Epoch[7] Resetting Data Iterator\n",
      "INFO:root:Epoch[7] Time cost=0.006\n",
      "INFO:root:Epoch[7] Validation-mse=3775.464355\n",
      "INFO:root:Epoch[8] Resetting Data Iterator\n",
      "INFO:root:Epoch[8] Time cost=0.005\n",
      "INFO:root:Epoch[8] Validation-mse=1635.793518\n",
      "INFO:root:Epoch[9] Resetting Data Iterator\n",
      "INFO:root:Epoch[9] Time cost=0.005\n",
      "INFO:root:Epoch[9] Validation-mse=723.968018\n",
      "INFO:root:Epoch[10] Resetting Data Iterator\n",
      "INFO:root:Epoch[10] Time cost=0.005\n",
      "INFO:root:Epoch[10] Validation-mse=337.514572\n",
      "INFO:root:Epoch[11] Resetting Data Iterator\n",
      "INFO:root:Epoch[11] Time cost=0.005\n",
      "INFO:root:Epoch[11] Validation-mse=175.133366\n",
      "INFO:root:Epoch[12] Resetting Data Iterator\n",
      "INFO:root:Epoch[12] Time cost=0.019\n",
      "INFO:root:Epoch[12] Validation-mse=107.841146\n",
      "INFO:root:Epoch[13] Resetting Data Iterator\n",
      "INFO:root:Epoch[13] Time cost=0.007\n",
      "INFO:root:Epoch[13] Validation-mse=80.584860\n",
      "INFO:root:Epoch[14] Resetting Data Iterator\n",
      "INFO:root:Epoch[14] Time cost=0.007\n",
      "INFO:root:Epoch[14] Validation-mse=69.974375\n"
     ]
    }
   ],
   "source": [
    "model1 = mx.model.FeedForward(\n",
    "    symbol=net1,\n",
    "    num_epoch=15,\n",
    "    learning_rate=0.1\n",
    ")\n",
    "model1.fit(\n",
    "    X=train_iter,\n",
    "    eval_data=valid_iter,\n",
    "    eval_metric='MSE'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "82.7316956117\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: plot Pages: 1 -->\n",
       "<svg width=\"226pt\" height=\"442pt\"\n",
       " viewBox=\"0.00 0.00 225.72 442.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 438)\">\n",
       "<title>plot</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-438 221.717,-438 221.717,4 -4,4\"/>\n",
       "<!-- data -->\n",
       "<g id=\"node1\" class=\"node\"><title>data</title>\n",
       "<ellipse fill=\"#8dd3c7\" stroke=\"black\" cx=\"52.7173\" cy=\"-29\" rx=\"47\" ry=\"29\"/>\n",
       "<text text-anchor=\"middle\" x=\"52.7173\" y=\"-24.8\" font-family=\"Times,serif\" font-size=\"14.00\">data</text>\n",
       "</g>\n",
       "<!-- fullyconnected0 -->\n",
       "<g id=\"node2\" class=\"node\"><title>fullyconnected0</title>\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"105.653,-152 -0.218184,-152 -0.218184,-94 105.653,-94 105.653,-152\"/>\n",
       "<text text-anchor=\"middle\" x=\"52.7173\" y=\"-125.8\" font-family=\"Times,serif\" font-size=\"14.00\">FullyConnected</text>\n",
       "<text text-anchor=\"middle\" x=\"52.7173\" y=\"-111.8\" font-family=\"Times,serif\" font-size=\"14.00\">3</text>\n",
       "</g>\n",
       "<!-- fullyconnected0&#45;&gt;data -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>fullyconnected0&#45;&gt;data</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M52.7173,-83.7443C52.7173,-75.2043 52.7173,-66.2977 52.7173,-58.2479\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"52.7173,-93.8971 48.2174,-83.897 52.7173,-88.8971 52.7174,-83.8971 52.7174,-83.8971 52.7174,-83.8971 52.7173,-88.8971 57.2174,-83.8971 52.7173,-93.8971 52.7173,-93.8971\"/>\n",
       "</g>\n",
       "<!-- activation0 -->\n",
       "<g id=\"node3\" class=\"node\"><title>activation0</title>\n",
       "<polygon fill=\"#ffffb3\" stroke=\"black\" points=\"99.7173,-246 5.71729,-246 5.71729,-188 99.7173,-188 99.7173,-246\"/>\n",
       "<text text-anchor=\"middle\" x=\"52.7173\" y=\"-219.8\" font-family=\"Times,serif\" font-size=\"14.00\">Activation</text>\n",
       "<text text-anchor=\"middle\" x=\"52.7173\" y=\"-205.8\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</g>\n",
       "<!-- activation0&#45;&gt;fullyconnected0 -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>activation0&#45;&gt;fullyconnected0</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M52.7173,-177.744C52.7173,-169.204 52.7173,-160.298 52.7173,-152.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"52.7173,-187.897 48.2174,-177.897 52.7173,-182.897 52.7174,-177.897 52.7174,-177.897 52.7174,-177.897 52.7173,-182.897 57.2174,-177.897 52.7173,-187.897 52.7173,-187.897\"/>\n",
       "</g>\n",
       "<!-- fullyconnected1 -->\n",
       "<g id=\"node4\" class=\"node\"><title>fullyconnected1</title>\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"105.653,-340 -0.218184,-340 -0.218184,-282 105.653,-282 105.653,-340\"/>\n",
       "<text text-anchor=\"middle\" x=\"52.7173\" y=\"-313.8\" font-family=\"Times,serif\" font-size=\"14.00\">FullyConnected</text>\n",
       "<text text-anchor=\"middle\" x=\"52.7173\" y=\"-299.8\" font-family=\"Times,serif\" font-size=\"14.00\">1</text>\n",
       "</g>\n",
       "<!-- fullyconnected1&#45;&gt;activation0 -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>fullyconnected1&#45;&gt;activation0</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M52.7173,-271.744C52.7173,-263.204 52.7173,-254.298 52.7173,-246.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"52.7173,-281.897 48.2174,-271.897 52.7173,-276.897 52.7174,-271.897 52.7174,-271.897 52.7174,-271.897 52.7173,-276.897 57.2174,-271.897 52.7173,-281.897 52.7173,-281.897\"/>\n",
       "</g>\n",
       "<!-- label -->\n",
       "<g id=\"node5\" class=\"node\"><title>label</title>\n",
       "<ellipse fill=\"#8dd3c7\" stroke=\"black\" cx=\"170.717\" cy=\"-311\" rx=\"47\" ry=\"29\"/>\n",
       "<text text-anchor=\"middle\" x=\"170.717\" y=\"-306.8\" font-family=\"Times,serif\" font-size=\"14.00\">label</text>\n",
       "</g>\n",
       "<!-- linearregressionoutput0 -->\n",
       "<g id=\"node6\" class=\"node\"><title>linearregressionoutput0</title>\n",
       "<polygon fill=\"#fccde5\" stroke=\"black\" points=\"188.352,-434 35.0821,-434 35.0821,-376 188.352,-376 188.352,-434\"/>\n",
       "<text text-anchor=\"middle\" x=\"111.717\" y=\"-400.8\" font-family=\"Times,serif\" font-size=\"14.00\">LinearRegressionOutput</text>\n",
       "</g>\n",
       "<!-- linearregressionoutput0&#45;&gt;fullyconnected1 -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>linearregressionoutput0&#45;&gt;fullyconnected1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M88.2637,-367.428C82.4626,-358.383 76.3356,-348.828 70.8328,-340.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"93.6948,-375.897 84.5084,-369.909 90.9956,-371.688 88.2964,-367.479 88.2964,-367.479 88.2964,-367.479 90.9956,-371.688 92.0844,-365.05 93.6948,-375.897 93.6948,-375.897\"/>\n",
       "</g>\n",
       "<!-- linearregressionoutput0&#45;&gt;label -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>linearregressionoutput0&#45;&gt;label</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M135.293,-367.237C141.488,-357.578 148.046,-347.353 153.813,-338.36\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"129.74,-375.897 131.35,-365.05 132.439,-371.688 135.138,-367.479 135.138,-367.479 135.138,-367.479 132.439,-371.688 138.926,-369.909 129.74,-375.897 129.74,-375.897\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x1123e9e80>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred = model1.predict(test_iter)\n",
    "print(mean_squared_error(y[flg_valid], pred))\n",
    "mx.visualization.plot_network(net1, node_attrs={'fixedsize': 'false'})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2. Custom function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Start training with [cpu(0)]\n",
      "INFO:root:Epoch[0] Resetting Data Iterator\n",
      "INFO:root:Epoch[0] Time cost=0.011\n",
      "INFO:root:Epoch[0] Validation-mse=123389273633193984.000000\n",
      "INFO:root:Epoch[1] Resetting Data Iterator\n",
      "INFO:root:Epoch[1] Time cost=0.007\n",
      "INFO:root:Epoch[1] Validation-mse=3472040824668160.000000\n",
      "INFO:root:Epoch[2] Resetting Data Iterator\n",
      "INFO:root:Epoch[2] Time cost=0.007\n",
      "INFO:root:Epoch[2] Validation-mse=97658467254272.000000\n",
      "INFO:root:Epoch[3] Resetting Data Iterator\n",
      "INFO:root:Epoch[3] Time cost=0.007\n",
      "INFO:root:Epoch[3] Validation-mse=2744262066176.000000\n",
      "INFO:root:Epoch[4] Resetting Data Iterator\n",
      "INFO:root:Epoch[4] Time cost=0.006\n",
      "INFO:root:Epoch[4] Validation-mse=76978479104.000000\n",
      "INFO:root:Epoch[5] Resetting Data Iterator\n",
      "INFO:root:Epoch[5] Time cost=0.007\n",
      "INFO:root:Epoch[5] Validation-mse=2157003904.000000\n",
      "INFO:root:Epoch[6] Resetting Data Iterator\n",
      "INFO:root:Epoch[6] Time cost=0.007\n",
      "INFO:root:Epoch[6] Validation-mse=61591550.000000\n",
      "INFO:root:Epoch[7] Resetting Data Iterator\n",
      "INFO:root:Epoch[7] Time cost=0.006\n",
      "INFO:root:Epoch[7] Validation-mse=2112167.125000\n",
      "INFO:root:Epoch[8] Resetting Data Iterator\n",
      "INFO:root:Epoch[8] Time cost=0.006\n",
      "INFO:root:Epoch[8] Validation-mse=172325.460938\n",
      "INFO:root:Epoch[9] Resetting Data Iterator\n",
      "INFO:root:Epoch[9] Time cost=0.006\n",
      "INFO:root:Epoch[9] Validation-mse=50338.280273\n",
      "INFO:root:Epoch[10] Resetting Data Iterator\n",
      "INFO:root:Epoch[10] Time cost=0.007\n",
      "INFO:root:Epoch[10] Validation-mse=29807.058594\n",
      "INFO:root:Epoch[11] Resetting Data Iterator\n",
      "INFO:root:Epoch[11] Time cost=0.009\n",
      "INFO:root:Epoch[11] Validation-mse=24232.414062\n",
      "INFO:root:Epoch[12] Resetting Data Iterator\n",
      "INFO:root:Epoch[12] Time cost=0.009\n",
      "INFO:root:Epoch[12] Validation-mse=22382.235352\n",
      "INFO:root:Epoch[13] Resetting Data Iterator\n",
      "INFO:root:Epoch[13] Time cost=0.010\n",
      "INFO:root:Epoch[13] Validation-mse=21697.045898\n",
      "INFO:root:Epoch[14] Resetting Data Iterator\n",
      "INFO:root:Epoch[14] Time cost=0.007\n",
      "INFO:root:Epoch[14] Validation-mse=21428.656250\n"
     ]
    }
   ],
   "source": [
    "loss = mx.sym.square(\n",
    "    mx.sym.Reshape(fco, shape=(-1,)) - label\n",
    ")\n",
    "net2 = mx.sym.MakeLoss(loss)\n",
    "model2 = mx.model.FeedForward(\n",
    "    symbol=net2,\n",
    "    num_epoch=15,\n",
    "    learning_rate=0.1\n",
    ")\n",
    "model2.fit(\n",
    "    X=train_iter,\n",
    "    eval_data=valid_iter,\n",
    "    eval_metric='MSE'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "76.778488739\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: plot Pages: 1 -->\n",
       "<svg width=\"220pt\" height=\"724pt\"\n",
       " viewBox=\"0.00 0.00 219.72 724.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 720)\">\n",
       "<title>plot</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-720 215.717,-720 215.717,4 -4,4\"/>\n",
       "<!-- data -->\n",
       "<g id=\"node1\" class=\"node\"><title>data</title>\n",
       "<ellipse fill=\"#8dd3c7\" stroke=\"black\" cx=\"52.7173\" cy=\"-29\" rx=\"47\" ry=\"29\"/>\n",
       "<text text-anchor=\"middle\" x=\"52.7173\" y=\"-24.8\" font-family=\"Times,serif\" font-size=\"14.00\">data</text>\n",
       "</g>\n",
       "<!-- fullyconnected0 -->\n",
       "<g id=\"node2\" class=\"node\"><title>fullyconnected0</title>\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"105.653,-152 -0.218184,-152 -0.218184,-94 105.653,-94 105.653,-152\"/>\n",
       "<text text-anchor=\"middle\" x=\"52.7173\" y=\"-125.8\" font-family=\"Times,serif\" font-size=\"14.00\">FullyConnected</text>\n",
       "<text text-anchor=\"middle\" x=\"52.7173\" y=\"-111.8\" font-family=\"Times,serif\" font-size=\"14.00\">3</text>\n",
       "</g>\n",
       "<!-- fullyconnected0&#45;&gt;data -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>fullyconnected0&#45;&gt;data</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M52.7173,-83.7443C52.7173,-75.2043 52.7173,-66.2977 52.7173,-58.2479\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"52.7173,-93.8971 48.2174,-83.897 52.7173,-88.8971 52.7174,-83.8971 52.7174,-83.8971 52.7174,-83.8971 52.7173,-88.8971 57.2174,-83.8971 52.7173,-93.8971 52.7173,-93.8971\"/>\n",
       "</g>\n",
       "<!-- activation0 -->\n",
       "<g id=\"node3\" class=\"node\"><title>activation0</title>\n",
       "<polygon fill=\"#ffffb3\" stroke=\"black\" points=\"99.7173,-246 5.71729,-246 5.71729,-188 99.7173,-188 99.7173,-246\"/>\n",
       "<text text-anchor=\"middle\" x=\"52.7173\" y=\"-219.8\" font-family=\"Times,serif\" font-size=\"14.00\">Activation</text>\n",
       "<text text-anchor=\"middle\" x=\"52.7173\" y=\"-205.8\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</g>\n",
       "<!-- activation0&#45;&gt;fullyconnected0 -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>activation0&#45;&gt;fullyconnected0</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M52.7173,-177.744C52.7173,-169.204 52.7173,-160.298 52.7173,-152.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"52.7173,-187.897 48.2174,-177.897 52.7173,-182.897 52.7174,-177.897 52.7174,-177.897 52.7174,-177.897 52.7173,-182.897 57.2174,-177.897 52.7173,-187.897 52.7173,-187.897\"/>\n",
       "</g>\n",
       "<!-- fullyconnected1 -->\n",
       "<g id=\"node4\" class=\"node\"><title>fullyconnected1</title>\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"105.653,-340 -0.218184,-340 -0.218184,-282 105.653,-282 105.653,-340\"/>\n",
       "<text text-anchor=\"middle\" x=\"52.7173\" y=\"-313.8\" font-family=\"Times,serif\" font-size=\"14.00\">FullyConnected</text>\n",
       "<text text-anchor=\"middle\" x=\"52.7173\" y=\"-299.8\" font-family=\"Times,serif\" font-size=\"14.00\">1</text>\n",
       "</g>\n",
       "<!-- fullyconnected1&#45;&gt;activation0 -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>fullyconnected1&#45;&gt;activation0</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M52.7173,-271.744C52.7173,-263.204 52.7173,-254.298 52.7173,-246.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"52.7173,-281.897 48.2174,-271.897 52.7173,-276.897 52.7174,-271.897 52.7174,-271.897 52.7174,-271.897 52.7173,-276.897 57.2174,-271.897 52.7173,-281.897 52.7173,-281.897\"/>\n",
       "</g>\n",
       "<!-- reshape0 -->\n",
       "<g id=\"node5\" class=\"node\"><title>reshape0</title>\n",
       "<polygon fill=\"#fdb462\" stroke=\"black\" points=\"99.7173,-434 5.71729,-434 5.71729,-376 99.7173,-376 99.7173,-434\"/>\n",
       "<text text-anchor=\"middle\" x=\"52.7173\" y=\"-400.8\" font-family=\"Times,serif\" font-size=\"14.00\">Reshape</text>\n",
       "</g>\n",
       "<!-- reshape0&#45;&gt;fullyconnected1 -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>reshape0&#45;&gt;fullyconnected1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M52.7173,-365.744C52.7173,-357.204 52.7173,-348.298 52.7173,-340.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"52.7173,-375.897 48.2174,-365.897 52.7173,-370.897 52.7174,-365.897 52.7174,-365.897 52.7174,-365.897 52.7173,-370.897 57.2174,-365.897 52.7173,-375.897 52.7173,-375.897\"/>\n",
       "</g>\n",
       "<!-- label -->\n",
       "<g id=\"node6\" class=\"node\"><title>label</title>\n",
       "<ellipse fill=\"#8dd3c7\" stroke=\"black\" cx=\"164.717\" cy=\"-405\" rx=\"47\" ry=\"29\"/>\n",
       "<text text-anchor=\"middle\" x=\"164.717\" y=\"-400.8\" font-family=\"Times,serif\" font-size=\"14.00\">label</text>\n",
       "</g>\n",
       "<!-- _minus0 -->\n",
       "<g id=\"node7\" class=\"node\"><title>_minus0</title>\n",
       "<polygon fill=\"#fccde5\" stroke=\"black\" points=\"155.717,-528 61.7173,-528 61.7173,-470 155.717,-470 155.717,-528\"/>\n",
       "<text text-anchor=\"middle\" x=\"108.717\" y=\"-494.8\" font-family=\"Times,serif\" font-size=\"14.00\">_Minus</text>\n",
       "</g>\n",
       "<!-- _minus0&#45;&gt;reshape0 -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>_minus0&#45;&gt;reshape0</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M86.2859,-461.148C80.8303,-452.186 75.0808,-442.74 69.9116,-434.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"91.6112,-469.897 82.5677,-463.695 89.0114,-465.626 86.4116,-461.355 86.4116,-461.355 86.4116,-461.355 89.0114,-465.626 90.2555,-459.015 91.6112,-469.897 91.6112,-469.897\"/>\n",
       "</g>\n",
       "<!-- _minus0&#45;&gt;label -->\n",
       "<g id=\"edge6\" class=\"edge\"><title>_minus0&#45;&gt;label</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M131.095,-461.237C136.974,-451.578 143.198,-441.353 148.672,-432.36\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"125.823,-469.897 127.179,-459.015 128.423,-465.626 131.023,-461.355 131.023,-461.355 131.023,-461.355 128.423,-465.626 134.867,-463.695 125.823,-469.897 125.823,-469.897\"/>\n",
       "</g>\n",
       "<!-- square0 -->\n",
       "<g id=\"node8\" class=\"node\"><title>square0</title>\n",
       "<polygon fill=\"#fccde5\" stroke=\"black\" points=\"155.717,-622 61.7173,-622 61.7173,-564 155.717,-564 155.717,-622\"/>\n",
       "<text text-anchor=\"middle\" x=\"108.717\" y=\"-588.8\" font-family=\"Times,serif\" font-size=\"14.00\">square</text>\n",
       "</g>\n",
       "<!-- square0&#45;&gt;_minus0 -->\n",
       "<g id=\"edge7\" class=\"edge\"><title>square0&#45;&gt;_minus0</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M108.717,-553.744C108.717,-545.204 108.717,-536.298 108.717,-528.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"108.717,-563.897 104.217,-553.897 108.717,-558.897 108.717,-553.897 108.717,-553.897 108.717,-553.897 108.717,-558.897 113.217,-553.897 108.717,-563.897 108.717,-563.897\"/>\n",
       "</g>\n",
       "<!-- makeloss0 -->\n",
       "<g id=\"node9\" class=\"node\"><title>makeloss0</title>\n",
       "<polygon fill=\"#fccde5\" stroke=\"black\" points=\"155.717,-716 61.7173,-716 61.7173,-658 155.717,-658 155.717,-716\"/>\n",
       "<text text-anchor=\"middle\" x=\"108.717\" y=\"-682.8\" font-family=\"Times,serif\" font-size=\"14.00\">MakeLoss</text>\n",
       "</g>\n",
       "<!-- makeloss0&#45;&gt;square0 -->\n",
       "<g id=\"edge8\" class=\"edge\"><title>makeloss0&#45;&gt;square0</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M108.717,-647.744C108.717,-639.204 108.717,-630.298 108.717,-622.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"108.717,-657.897 104.217,-647.897 108.717,-652.897 108.717,-647.897 108.717,-647.897 108.717,-647.897 108.717,-652.897 113.217,-647.897 108.717,-657.897 108.717,-657.897\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x1147e04a8>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred = np.sqrt(model2.predict(test_iter))\n",
    "print(mean_squared_error(y[flg_valid], pred))\n",
    "mx.visualization.plot_network(net2, node_attrs={'fixedsize': 'false'})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Classification (Boston House Prices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "((506, 13), (506,))\n"
     ]
    }
   ],
   "source": [
    "x = pd.DataFrame(boston.data, columns=boston.feature_names)\n",
    "y = (boston.target >= boston.target.mean()).astype(np.int8)\n",
    "print((x.shape, y.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "flg_train = np.random.choice([False, True], len(y), replace=True, p=[0.3, 0.7])\n",
    "flg_valid = np.logical_not(flg_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size = 100\n",
    "train_iter = mx.io.NDArrayIter(x[flg_train].values, y[flg_train],\n",
    "                               batch_size, shuffle=True,\n",
    "                               label_name='sm_label')\n",
    "valid_iter = mx.io.NDArrayIter(x[flg_valid].values, y[flg_valid],\n",
    "                               batch_size,\n",
    "                               label_name='sm_label')\n",
    "test_iter  = mx.io.NDArrayIter(x[flg_valid].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data  = mx.sym.Variable('data')\n",
    "label = mx.sym.Variable('sm_label')\n",
    "fc    = mx.sym.FullyConnected(data, num_hidden=3)\n",
    "act   = mx.sym.Activation(fc, act_type='relu')\n",
    "fco   = mx.sym.FullyConnected(act, num_hidden=2)\n",
    "net1  = mx.sym.SoftmaxOutput(fco, name='sm')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1. Build-in function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Start training with [cpu(0)]\n",
      "INFO:root:Epoch[0] Resetting Data Iterator\n",
      "INFO:root:Epoch[0] Time cost=0.007\n",
      "INFO:root:Epoch[0] Validation-cross-entropy=0.694896\n",
      "INFO:root:Epoch[1] Resetting Data Iterator\n",
      "INFO:root:Epoch[1] Time cost=0.005\n",
      "INFO:root:Epoch[1] Validation-cross-entropy=0.691014\n",
      "INFO:root:Epoch[2] Resetting Data Iterator\n",
      "INFO:root:Epoch[2] Time cost=0.006\n",
      "INFO:root:Epoch[2] Validation-cross-entropy=0.688830\n",
      "INFO:root:Epoch[3] Resetting Data Iterator\n",
      "INFO:root:Epoch[3] Time cost=0.008\n",
      "INFO:root:Epoch[3] Validation-cross-entropy=0.687697\n",
      "INFO:root:Epoch[4] Resetting Data Iterator\n",
      "INFO:root:Epoch[4] Time cost=0.007\n",
      "INFO:root:Epoch[4] Validation-cross-entropy=0.687203\n",
      "INFO:root:Epoch[5] Resetting Data Iterator\n",
      "INFO:root:Epoch[5] Time cost=0.006\n",
      "INFO:root:Epoch[5] Validation-cross-entropy=0.687085\n",
      "INFO:root:Epoch[6] Resetting Data Iterator\n",
      "INFO:root:Epoch[6] Time cost=0.005\n",
      "INFO:root:Epoch[6] Validation-cross-entropy=0.687178\n",
      "INFO:root:Epoch[7] Resetting Data Iterator\n",
      "INFO:root:Epoch[7] Time cost=0.006\n",
      "INFO:root:Epoch[7] Validation-cross-entropy=0.687380\n",
      "INFO:root:Epoch[8] Resetting Data Iterator\n",
      "INFO:root:Epoch[8] Time cost=0.006\n",
      "INFO:root:Epoch[8] Validation-cross-entropy=0.687629\n",
      "INFO:root:Epoch[9] Resetting Data Iterator\n",
      "INFO:root:Epoch[9] Time cost=0.007\n",
      "INFO:root:Epoch[9] Validation-cross-entropy=0.687889\n",
      "INFO:root:Epoch[10] Resetting Data Iterator\n",
      "INFO:root:Epoch[10] Time cost=0.005\n",
      "INFO:root:Epoch[10] Validation-cross-entropy=0.688140\n",
      "INFO:root:Epoch[11] Resetting Data Iterator\n",
      "INFO:root:Epoch[11] Time cost=0.011\n",
      "INFO:root:Epoch[11] Validation-cross-entropy=0.688370\n",
      "INFO:root:Epoch[12] Resetting Data Iterator\n",
      "INFO:root:Epoch[12] Time cost=0.008\n",
      "INFO:root:Epoch[12] Validation-cross-entropy=0.688575\n",
      "INFO:root:Epoch[13] Resetting Data Iterator\n",
      "INFO:root:Epoch[13] Time cost=0.008\n",
      "INFO:root:Epoch[13] Validation-cross-entropy=0.688754\n",
      "INFO:root:Epoch[14] Resetting Data Iterator\n",
      "INFO:root:Epoch[14] Time cost=0.008\n",
      "INFO:root:Epoch[14] Validation-cross-entropy=0.688909\n"
     ]
    }
   ],
   "source": [
    "model1 = mx.model.FeedForward(\n",
    "    symbol=net1,\n",
    "    num_epoch=15,\n",
    "    learning_rate=0.1\n",
    ")\n",
    "model1.fit(\n",
    "    X=train_iter,\n",
    "    eval_data=valid_iter,\n",
    "    eval_metric='CE'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.687112347189\n",
      "0.5\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: plot Pages: 1 -->\n",
       "<svg width=\"226pt\" height=\"442pt\"\n",
       " viewBox=\"0.00 0.00 225.72 442.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 438)\">\n",
       "<title>plot</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-438 221.717,-438 221.717,4 -4,4\"/>\n",
       "<!-- data -->\n",
       "<g id=\"node1\" class=\"node\"><title>data</title>\n",
       "<ellipse fill=\"#8dd3c7\" stroke=\"black\" cx=\"52.7173\" cy=\"-29\" rx=\"47\" ry=\"29\"/>\n",
       "<text text-anchor=\"middle\" x=\"52.7173\" y=\"-24.8\" font-family=\"Times,serif\" font-size=\"14.00\">data</text>\n",
       "</g>\n",
       "<!-- fullyconnected2 -->\n",
       "<g id=\"node2\" class=\"node\"><title>fullyconnected2</title>\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"105.653,-152 -0.218184,-152 -0.218184,-94 105.653,-94 105.653,-152\"/>\n",
       "<text text-anchor=\"middle\" x=\"52.7173\" y=\"-125.8\" font-family=\"Times,serif\" font-size=\"14.00\">FullyConnected</text>\n",
       "<text text-anchor=\"middle\" x=\"52.7173\" y=\"-111.8\" font-family=\"Times,serif\" font-size=\"14.00\">3</text>\n",
       "</g>\n",
       "<!-- fullyconnected2&#45;&gt;data -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>fullyconnected2&#45;&gt;data</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M52.7173,-83.7443C52.7173,-75.2043 52.7173,-66.2977 52.7173,-58.2479\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"52.7173,-93.8971 48.2174,-83.897 52.7173,-88.8971 52.7174,-83.8971 52.7174,-83.8971 52.7174,-83.8971 52.7173,-88.8971 57.2174,-83.8971 52.7173,-93.8971 52.7173,-93.8971\"/>\n",
       "</g>\n",
       "<!-- activation1 -->\n",
       "<g id=\"node3\" class=\"node\"><title>activation1</title>\n",
       "<polygon fill=\"#ffffb3\" stroke=\"black\" points=\"99.7173,-246 5.71729,-246 5.71729,-188 99.7173,-188 99.7173,-246\"/>\n",
       "<text text-anchor=\"middle\" x=\"52.7173\" y=\"-219.8\" font-family=\"Times,serif\" font-size=\"14.00\">Activation</text>\n",
       "<text text-anchor=\"middle\" x=\"52.7173\" y=\"-205.8\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</g>\n",
       "<!-- activation1&#45;&gt;fullyconnected2 -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>activation1&#45;&gt;fullyconnected2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M52.7173,-177.744C52.7173,-169.204 52.7173,-160.298 52.7173,-152.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"52.7173,-187.897 48.2174,-177.897 52.7173,-182.897 52.7174,-177.897 52.7174,-177.897 52.7174,-177.897 52.7173,-182.897 57.2174,-177.897 52.7173,-187.897 52.7173,-187.897\"/>\n",
       "</g>\n",
       "<!-- fullyconnected3 -->\n",
       "<g id=\"node4\" class=\"node\"><title>fullyconnected3</title>\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"105.653,-340 -0.218184,-340 -0.218184,-282 105.653,-282 105.653,-340\"/>\n",
       "<text text-anchor=\"middle\" x=\"52.7173\" y=\"-313.8\" font-family=\"Times,serif\" font-size=\"14.00\">FullyConnected</text>\n",
       "<text text-anchor=\"middle\" x=\"52.7173\" y=\"-299.8\" font-family=\"Times,serif\" font-size=\"14.00\">2</text>\n",
       "</g>\n",
       "<!-- fullyconnected3&#45;&gt;activation1 -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>fullyconnected3&#45;&gt;activation1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M52.7173,-271.744C52.7173,-263.204 52.7173,-254.298 52.7173,-246.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"52.7173,-281.897 48.2174,-271.897 52.7173,-276.897 52.7174,-271.897 52.7174,-271.897 52.7174,-271.897 52.7173,-276.897 57.2174,-271.897 52.7173,-281.897 52.7173,-281.897\"/>\n",
       "</g>\n",
       "<!-- sm_label -->\n",
       "<g id=\"node5\" class=\"node\"><title>sm_label</title>\n",
       "<ellipse fill=\"#8dd3c7\" stroke=\"black\" cx=\"170.717\" cy=\"-311\" rx=\"47\" ry=\"29\"/>\n",
       "<text text-anchor=\"middle\" x=\"170.717\" y=\"-306.8\" font-family=\"Times,serif\" font-size=\"14.00\">sm_label</text>\n",
       "</g>\n",
       "<!-- sm -->\n",
       "<g id=\"node6\" class=\"node\"><title>sm</title>\n",
       "<polygon fill=\"#fccde5\" stroke=\"black\" points=\"163.049,-434 60.3857,-434 60.3857,-376 163.049,-376 163.049,-434\"/>\n",
       "<text text-anchor=\"middle\" x=\"111.717\" y=\"-400.8\" font-family=\"Times,serif\" font-size=\"14.00\">SoftmaxOutput</text>\n",
       "</g>\n",
       "<!-- sm&#45;&gt;fullyconnected3 -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>sm&#45;&gt;fullyconnected3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M88.2637,-367.428C82.4626,-358.383 76.3356,-348.828 70.8328,-340.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"93.6948,-375.897 84.5084,-369.909 90.9956,-371.688 88.2964,-367.479 88.2964,-367.479 88.2964,-367.479 90.9956,-371.688 92.0844,-365.05 93.6948,-375.897 93.6948,-375.897\"/>\n",
       "</g>\n",
       "<!-- sm&#45;&gt;sm_label -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>sm&#45;&gt;sm_label</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M135.293,-367.237C141.488,-357.578 148.046,-347.353 153.813,-338.36\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"129.74,-375.897 131.35,-365.05 132.439,-371.688 135.138,-367.479 135.138,-367.479 135.138,-367.479 132.439,-371.688 138.926,-369.909 129.74,-375.897 129.74,-375.897\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x1147e87f0>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred = model1.predict(test_iter)[:, 1]\n",
    "print(log_loss(y[flg_valid], pred))\n",
    "print(roc_auc_score(y[flg_valid], pred))\n",
    "mx.visualization.plot_network(net1, node_attrs={'fixedsize': 'false'})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2. Custom function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_iter = mx.io.NDArrayIter(x[flg_train].values, y[flg_train],\n",
    "                               batch_size, shuffle=True,\n",
    "                               label_name='label')\n",
    "valid_iter = mx.io.NDArrayIter(x[flg_valid].values, y[flg_valid],\n",
    "                               batch_size,\n",
    "                               label_name='label')\n",
    "test_iter  = mx.io.NDArrayIter(x[flg_valid].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data  = mx.sym.Variable('data')\n",
    "label = mx.sym.Variable('label')\n",
    "fc    = mx.sym.FullyConnected(data, num_hidden=3)\n",
    "act   = mx.sym.Activation(fc, act_type='relu')\n",
    "fco   = mx.sym.FullyConnected(act, num_hidden=1)\n",
    "p     = mx.sym.Activation(fco, act_type='sigmoid')\n",
    "eps   = 1e-6\n",
    "p     = mx.sym.minimum(mx.sym.maximum(p, eps), 1.0-eps)\n",
    "q     = 1.0 - p\n",
    "lp    = mx.sym.Reshape(mx.sym.log(p), shape=(-1,))\n",
    "lq    = mx.sym.Reshape(mx.sym.log(q), shape=(-1,))\n",
    "loss  = label * lp + (1.0 - label) * lq\n",
    "net2  = mx.sym.MakeLoss(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Start training with [cpu(0)]\n",
      "INFO:root:Epoch[0] Resetting Data Iterator\n",
      "INFO:root:Epoch[0] Time cost=0.005\n",
      "INFO:root:Epoch[0] Validation-mae=8.105288\n",
      "INFO:root:Epoch[1] Resetting Data Iterator\n",
      "INFO:root:Epoch[1] Time cost=0.005\n",
      "INFO:root:Epoch[1] Validation-mae=8.105288\n",
      "INFO:root:Epoch[2] Resetting Data Iterator\n",
      "INFO:root:Epoch[2] Time cost=0.010\n",
      "INFO:root:Epoch[2] Validation-mae=8.105288\n",
      "INFO:root:Epoch[3] Resetting Data Iterator\n",
      "INFO:root:Epoch[3] Time cost=0.008\n",
      "INFO:root:Epoch[3] Validation-mae=8.105288\n",
      "INFO:root:Epoch[4] Resetting Data Iterator\n",
      "INFO:root:Epoch[4] Time cost=0.007\n",
      "INFO:root:Epoch[4] Validation-mae=8.105288\n",
      "INFO:root:Epoch[5] Resetting Data Iterator\n",
      "INFO:root:Epoch[5] Time cost=0.007\n",
      "INFO:root:Epoch[5] Validation-mae=8.105288\n",
      "INFO:root:Epoch[6] Resetting Data Iterator\n",
      "INFO:root:Epoch[6] Time cost=0.008\n",
      "INFO:root:Epoch[6] Validation-mae=8.105288\n",
      "INFO:root:Epoch[7] Resetting Data Iterator\n",
      "INFO:root:Epoch[7] Time cost=0.007\n",
      "INFO:root:Epoch[7] Validation-mae=8.105288\n",
      "INFO:root:Epoch[8] Resetting Data Iterator\n",
      "INFO:root:Epoch[8] Time cost=0.007\n",
      "INFO:root:Epoch[8] Validation-mae=8.105288\n",
      "INFO:root:Epoch[9] Resetting Data Iterator\n",
      "INFO:root:Epoch[9] Time cost=0.007\n",
      "INFO:root:Epoch[9] Validation-mae=8.105288\n",
      "INFO:root:Epoch[10] Resetting Data Iterator\n",
      "INFO:root:Epoch[10] Time cost=0.006\n",
      "INFO:root:Epoch[10] Validation-mae=8.105288\n",
      "INFO:root:Epoch[11] Resetting Data Iterator\n",
      "INFO:root:Epoch[11] Time cost=0.007\n",
      "INFO:root:Epoch[11] Validation-mae=8.105288\n",
      "INFO:root:Epoch[12] Resetting Data Iterator\n",
      "INFO:root:Epoch[12] Time cost=0.007\n",
      "INFO:root:Epoch[12] Validation-mae=8.105288\n",
      "INFO:root:Epoch[13] Resetting Data Iterator\n",
      "INFO:root:Epoch[13] Time cost=0.006\n",
      "INFO:root:Epoch[13] Validation-mae=8.105288\n",
      "INFO:root:Epoch[14] Resetting Data Iterator\n",
      "INFO:root:Epoch[14] Time cost=0.006\n",
      "INFO:root:Epoch[14] Validation-mae=8.105288\n"
     ]
    }
   ],
   "source": [
    "model2 = mx.model.FeedForward(\n",
    "    symbol=net2,\n",
    "    num_epoch=15,\n",
    "    learning_rate=0.1\n",
    ")\n",
    "model2.fit(\n",
    "    X=train_iter,\n",
    "    eval_data=valid_iter,\n",
    "    eval_metric='MAE'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6.0696948799\n",
      "0.5\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: plot Pages: 1 -->\n",
       "<svg width=\"336pt\" height=\"1194pt\"\n",
       " viewBox=\"0.00 0.00 336.00 1194.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 1190)\">\n",
       "<title>plot</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-1190 332,-1190 332,4 -4,4\"/>\n",
       "<!-- label -->\n",
       "<g id=\"node1\" class=\"node\"><title>label</title>\n",
       "<ellipse fill=\"#8dd3c7\" stroke=\"black\" cx=\"162\" cy=\"-781\" rx=\"47\" ry=\"29\"/>\n",
       "<text text-anchor=\"middle\" x=\"162\" y=\"-776.8\" font-family=\"Times,serif\" font-size=\"14.00\">label</text>\n",
       "</g>\n",
       "<!-- data -->\n",
       "<g id=\"node2\" class=\"node\"><title>data</title>\n",
       "<ellipse fill=\"#8dd3c7\" stroke=\"black\" cx=\"176\" cy=\"-29\" rx=\"47\" ry=\"29\"/>\n",
       "<text text-anchor=\"middle\" x=\"176\" y=\"-24.8\" font-family=\"Times,serif\" font-size=\"14.00\">data</text>\n",
       "</g>\n",
       "<!-- fullyconnected4 -->\n",
       "<g id=\"node3\" class=\"node\"><title>fullyconnected4</title>\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"228.935,-152 123.065,-152 123.065,-94 228.935,-94 228.935,-152\"/>\n",
       "<text text-anchor=\"middle\" x=\"176\" y=\"-125.8\" font-family=\"Times,serif\" font-size=\"14.00\">FullyConnected</text>\n",
       "<text text-anchor=\"middle\" x=\"176\" y=\"-111.8\" font-family=\"Times,serif\" font-size=\"14.00\">3</text>\n",
       "</g>\n",
       "<!-- fullyconnected4&#45;&gt;data -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>fullyconnected4&#45;&gt;data</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M176,-83.7443C176,-75.2043 176,-66.2977 176,-58.2479\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"176,-93.8971 171.5,-83.897 176,-88.8971 176,-83.8971 176,-83.8971 176,-83.8971 176,-88.8971 180.5,-83.8971 176,-93.8971 176,-93.8971\"/>\n",
       "</g>\n",
       "<!-- activation2 -->\n",
       "<g id=\"node4\" class=\"node\"><title>activation2</title>\n",
       "<polygon fill=\"#ffffb3\" stroke=\"black\" points=\"223,-246 129,-246 129,-188 223,-188 223,-246\"/>\n",
       "<text text-anchor=\"middle\" x=\"176\" y=\"-219.8\" font-family=\"Times,serif\" font-size=\"14.00\">Activation</text>\n",
       "<text text-anchor=\"middle\" x=\"176\" y=\"-205.8\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</g>\n",
       "<!-- activation2&#45;&gt;fullyconnected4 -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>activation2&#45;&gt;fullyconnected4</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M176,-177.744C176,-169.204 176,-160.298 176,-152.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"176,-187.897 171.5,-177.897 176,-182.897 176,-177.897 176,-177.897 176,-177.897 176,-182.897 180.5,-177.897 176,-187.897 176,-187.897\"/>\n",
       "</g>\n",
       "<!-- fullyconnected5 -->\n",
       "<g id=\"node5\" class=\"node\"><title>fullyconnected5</title>\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"228.935,-340 123.065,-340 123.065,-282 228.935,-282 228.935,-340\"/>\n",
       "<text text-anchor=\"middle\" x=\"176\" y=\"-313.8\" font-family=\"Times,serif\" font-size=\"14.00\">FullyConnected</text>\n",
       "<text text-anchor=\"middle\" x=\"176\" y=\"-299.8\" font-family=\"Times,serif\" font-size=\"14.00\">1</text>\n",
       "</g>\n",
       "<!-- fullyconnected5&#45;&gt;activation2 -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>fullyconnected5&#45;&gt;activation2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M176,-271.744C176,-263.204 176,-254.298 176,-246.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"176,-281.897 171.5,-271.897 176,-276.897 176,-271.897 176,-271.897 176,-271.897 176,-276.897 180.5,-271.897 176,-281.897 176,-281.897\"/>\n",
       "</g>\n",
       "<!-- activation3 -->\n",
       "<g id=\"node6\" class=\"node\"><title>activation3</title>\n",
       "<polygon fill=\"#ffffb3\" stroke=\"black\" points=\"223,-434 129,-434 129,-376 223,-376 223,-434\"/>\n",
       "<text text-anchor=\"middle\" x=\"176\" y=\"-407.8\" font-family=\"Times,serif\" font-size=\"14.00\">Activation</text>\n",
       "<text text-anchor=\"middle\" x=\"176\" y=\"-393.8\" font-family=\"Times,serif\" font-size=\"14.00\">sigmoid</text>\n",
       "</g>\n",
       "<!-- activation3&#45;&gt;fullyconnected5 -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>activation3&#45;&gt;fullyconnected5</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M176,-365.744C176,-357.204 176,-348.298 176,-340.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"176,-375.897 171.5,-365.897 176,-370.897 176,-365.897 176,-365.897 176,-365.897 176,-370.897 180.5,-365.897 176,-375.897 176,-375.897\"/>\n",
       "</g>\n",
       "<!-- _maximumscalar0 -->\n",
       "<g id=\"node7\" class=\"node\"><title>_maximumscalar0</title>\n",
       "<polygon fill=\"#fccde5\" stroke=\"black\" points=\"234.311,-528 117.689,-528 117.689,-470 234.311,-470 234.311,-528\"/>\n",
       "<text text-anchor=\"middle\" x=\"176\" y=\"-494.8\" font-family=\"Times,serif\" font-size=\"14.00\">_MaximumScalar</text>\n",
       "</g>\n",
       "<!-- _maximumscalar0&#45;&gt;activation3 -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>_maximumscalar0&#45;&gt;activation3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M176,-459.744C176,-451.204 176,-442.298 176,-434.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"176,-469.897 171.5,-459.897 176,-464.897 176,-459.897 176,-459.897 176,-459.897 176,-464.897 180.5,-459.897 176,-469.897 176,-469.897\"/>\n",
       "</g>\n",
       "<!-- _minimumscalar0 -->\n",
       "<g id=\"node8\" class=\"node\"><title>_minimumscalar0</title>\n",
       "<polygon fill=\"#fccde5\" stroke=\"black\" points=\"232.986,-622 119.014,-622 119.014,-564 232.986,-564 232.986,-622\"/>\n",
       "<text text-anchor=\"middle\" x=\"176\" y=\"-588.8\" font-family=\"Times,serif\" font-size=\"14.00\">_MinimumScalar</text>\n",
       "</g>\n",
       "<!-- _minimumscalar0&#45;&gt;_maximumscalar0 -->\n",
       "<g id=\"edge6\" class=\"edge\"><title>_minimumscalar0&#45;&gt;_maximumscalar0</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M176,-553.744C176,-545.204 176,-536.298 176,-528.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"176,-563.897 171.5,-553.897 176,-558.897 176,-553.897 176,-553.897 176,-553.897 176,-558.897 180.5,-553.897 176,-563.897 176,-563.897\"/>\n",
       "</g>\n",
       "<!-- log0 -->\n",
       "<g id=\"node9\" class=\"node\"><title>log0</title>\n",
       "<polygon fill=\"#fccde5\" stroke=\"black\" points=\"151,-716 57,-716 57,-658 151,-658 151,-716\"/>\n",
       "<text text-anchor=\"middle\" x=\"104\" y=\"-682.8\" font-family=\"Times,serif\" font-size=\"14.00\">log</text>\n",
       "</g>\n",
       "<!-- log0&#45;&gt;_minimumscalar0 -->\n",
       "<g id=\"edge7\" class=\"edge\"><title>log0&#45;&gt;_minimumscalar0</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M132.184,-649.987C139.392,-640.777 147.039,-631.005 153.893,-622.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"125.994,-657.897 128.613,-647.249 129.075,-653.96 132.157,-650.022 132.157,-650.022 132.157,-650.022 129.075,-653.96 135.701,-652.795 125.994,-657.897 125.994,-657.897\"/>\n",
       "</g>\n",
       "<!-- reshape1 -->\n",
       "<g id=\"node10\" class=\"node\"><title>reshape1</title>\n",
       "<polygon fill=\"#fdb462\" stroke=\"black\" points=\"96,-810 2,-810 2,-752 96,-752 96,-810\"/>\n",
       "<text text-anchor=\"middle\" x=\"49\" y=\"-776.8\" font-family=\"Times,serif\" font-size=\"14.00\">Reshape</text>\n",
       "</g>\n",
       "<!-- reshape1&#45;&gt;log0 -->\n",
       "<g id=\"edge8\" class=\"edge\"><title>reshape1&#45;&gt;log0</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M71.0308,-743.148C76.389,-734.186 82.0359,-724.74 87.1127,-716.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"65.8007,-751.897 67.0695,-741.005 68.3663,-747.605 70.9319,-743.314 70.9319,-743.314 70.9319,-743.314 68.3663,-747.605 74.7944,-745.623 65.8007,-751.897 65.8007,-751.897\"/>\n",
       "</g>\n",
       "<!-- _mul0 -->\n",
       "<g id=\"node11\" class=\"node\"><title>_mul0</title>\n",
       "<polygon fill=\"#fccde5\" stroke=\"black\" points=\"94,-904 -7.10543e-15,-904 -7.10543e-15,-846 94,-846 94,-904\"/>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-870.8\" font-family=\"Times,serif\" font-size=\"14.00\">_Mul</text>\n",
       "</g>\n",
       "<!-- _mul0&#45;&gt;label -->\n",
       "<g id=\"edge9\" class=\"edge\"><title>_mul0&#45;&gt;label</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M90.1218,-839.503C104.805,-827.756 120.824,-814.941 134.006,-804.395\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"82.1287,-845.897 87.1262,-836.136 86.033,-842.774 89.9374,-839.65 89.9374,-839.65 89.9374,-839.65 86.033,-842.774 92.7485,-843.164 82.1287,-845.897 82.1287,-845.897\"/>\n",
       "</g>\n",
       "<!-- _mul0&#45;&gt;reshape1 -->\n",
       "<g id=\"edge10\" class=\"edge\"><title>_mul0&#45;&gt;reshape1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47.8316,-835.744C48.0173,-827.204 48.2109,-818.298 48.3859,-810.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47.6109,-845.897 43.3294,-835.802 47.7197,-840.898 47.8284,-835.899 47.8284,-835.899 47.8284,-835.899 47.7197,-840.898 52.3273,-835.997 47.6109,-845.897 47.6109,-845.897\"/>\n",
       "</g>\n",
       "<!-- _rminusscalar1 -->\n",
       "<g id=\"node12\" class=\"node\"><title>_rminusscalar1</title>\n",
       "<polygon fill=\"#fccde5\" stroke=\"black\" points=\"215.604,-904 112.396,-904 112.396,-846 215.604,-846 215.604,-904\"/>\n",
       "<text text-anchor=\"middle\" x=\"164\" y=\"-870.8\" font-family=\"Times,serif\" font-size=\"14.00\">_RMinusScalar</text>\n",
       "</g>\n",
       "<!-- _rminusscalar1&#45;&gt;label -->\n",
       "<g id=\"edge11\" class=\"edge\"><title>_rminusscalar1&#45;&gt;label</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M163.168,-835.744C162.983,-827.204 162.789,-818.298 162.614,-810.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"163.389,-845.897 158.673,-835.997 163.28,-840.898 163.172,-835.899 163.172,-835.899 163.172,-835.899 163.28,-840.898 167.671,-835.802 163.389,-845.897 163.389,-845.897\"/>\n",
       "</g>\n",
       "<!-- _rminusscalar0 -->\n",
       "<g id=\"node13\" class=\"node\"><title>_rminusscalar0</title>\n",
       "<polygon fill=\"#fccde5\" stroke=\"black\" points=\"300.604,-716 197.396,-716 197.396,-658 300.604,-658 300.604,-716\"/>\n",
       "<text text-anchor=\"middle\" x=\"249\" y=\"-682.8\" font-family=\"Times,serif\" font-size=\"14.00\">_RMinusScalar</text>\n",
       "</g>\n",
       "<!-- _rminusscalar0&#45;&gt;_minimumscalar0 -->\n",
       "<g id=\"edge12\" class=\"edge\"><title>_rminusscalar0&#45;&gt;_minimumscalar0</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M220.425,-649.987C213.116,-640.777 205.363,-631.005 198.414,-622.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"226.701,-657.897 216.96,-652.861 223.593,-653.98 220.485,-650.063 220.485,-650.063 220.485,-650.063 223.593,-653.98 224.01,-647.266 226.701,-657.897 226.701,-657.897\"/>\n",
       "</g>\n",
       "<!-- log1 -->\n",
       "<g id=\"node14\" class=\"node\"><title>log1</title>\n",
       "<polygon fill=\"#fccde5\" stroke=\"black\" points=\"324,-810 230,-810 230,-752 324,-752 324,-810\"/>\n",
       "<text text-anchor=\"middle\" x=\"277\" y=\"-776.8\" font-family=\"Times,serif\" font-size=\"14.00\">log</text>\n",
       "</g>\n",
       "<!-- log1&#45;&gt;_rminusscalar0 -->\n",
       "<g id=\"edge13\" class=\"edge\"><title>log1&#45;&gt;_rminusscalar0</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M265.528,-742.307C262.877,-733.596 260.101,-724.475 257.597,-716.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"268.447,-751.897 261.23,-743.641 266.991,-747.114 265.535,-742.33 265.535,-742.33 265.535,-742.33 266.991,-747.114 269.84,-741.02 268.447,-751.897 268.447,-751.897\"/>\n",
       "</g>\n",
       "<!-- reshape2 -->\n",
       "<g id=\"node15\" class=\"node\"><title>reshape2</title>\n",
       "<polygon fill=\"#fdb462\" stroke=\"black\" points=\"328,-904 234,-904 234,-846 328,-846 328,-904\"/>\n",
       "<text text-anchor=\"middle\" x=\"281\" y=\"-870.8\" font-family=\"Times,serif\" font-size=\"14.00\">Reshape</text>\n",
       "</g>\n",
       "<!-- reshape2&#45;&gt;log1 -->\n",
       "<g id=\"edge14\" class=\"edge\"><title>reshape2&#45;&gt;log1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M279.337,-835.744C278.965,-827.204 278.578,-818.298 278.228,-810.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"279.778,-845.897 274.848,-836.102 279.561,-840.902 279.344,-835.907 279.344,-835.907 279.344,-835.907 279.561,-840.902 283.839,-835.711 279.778,-845.897 279.778,-845.897\"/>\n",
       "</g>\n",
       "<!-- _mul1 -->\n",
       "<g id=\"node16\" class=\"node\"><title>_mul1</title>\n",
       "<polygon fill=\"#fccde5\" stroke=\"black\" points=\"211,-998 117,-998 117,-940 211,-940 211,-998\"/>\n",
       "<text text-anchor=\"middle\" x=\"164\" y=\"-964.8\" font-family=\"Times,serif\" font-size=\"14.00\">_Mul</text>\n",
       "</g>\n",
       "<!-- _mul1&#45;&gt;_rminusscalar1 -->\n",
       "<g id=\"edge15\" class=\"edge\"><title>_mul1&#45;&gt;_rminusscalar1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M164,-929.744C164,-921.204 164,-912.298 164,-904.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"164,-939.897 159.5,-929.897 164,-934.897 164,-929.897 164,-929.897 164,-929.897 164,-934.897 168.5,-929.897 164,-939.897 164,-939.897\"/>\n",
       "</g>\n",
       "<!-- _mul1&#45;&gt;reshape2 -->\n",
       "<g id=\"edge16\" class=\"edge\"><title>_mul1&#45;&gt;reshape2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M207.677,-933.655C220.001,-923.965 233.264,-913.536 245.076,-904.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"199.74,-939.897 204.819,-930.178 203.67,-936.806 207.6,-933.716 207.6,-933.716 207.6,-933.716 203.67,-936.806 210.382,-937.253 199.74,-939.897 199.74,-939.897\"/>\n",
       "</g>\n",
       "<!-- _plus0 -->\n",
       "<g id=\"node17\" class=\"node\"><title>_plus0</title>\n",
       "<polygon fill=\"#fccde5\" stroke=\"black\" points=\"173,-1092 79,-1092 79,-1034 173,-1034 173,-1092\"/>\n",
       "<text text-anchor=\"middle\" x=\"126\" y=\"-1058.8\" font-family=\"Times,serif\" font-size=\"14.00\">_Plus</text>\n",
       "</g>\n",
       "<!-- _plus0&#45;&gt;_mul0 -->\n",
       "<g id=\"edge17\" class=\"edge\"><title>_plus0&#45;&gt;_mul0</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M109.946,-1024.2C94.8476,-988.654 72.5864,-936.241 58.9816,-904.21\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"114.002,-1033.75 105.95,-1026.31 112.047,-1029.15 110.092,-1024.55 110.092,-1024.55 110.092,-1024.55 112.047,-1029.15 114.234,-1022.79 114.002,-1033.75 114.002,-1033.75\"/>\n",
       "</g>\n",
       "<!-- _plus0&#45;&gt;_mul1 -->\n",
       "<g id=\"edge18\" class=\"edge\"><title>_plus0&#45;&gt;_mul1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M141.453,-1024.59C145.086,-1015.79 148.898,-1006.56 152.332,-998.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"137.608,-1033.9 137.266,-1022.94 139.517,-1029.28 141.425,-1024.65 141.425,-1024.65 141.425,-1024.65 139.517,-1029.28 145.585,-1026.37 137.608,-1033.9 137.608,-1033.9\"/>\n",
       "</g>\n",
       "<!-- makeloss1 -->\n",
       "<g id=\"node18\" class=\"node\"><title>makeloss1</title>\n",
       "<polygon fill=\"#fccde5\" stroke=\"black\" points=\"173,-1186 79,-1186 79,-1128 173,-1128 173,-1186\"/>\n",
       "<text text-anchor=\"middle\" x=\"126\" y=\"-1152.8\" font-family=\"Times,serif\" font-size=\"14.00\">MakeLoss</text>\n",
       "</g>\n",
       "<!-- makeloss1&#45;&gt;_plus0 -->\n",
       "<g id=\"edge19\" class=\"edge\"><title>makeloss1&#45;&gt;_plus0</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M126,-1117.74C126,-1109.2 126,-1100.3 126,-1092.25\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"126,-1127.9 121.5,-1117.9 126,-1122.9 126,-1117.9 126,-1117.9 126,-1117.9 126,-1122.9 130.5,-1117.9 126,-1127.9 126,-1127.9\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x1147f6ba8>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred = np.exp(model2.predict(test_iter))\n",
    "print(log_loss(y[flg_valid], pred))\n",
    "print(roc_auc_score(y[flg_valid], pred))\n",
    "mx.visualization.plot_network(net2, node_attrs={'fixedsize': 'false'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}