mxnet_plot_network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import mxnet as mx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data = mx.sym.Variable('data')\n",
    "net  = data\n",
    "pavg = mx.sym.Pooling(net, pool_type='avg', kernel=(8,8))\n",
    "pmax = mx.sym.Pooling(net, pool_type='max', kernel=(8,8))\n",
    "pavg = mx.sym.Flatten(pavg)\n",
    "pmax = mx.sym.Flatten(pmax)\n",
    "pmax = mx.sym.Activation(pmax, act_type='relu')\n",
    "net  = pavg + pmax\n",
    "out  = mx.sym.SoftmaxOutput(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: plot Pages: 1 -->\n",
       "<svg width=\"214pt\" height=\"536pt\"\n",
       " viewBox=\"0.00 0.00 214.00 536.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 532)\">\n",
       "<title>plot</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-532 210,-532 210,4 -4,4\"/>\n",
       "<!-- data -->\n",
       "<g id=\"node1\" class=\"node\"><title>data</title>\n",
       "<polygon fill=\"#8dd3c7\" stroke=\"black\" points=\"150,-58 56,-58 56,-0 150,-0 150,-58\"/>\n",
       "<text text-anchor=\"middle\" x=\"103\" y=\"-24.8\" font-family=\"Times,serif\" font-size=\"14.00\">data</text>\n",
       "</g>\n",
       "<!-- pooling0 -->\n",
       "<g id=\"node2\" class=\"node\"><title>pooling0</title>\n",
       "<polygon fill=\"#80b1d3\" stroke=\"black\" points=\"94,-152 -7.10543e-15,-152 -7.10543e-15,-94 94,-94 94,-152\"/>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-125.8\" font-family=\"Times,serif\" font-size=\"14.00\">Pooling</text>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-111.8\" font-family=\"Times,serif\" font-size=\"14.00\">avg, 8x8/1</text>\n",
       "</g>\n",
       "<!-- pooling0&#45;&gt;data -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>pooling0&#45;&gt;data</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M69.4314,-85.1484C74.887,-76.1857 80.6365,-66.74 85.8057,-58.2479\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"64.1061,-93.8971 65.4617,-83.0153 66.7059,-89.6261 69.3056,-85.3551 69.3056,-85.3551 69.3056,-85.3551 66.7059,-89.6261 73.1495,-87.6949 64.1061,-93.8971 64.1061,-93.8971\"/>\n",
       "</g>\n",
       "<!-- flatten0 -->\n",
       "<g id=\"node3\" class=\"node\"><title>flatten0</title>\n",
       "<polygon fill=\"#fdb462\" stroke=\"black\" points=\"94,-246 -7.10543e-15,-246 -7.10543e-15,-188 94,-188 94,-246\"/>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-212.8\" font-family=\"Times,serif\" font-size=\"14.00\">Flatten</text>\n",
       "</g>\n",
       "<!-- flatten0&#45;&gt;pooling0 -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>flatten0&#45;&gt;pooling0</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-177.744C47,-169.204 47,-160.298 47,-152.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-187.897 42.5001,-177.897 47,-182.897 47.0001,-177.897 47.0001,-177.897 47.0001,-177.897 47,-182.897 51.5001,-177.897 47,-187.897 47,-187.897\"/>\n",
       "</g>\n",
       "<!-- pooling1 -->\n",
       "<g id=\"node4\" class=\"node\"><title>pooling1</title>\n",
       "<polygon fill=\"#80b1d3\" stroke=\"black\" points=\"206,-152 112,-152 112,-94 206,-94 206,-152\"/>\n",
       "<text text-anchor=\"middle\" x=\"159\" y=\"-125.8\" font-family=\"Times,serif\" font-size=\"14.00\">Pooling</text>\n",
       "<text text-anchor=\"middle\" x=\"159\" y=\"-111.8\" font-family=\"Times,serif\" font-size=\"14.00\">max, 8x8/1</text>\n",
       "</g>\n",
       "<!-- pooling1&#45;&gt;data -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>pooling1&#45;&gt;data</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M136.569,-85.1484C131.113,-76.1857 125.363,-66.74 120.194,-58.2479\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"141.894,-93.8971 132.85,-87.6949 139.294,-89.6261 136.694,-85.3551 136.694,-85.3551 136.694,-85.3551 139.294,-89.6261 140.538,-83.0153 141.894,-93.8971 141.894,-93.8971\"/>\n",
       "</g>\n",
       "<!-- flatten1 -->\n",
       "<g id=\"node5\" class=\"node\"><title>flatten1</title>\n",
       "<polygon fill=\"#fdb462\" stroke=\"black\" points=\"206,-246 112,-246 112,-188 206,-188 206,-246\"/>\n",
       "<text text-anchor=\"middle\" x=\"159\" y=\"-212.8\" font-family=\"Times,serif\" font-size=\"14.00\">Flatten</text>\n",
       "</g>\n",
       "<!-- flatten1&#45;&gt;pooling1 -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>flatten1&#45;&gt;pooling1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M159,-177.744C159,-169.204 159,-160.298 159,-152.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"159,-187.897 154.5,-177.897 159,-182.897 159,-177.897 159,-177.897 159,-177.897 159,-182.897 163.5,-177.897 159,-187.897 159,-187.897\"/>\n",
       "</g>\n",
       "<!-- activation0 -->\n",
       "<g id=\"node6\" class=\"node\"><title>activation0</title>\n",
       "<polygon fill=\"#ffffb3\" stroke=\"black\" points=\"196,-340 102,-340 102,-282 196,-282 196,-340\"/>\n",
       "<text text-anchor=\"middle\" x=\"149\" y=\"-313.8\" font-family=\"Times,serif\" font-size=\"14.00\">Activation</text>\n",
       "<text text-anchor=\"middle\" x=\"149\" y=\"-299.8\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</g>\n",
       "<!-- activation0&#45;&gt;flatten1 -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>activation0&#45;&gt;flatten1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M153.158,-271.744C154.086,-263.204 155.055,-254.298 155.93,-246.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"152.055,-281.897 148.662,-271.469 152.595,-276.926 153.135,-271.956 153.135,-271.956 153.135,-271.956 152.595,-276.926 157.609,-272.442 152.055,-281.897 152.055,-281.897\"/>\n",
       "</g>\n",
       "<!-- _plus0 -->\n",
       "<g id=\"node7\" class=\"node\"><title>_plus0</title>\n",
       "<polygon fill=\"#fccde5\" stroke=\"black\" points=\"140,-434 46,-434 46,-376 140,-376 140,-434\"/>\n",
       "<text text-anchor=\"middle\" x=\"93\" y=\"-400.8\" font-family=\"Times,serif\" font-size=\"14.00\">_Plus</text>\n",
       "</g>\n",
       "<!-- _plus0&#45;&gt;flatten0 -->\n",
       "<g id=\"edge6\" class=\"edge\"><title>_plus0&#45;&gt;flatten0</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M83.5392,-365.746C74.7499,-330.206 61.8645,-278.104 53.9766,-246.21\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"86.0136,-375.751 79.2443,-367.123 84.8132,-370.897 83.6127,-366.043 83.6127,-366.043 83.6127,-366.043 84.8132,-370.897 87.9811,-364.963 86.0136,-375.751 86.0136,-375.751\"/>\n",
       "</g>\n",
       "<!-- _plus0&#45;&gt;activation0 -->\n",
       "<g id=\"edge7\" class=\"edge\"><title>_plus0&#45;&gt;activation0</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M115.431,-367.148C120.887,-358.186 126.637,-348.74 131.806,-340.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"110.106,-375.897 111.462,-365.015 112.706,-371.626 115.306,-367.355 115.306,-367.355 115.306,-367.355 112.706,-371.626 119.15,-369.695 110.106,-375.897 110.106,-375.897\"/>\n",
       "</g>\n",
       "<!-- softmaxoutput0 -->\n",
       "<g id=\"node8\" class=\"node\"><title>softmaxoutput0</title>\n",
       "<polygon fill=\"#fccde5\" stroke=\"black\" points=\"144.332,-528 41.6684,-528 41.6684,-470 144.332,-470 144.332,-528\"/>\n",
       "<text text-anchor=\"middle\" x=\"93\" y=\"-494.8\" font-family=\"Times,serif\" font-size=\"14.00\">SoftmaxOutput</text>\n",
       "</g>\n",
       "<!-- softmaxoutput0&#45;&gt;_plus0 -->\n",
       "<g id=\"edge8\" class=\"edge\"><title>softmaxoutput0&#45;&gt;_plus0</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M93,-459.744C93,-451.204 93,-442.298 93,-434.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"93,-469.897 88.5001,-459.897 93,-464.897 93.0001,-459.897 93.0001,-459.897 93.0001,-459.897 93,-464.897 97.5001,-459.897 93,-469.897 93,-469.897\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x11488e940>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mx.viz.plot_network(\n",
    "    out,\n",
    "    node_attrs={'fixedsize': 'fasle'}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = mx.sym.Variable('data')\n",
    "net  = data\n",
    "fco  = []\n",
    "smo  = []\n",
    "cum  = None\n",
    "for ii in range(3):\n",
    "    if ii > 0:\n",
    "        if cum is None:\n",
    "            cum = pos\n",
    "        else:\n",
    "            cum = cum + pos\n",
    "        net = mx.sym.Concat(*[net, cum])\n",
    "    neg = mx.sym.FullyConnected(net, num_hidden=1)\n",
    "    pos = mx.sym.FullyConnected(net, num_hidden=1)\n",
    "    fco.append(mx.sym.Concat(*[neg, pos]))\n",
    "    smo.append(mx.sym.SoftmaxOutput(fco[ii]))\n",
    "out = mx.sym.Group(smo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: plot Pages: 1 -->\n",
       "<svg width=\"561pt\" height=\"818pt\"\n",
       " viewBox=\"0.00 0.00 561.43 818.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 814)\">\n",
       "<title>plot</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-814 557.435,-814 557.435,4 -4,4\"/>\n",
       "<!-- data -->\n",
       "<g id=\"node1\" class=\"node\"><title>data</title>\n",
       "<polygon fill=\"#8dd3c7\" stroke=\"black\" points=\"222.717,-58 128.717,-58 128.717,-0 222.717,-0 222.717,-58\"/>\n",
       "<text text-anchor=\"middle\" x=\"175.717\" y=\"-24.8\" font-family=\"Times,serif\" font-size=\"14.00\">data</text>\n",
       "</g>\n",
       "<!-- fullyconnected0 -->\n",
       "<g id=\"node2\" class=\"node\"><title>fullyconnected0</title>\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"105.653,-152 -0.218184,-152 -0.218184,-94 105.653,-94 105.653,-152\"/>\n",
       "<text text-anchor=\"middle\" x=\"52.7173\" y=\"-125.8\" font-family=\"Times,serif\" font-size=\"14.00\">FullyConnected</text>\n",
       "<text text-anchor=\"middle\" x=\"52.7173\" y=\"-111.8\" font-family=\"Times,serif\" font-size=\"14.00\">1</text>\n",
       "</g>\n",
       "<!-- fullyconnected0&#45;&gt;data -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>fullyconnected0&#45;&gt;data</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M98.6346,-87.6553C111.59,-77.9651 125.533,-67.5362 137.951,-58.2479\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"90.2897,-93.8971 95.6022,-84.3039 94.2936,-90.9023 98.2975,-87.9075 98.2975,-87.9075 98.2975,-87.9075 94.2936,-90.9023 100.993,-91.511 90.2897,-93.8971 90.2897,-93.8971\"/>\n",
       "</g>\n",
       "<!-- fullyconnected1 -->\n",
       "<g id=\"node3\" class=\"node\"><title>fullyconnected1</title>\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"228.653,-152 122.782,-152 122.782,-94 228.653,-94 228.653,-152\"/>\n",
       "<text text-anchor=\"middle\" x=\"175.717\" y=\"-125.8\" font-family=\"Times,serif\" font-size=\"14.00\">FullyConnected</text>\n",
       "<text text-anchor=\"middle\" x=\"175.717\" y=\"-111.8\" font-family=\"Times,serif\" font-size=\"14.00\">1</text>\n",
       "</g>\n",
       "<!-- fullyconnected1&#45;&gt;data -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>fullyconnected1&#45;&gt;data</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M175.717,-83.7443C175.717,-75.2043 175.717,-66.2977 175.717,-58.2479\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"175.717,-93.8971 171.217,-83.897 175.717,-88.8971 175.717,-83.8971 175.717,-83.8971 175.717,-83.8971 175.717,-88.8971 180.217,-83.8971 175.717,-93.8971 175.717,-93.8971\"/>\n",
       "</g>\n",
       "<!-- concat0 -->\n",
       "<g id=\"node4\" class=\"node\"><title>concat0</title>\n",
       "<polygon fill=\"#fdb462\" stroke=\"black\" points=\"110.717,-246 16.7173,-246 16.7173,-188 110.717,-188 110.717,-246\"/>\n",
       "<text text-anchor=\"middle\" x=\"63.7173\" y=\"-212.8\" font-family=\"Times,serif\" font-size=\"14.00\">Concat</text>\n",
       "</g>\n",
       "<!-- concat0&#45;&gt;fullyconnected0 -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>concat0&#45;&gt;fullyconnected0</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M59.1432,-177.744C58.1222,-169.204 57.0572,-160.298 56.0947,-152.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"60.3572,-187.897 54.7017,-178.502 59.7635,-182.932 59.1699,-177.968 59.1699,-177.968 59.1699,-177.968 59.7635,-182.932 63.638,-177.434 60.3572,-187.897 60.3572,-187.897\"/>\n",
       "</g>\n",
       "<!-- concat0&#45;&gt;fullyconnected1 -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>concat0&#45;&gt;fullyconnected1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M105.866,-181.378C117.567,-171.767 130.129,-161.448 141.329,-152.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"97.9296,-187.897 102.8,-178.072 101.793,-184.723 105.657,-181.55 105.657,-181.55 105.657,-181.55 101.793,-184.723 108.513,-185.027 97.9296,-187.897 97.9296,-187.897\"/>\n",
       "</g>\n",
       "<!-- softmaxoutput1 -->\n",
       "<g id=\"node5\" class=\"node\"><title>softmaxoutput1</title>\n",
       "<polygon fill=\"#fccde5\" stroke=\"black\" points=\"115.049,-340 12.3857,-340 12.3857,-282 115.049,-282 115.049,-340\"/>\n",
       "<text text-anchor=\"middle\" x=\"63.7173\" y=\"-306.8\" font-family=\"Times,serif\" font-size=\"14.00\">SoftmaxOutput</text>\n",
       "</g>\n",
       "<!-- softmaxoutput1&#45;&gt;concat0 -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>softmaxoutput1&#45;&gt;concat0</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M63.7173,-271.744C63.7173,-263.204 63.7173,-254.298 63.7173,-246.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"63.7173,-281.897 59.2174,-271.897 63.7173,-276.897 63.7174,-271.897 63.7174,-271.897 63.7174,-271.897 63.7173,-276.897 68.2174,-271.897 63.7173,-281.897 63.7173,-281.897\"/>\n",
       "</g>\n",
       "<!-- concat1 -->\n",
       "<g id=\"node6\" class=\"node\"><title>concat1</title>\n",
       "<polygon fill=\"#fdb462\" stroke=\"black\" points=\"303.717,-246 209.717,-246 209.717,-188 303.717,-188 303.717,-246\"/>\n",
       "<text text-anchor=\"middle\" x=\"256.717\" y=\"-212.8\" font-family=\"Times,serif\" font-size=\"14.00\">Concat</text>\n",
       "</g>\n",
       "<!-- concat1&#45;&gt;data -->\n",
       "<g id=\"edge6\" class=\"edge\"><title>concat1&#45;&gt;data</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M256.661,-177.706C255.237,-152.716 250.676,-120.024 237.717,-94 231.097,-80.7059 220.724,-68.3824 210.342,-58.171\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"257.094,-187.903 252.174,-178.103 256.882,-182.908 256.67,-177.912 256.67,-177.912 256.67,-177.912 256.882,-182.908 261.166,-177.721 257.094,-187.903 257.094,-187.903\"/>\n",
       "</g>\n",
       "<!-- concat1&#45;&gt;fullyconnected1 -->\n",
       "<g id=\"edge7\" class=\"edge\"><title>concat1&#45;&gt;fullyconnected1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M225.256,-180.266C217.075,-170.974 208.376,-161.094 200.588,-152.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"231.974,-187.897 221.989,-183.365 228.67,-184.144 225.366,-180.392 225.366,-180.392 225.366,-180.392 228.67,-184.144 228.744,-177.418 231.974,-187.897 231.974,-187.897\"/>\n",
       "</g>\n",
       "<!-- fullyconnected2 -->\n",
       "<g id=\"node7\" class=\"node\"><title>fullyconnected2</title>\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"410.653,-340 304.782,-340 304.782,-282 410.653,-282 410.653,-340\"/>\n",
       "<text text-anchor=\"middle\" x=\"357.717\" y=\"-313.8\" font-family=\"Times,serif\" font-size=\"14.00\">FullyConnected</text>\n",
       "<text text-anchor=\"middle\" x=\"357.717\" y=\"-299.8\" font-family=\"Times,serif\" font-size=\"14.00\">1</text>\n",
       "</g>\n",
       "<!-- fullyconnected2&#45;&gt;concat1 -->\n",
       "<g id=\"edge8\" class=\"edge\"><title>fullyconnected2&#45;&gt;concat1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M319.404,-275.101C308.939,-265.568 297.731,-255.359 287.729,-246.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"326.865,-281.897 316.442,-278.49 323.169,-278.53 319.472,-275.163 319.472,-275.163 319.472,-275.163 323.169,-278.53 322.503,-271.836 326.865,-281.897 326.865,-281.897\"/>\n",
       "</g>\n",
       "<!-- fullyconnected3 -->\n",
       "<g id=\"node8\" class=\"node\"><title>fullyconnected3</title>\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"287.653,-340 181.782,-340 181.782,-282 287.653,-282 287.653,-340\"/>\n",
       "<text text-anchor=\"middle\" x=\"234.717\" y=\"-313.8\" font-family=\"Times,serif\" font-size=\"14.00\">FullyConnected</text>\n",
       "<text text-anchor=\"middle\" x=\"234.717\" y=\"-299.8\" font-family=\"Times,serif\" font-size=\"14.00\">1</text>\n",
       "</g>\n",
       "<!-- fullyconnected3&#45;&gt;concat1 -->\n",
       "<g id=\"edge9\" class=\"edge\"><title>fullyconnected3&#45;&gt;concat1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M243.798,-272.026C245.861,-263.4 248.016,-254.386 249.962,-246.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"241.438,-281.897 239.387,-271.125 242.6,-277.034 243.763,-272.171 243.763,-272.171 243.763,-272.171 242.6,-277.034 248.14,-273.218 241.438,-281.897 241.438,-281.897\"/>\n",
       "</g>\n",
       "<!-- concat2 -->\n",
       "<g id=\"node9\" class=\"node\"><title>concat2</title>\n",
       "<polygon fill=\"#fdb462\" stroke=\"black\" points=\"381.717,-434 287.717,-434 287.717,-376 381.717,-376 381.717,-434\"/>\n",
       "<text text-anchor=\"middle\" x=\"334.717\" y=\"-400.8\" font-family=\"Times,serif\" font-size=\"14.00\">Concat</text>\n",
       "</g>\n",
       "<!-- concat2&#45;&gt;fullyconnected2 -->\n",
       "<g id=\"edge10\" class=\"edge\"><title>concat2&#45;&gt;fullyconnected2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M344.211,-366.026C346.367,-357.4 348.621,-348.386 350.655,-340.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"341.743,-375.897 339.803,-365.104 342.956,-371.046 344.168,-366.196 344.168,-366.196 344.168,-366.196 342.956,-371.046 348.534,-367.287 341.743,-375.897 341.743,-375.897\"/>\n",
       "</g>\n",
       "<!-- concat2&#45;&gt;fullyconnected3 -->\n",
       "<g id=\"edge11\" class=\"edge\"><title>concat2&#45;&gt;fullyconnected3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M296.783,-369.101C286.422,-359.568 275.325,-349.359 265.421,-340.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"304.171,-375.897 293.765,-372.438 300.491,-372.512 296.811,-369.126 296.811,-369.126 296.811,-369.126 300.491,-372.512 299.858,-365.815 304.171,-375.897 304.171,-375.897\"/>\n",
       "</g>\n",
       "<!-- softmaxoutput2 -->\n",
       "<g id=\"node10\" class=\"node\"><title>softmaxoutput2</title>\n",
       "<polygon fill=\"#fccde5\" stroke=\"black\" points=\"374.049,-528 271.386,-528 271.386,-470 374.049,-470 374.049,-528\"/>\n",
       "<text text-anchor=\"middle\" x=\"322.717\" y=\"-494.8\" font-family=\"Times,serif\" font-size=\"14.00\">SoftmaxOutput</text>\n",
       "</g>\n",
       "<!-- softmaxoutput2&#45;&gt;concat2 -->\n",
       "<g id=\"edge12\" class=\"edge\"><title>softmaxoutput2&#45;&gt;concat2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M327.707,-459.744C328.821,-451.204 329.983,-442.298 331.033,-434.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"326.383,-469.897 323.214,-459.399 327.03,-464.939 327.676,-459.981 327.676,-459.981 327.676,-459.981 327.03,-464.939 332.139,-460.563 326.383,-469.897 326.383,-469.897\"/>\n",
       "</g>\n",
       "<!-- _plus1 -->\n",
       "<g id=\"node11\" class=\"node\"><title>_plus1</title>\n",
       "<polygon fill=\"#fccde5\" stroke=\"black\" points=\"269.717,-434 175.717,-434 175.717,-376 269.717,-376 269.717,-434\"/>\n",
       "<text text-anchor=\"middle\" x=\"222.717\" y=\"-400.8\" font-family=\"Times,serif\" font-size=\"14.00\">_Plus</text>\n",
       "</g>\n",
       "<!-- _plus1&#45;&gt;fullyconnected1 -->\n",
       "<g id=\"edge13\" class=\"edge\"><title>_plus1&#45;&gt;fullyconnected1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M188.054,-368.002C181.765,-359.444 176.126,-349.901 172.717,-340 150.618,-275.816 161.268,-194.416 169.405,-152.229\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"194.23,-375.886 184.521,-370.789 191.146,-371.95 188.063,-368.014 188.063,-368.014 188.063,-368.014 191.146,-371.95 191.605,-365.239 194.23,-375.886 194.23,-375.886\"/>\n",
       "</g>\n",
       "<!-- _plus1&#45;&gt;fullyconnected3 -->\n",
       "<g id=\"edge14\" class=\"edge\"><title>_plus1&#45;&gt;fullyconnected3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M227.707,-365.744C228.821,-357.204 229.983,-348.298 231.033,-340.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"226.383,-375.897 223.214,-365.399 227.03,-370.939 227.676,-365.981 227.676,-365.981 227.676,-365.981 227.03,-370.939 232.139,-366.563 226.383,-375.897 226.383,-375.897\"/>\n",
       "</g>\n",
       "<!-- concat3 -->\n",
       "<g id=\"node12\" class=\"node\"><title>concat3</title>\n",
       "<polygon fill=\"#fdb462\" stroke=\"black\" points=\"485.717,-528 391.717,-528 391.717,-470 485.717,-470 485.717,-528\"/>\n",
       "<text text-anchor=\"middle\" x=\"438.717\" y=\"-494.8\" font-family=\"Times,serif\" font-size=\"14.00\">Concat</text>\n",
       "</g>\n",
       "<!-- concat3&#45;&gt;concat1 -->\n",
       "<g id=\"edge15\" class=\"edge\"><title>concat3&#45;&gt;concat1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M446.509,-459.856C453.771,-413.095 458.392,-333.707 419.717,-282 392.494,-245.604 341.478,-229.891 303.742,-223.116\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"444.834,-469.879 442.044,-459.274 445.658,-464.948 446.483,-460.016 446.483,-460.016 446.483,-460.016 445.658,-464.948 450.921,-460.758 444.834,-469.879 444.834,-469.879\"/>\n",
       "</g>\n",
       "<!-- concat3&#45;&gt;_plus1 -->\n",
       "<g id=\"edge16\" class=\"edge\"><title>concat3&#45;&gt;_plus1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M382.193,-469.992C335.846,-451.182 321.069,-451.829 278.717,-434 275.812,-432.777 272.846,-431.472 269.871,-430.119\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"391.563,-473.881 380.602,-474.204 386.945,-471.965 382.327,-470.048 382.327,-470.048 382.327,-470.048 386.945,-471.965 384.053,-465.892 391.563,-473.881 391.563,-473.881\"/>\n",
       "</g>\n",
       "<!-- fullyconnected4 -->\n",
       "<g id=\"node13\" class=\"node\"><title>fullyconnected4</title>\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"430.653,-622 324.782,-622 324.782,-564 430.653,-564 430.653,-622\"/>\n",
       "<text text-anchor=\"middle\" x=\"377.717\" y=\"-595.8\" font-family=\"Times,serif\" font-size=\"14.00\">FullyConnected</text>\n",
       "<text text-anchor=\"middle\" x=\"377.717\" y=\"-581.8\" font-family=\"Times,serif\" font-size=\"14.00\">1</text>\n",
       "</g>\n",
       "<!-- fullyconnected4&#45;&gt;concat3 -->\n",
       "<g id=\"edge17\" class=\"edge\"><title>fullyconnected4&#45;&gt;concat3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M401.966,-555.428C407.964,-546.383 414.298,-536.828 419.988,-528.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"396.351,-563.897 398.126,-553.076 399.114,-559.73 401.877,-555.563 401.877,-555.563 401.877,-555.563 399.114,-559.73 405.627,-558.049 396.351,-563.897 396.351,-563.897\"/>\n",
       "</g>\n",
       "<!-- fullyconnected5 -->\n",
       "<g id=\"node14\" class=\"node\"><title>fullyconnected5</title>\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"553.653,-622 447.782,-622 447.782,-564 553.653,-564 553.653,-622\"/>\n",
       "<text text-anchor=\"middle\" x=\"500.717\" y=\"-595.8\" font-family=\"Times,serif\" font-size=\"14.00\">FullyConnected</text>\n",
       "<text text-anchor=\"middle\" x=\"500.717\" y=\"-581.8\" font-family=\"Times,serif\" font-size=\"14.00\">1</text>\n",
       "</g>\n",
       "<!-- fullyconnected5&#45;&gt;concat3 -->\n",
       "<g id=\"edge18\" class=\"edge\"><title>fullyconnected5&#45;&gt;concat3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M476.071,-555.428C469.975,-546.383 463.536,-536.828 457.754,-528.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"481.778,-563.897 472.458,-558.119 478.984,-559.751 476.19,-555.604 476.19,-555.604 476.19,-555.604 478.984,-559.751 479.921,-553.09 481.778,-563.897 481.778,-563.897\"/>\n",
       "</g>\n",
       "<!-- concat4 -->\n",
       "<g id=\"node15\" class=\"node\"><title>concat4</title>\n",
       "<polygon fill=\"#fdb462\" stroke=\"black\" points=\"485.717,-716 391.717,-716 391.717,-658 485.717,-658 485.717,-716\"/>\n",
       "<text text-anchor=\"middle\" x=\"438.717\" y=\"-682.8\" font-family=\"Times,serif\" font-size=\"14.00\">Concat</text>\n",
       "</g>\n",
       "<!-- concat4&#45;&gt;fullyconnected4 -->\n",
       "<g id=\"edge19\" class=\"edge\"><title>concat4&#45;&gt;fullyconnected4</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M414.469,-649.428C408.471,-640.383 402.136,-630.828 396.447,-622.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"420.084,-657.897 410.807,-652.049 417.321,-653.73 414.558,-649.563 414.558,-649.563 414.558,-649.563 417.321,-653.73 418.308,-647.076 420.084,-657.897 420.084,-657.897\"/>\n",
       "</g>\n",
       "<!-- concat4&#45;&gt;fullyconnected5 -->\n",
       "<g id=\"edge20\" class=\"edge\"><title>concat4&#45;&gt;fullyconnected5</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M463.363,-649.428C469.459,-640.383 475.898,-630.828 481.681,-622.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"457.656,-657.897 459.513,-647.09 460.451,-653.751 463.245,-649.604 463.245,-649.604 463.245,-649.604 460.451,-653.751 466.977,-652.119 457.656,-657.897 457.656,-657.897\"/>\n",
       "</g>\n",
       "<!-- softmaxoutput3 -->\n",
       "<g id=\"node16\" class=\"node\"><title>softmaxoutput3</title>\n",
       "<polygon fill=\"#fccde5\" stroke=\"black\" points=\"490.049,-810 387.386,-810 387.386,-752 490.049,-752 490.049,-810\"/>\n",
       "<text text-anchor=\"middle\" x=\"438.717\" y=\"-776.8\" font-family=\"Times,serif\" font-size=\"14.00\">SoftmaxOutput</text>\n",
       "</g>\n",
       "<!-- softmaxoutput3&#45;&gt;concat4 -->\n",
       "<g id=\"edge21\" class=\"edge\"><title>softmaxoutput3&#45;&gt;concat4</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M438.717,-741.744C438.717,-733.204 438.717,-724.298 438.717,-716.248\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"438.717,-751.897 434.217,-741.897 438.717,-746.897 438.717,-741.897 438.717,-741.897 438.717,-741.897 438.717,-746.897 443.217,-741.897 438.717,-751.897 438.717,-751.897\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x11488e898>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mx.viz.plot_network(\n",
    "    out,\n",
    "    node_attrs={'fixedsize': 'fasle'}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}