neural_network_automation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Goal : find hyperparameters\n",
    "--"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "from sklearn.model_selection import train_test_split\n",
    "from scipy.optimize import minimize\n",
    "import tuxml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = tuxml.load_dataset()\n",
    "selection = pd.read_csv(\"feature_net.csv\", index_col = 0, skiprows = 1, names = ['feat'])['feat']\n",
    "selection[len(selection)] = 'vmlinux'\n",
    "features = features[selection].replace([2,-1],0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training-test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "n = 65000\n",
    "sizes = np.array(features[0:n]['vmlinux'])\n",
    "x_train, x_test, y_train, y_test = train_test_split(features.drop('vmlinux', axis=1)[0:n], sizes, test_size = 0.1)\n",
    "\n",
    "nbCol = len(features.columns)\n",
    "\n",
    "x_train = np.array(x_train, dtype = np.float32)\n",
    "x_test =  np.array(x_test, dtype = np.float32)\n",
    "\n",
    "y_train = np.array(y_train, dtype = np.float32)\n",
    "y_test = np.array(y_test, dtype = np.float32)\n",
    "\n",
    "nb_features = x_train.shape[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Function to minimize:\n",
    "- take the data in input\n",
    "- compute the neural network\n",
    "- return the test MAPE in output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_net(batch_size = 100, nb_epochs = 30, nb_node_layer1 = 200, nb_node_layer2 = 100, lr1 = 1, lr2 = 0.01):\n",
    "    \n",
    "    #     batch_size = int(x[0])\n",
    "    #     nb_epochs = int(x[1])\n",
    "    #     nb_node_layer1 = int(x[2])\n",
    "    #     nb_node_layer2 = int(x[3])\n",
    "    #     lr1 = x[4]\n",
    "    #     lr2 = x[5]\n",
    "    \n",
    "    nb_batch_train = int(len(x_train)/batch_size)-1\n",
    "    nb_batch_test = int(len(x_test)/batch_size)\n",
    "    \n",
    "    # slice the datasets => feed_dict was very slow, so I choose an iterator solution\n",
    "    dataset_train = tf.data.Dataset.from_tensor_slices((x_train, y_train)).batch(batch_size)\n",
    "    iterator_train = tf.compat.v1.data.make_initializable_iterator(dataset_train)\n",
    "    xtr, ytr = iterator_train.get_next()\n",
    "\n",
    "    dataset_test = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(batch_size)\n",
    "    iterator_test = tf.compat.v1.data.make_initializable_iterator(dataset_test)\n",
    "    xte, yte = iterator_test.get_next()\n",
    "\n",
    "    with tf.device(\"/gpu:0\"):\n",
    "        # Layers training\n",
    "        w_h1_tr = tf.Variable(tf.glorot_uniform_initializer()((nb_features, nb_node_layer1)), name = \"w_h1_tr\")\n",
    "        mat_h1_tr = tf.matmul(xtr, w_h1_tr)\n",
    "        b_h1_tr = tf.Variable(tf.zeros(nb_node_layer1), name=\"b_h1_tr\")\n",
    "        out_h1_tr = tf.nn.relu(tf.add(mat_h1_tr, b_h1_tr))\n",
    "\n",
    "        w_h2_tr = tf.Variable(tf.glorot_uniform_initializer()((nb_node_layer1, nb_node_layer2)), name = \"w_h2_tr\")\n",
    "        mat_h2_tr = tf.matmul(out_h1_tr, w_h2_tr)\n",
    "        b_h2_tr = tf.Variable(tf.zeros(nb_node_layer2), name=\"b_h2_tr\")\n",
    "        out_h2_tr = tf.nn.relu(tf.add(mat_h2_tr, b_h2_tr))\n",
    "\n",
    "        w_final_tr = tf.Variable(tf.glorot_uniform_initializer()((nb_node_layer2, 1)), name = \"w_final_tr\")\n",
    "        outputs_tr = tf.reshape(tf.matmul(out_h2_tr, w_final_tr), shape=[batch_size])\n",
    "        ytr = tf.reshape(ytr, [batch_size])\n",
    "\n",
    "        # Layers test\n",
    "        mat_h1_te = tf.matmul(xte, w_h1_tr)\n",
    "        out_h1_te = tf.nn.relu(tf.add(mat_h1_te, b_h1_tr))\n",
    "\n",
    "        mat_h2_te = tf.matmul(out_h1_te, w_h2_tr)\n",
    "        out_h2_te = tf.nn.relu(tf.add(mat_h2_te, b_h2_tr))\n",
    "\n",
    "        outputs_te =  tf.reshape(tf.matmul(out_h2_te, w_final_tr), shape=[batch_size])\n",
    "        yte = tf.reshape(yte, [batch_size])\n",
    "\n",
    "        # Cost => MAPE\n",
    "        train_cost = tf.keras.losses.MAPE(ytr, outputs_tr)\n",
    "        test_cost = tf.keras.losses.MAPE(yte, outputs_te)\n",
    "\n",
    "        # Convergence function => AdamOptimizer\n",
    "        train_step = tf.train.AdamOptimizer(learning_rate=lr1).minimize(train_cost)\n",
    "\n",
    "        # train step with a lower learning rate => gain few % at the end\n",
    "        tiny_train_step = tf.train.AdamOptimizer(learning_rate=lr2).minimize(train_cost)\n",
    "\n",
    "        # allocate memory for tensors\n",
    "        init = tf.global_variables_initializer()\n",
    "\n",
    "    with tf.Session() as sess:\n",
    "        sess.run(init)\n",
    "        for j in range(nb_epochs):\n",
    "            if j < 20:\n",
    "                sess.run(iterator_train.initializer)\n",
    "                for i in range(nb_batch_train):\n",
    "                    val = sess.run(train_step)\n",
    "                #print(\"Cout entrainement epoch n°\", j+1, \":\",  sess.run(train_cost))\n",
    "            else:\n",
    "                sess.run(iterator_train.initializer)\n",
    "                for i in range(nb_batch_train):\n",
    "                    val = sess.run(tiny_train_step)\n",
    "                #print(\"Cout entrainement epoch n°\", j+1, \":\",  sess.run(train_cost))\n",
    "        mape = 0\n",
    "        sess.run(iterator_train.initializer)\n",
    "        for i in range(nb_batch_train):\n",
    "            mape+=sess.run(train_cost)\n",
    "        mape = 0\n",
    "        sess.run(iterator_test.initializer)\n",
    "        for i in range(nb_batch_test):\n",
    "            mape+=sess.run(test_cost)\n",
    "    return mape/nb_batch_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Search for hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\llesoil\\Documents\\Anaconda3\\envs\\tensor_gpu\\lib\\site-packages\\tensorflow\\python\\data\\ops\\dataset_ops.py:1419: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From C:\\Users\\llesoil\\Documents\\Anaconda3\\envs\\tensor_gpu\\lib\\site-packages\\tensorflow\\python\\ops\\math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{(0.5, 0.01, 50): 6.1139626759749195,\n",
       " (0.5, 0.01, 100): 6.288037366133469,\n",
       " (0.5, 0.01, 200): 6.3636893182992935,\n",
       " (0.5, 0.05, 50): 5.927036344088041,\n",
       " (0.5, 0.05, 100): 6.302827460949238,\n",
       " (0.5, 0.05, 200): 6.8108696937561035,\n",
       " (1, 0.01, 50): 6.401567217019888,\n",
       " (1, 0.01, 100): 6.15475372167734,\n",
       " (1, 0.01, 200): 6.420121535658836,\n",
       " (1, 0.05, 50): 6.04258550130404,\n",
       " (1, 0.05, 100): 6.142240516956036,\n",
       " (1, 0.05, 200): 6.298232853412628,\n",
       " (2, 0.01, 50): 6.474702497629019,\n",
       " (2, 0.01, 100): 6.676808716700627,\n",
       " (2, 0.01, 200): 6.442576393485069,\n",
       " (2, 0.05, 50): 6.371084561714759,\n",
       " (2, 0.05, 100): 6.497246683560885,\n",
       " (2, 0.05, 200): 6.085170641541481,\n",
       " (5, 0.01, 50): 40.140974279550406,\n",
       " (5, 0.01, 100): 40.13643575815054,\n",
       " (5, 0.01, 200): 40.16221237182617,\n",
       " (5, 0.05, 50): 40.138973910991965,\n",
       " (5, 0.05, 100): 40.13834428053636,\n",
       " (5, 0.05, 200): 40.16430842876434,\n",
       " (10, 0.01, 50): 40.14627510951116,\n",
       " (10, 0.01, 100): 40.136890646127554,\n",
       " (10, 0.01, 200): 100.0,\n",
       " (10, 0.05, 50): 40.13960227966309,\n",
       " (10, 0.05, 100): 40.13918938269982,\n",
       " (10, 0.05, 200): 100.0}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr1_values = [0.5, 1, 2, 5, 10]\n",
    "lr2_values = [0.01, 0.05]\n",
    "batch_size = [50, 100, 200]\n",
    "nb_node_layer1 = [200, 300, 400]\n",
    "nb_node_layer2 = [100, 200, 300]\n",
    "\n",
    "res = dict()\n",
    "\n",
    "for lr1 in lr1_values:\n",
    "    for lr2 in lr2_values:\n",
    "        for bs in batch_size:\n",
    "            #for nbl1 in nb_node_layer1:\n",
    "            #for nbl2 in nb_node_layer2:\n",
    "            res[lr1, lr2, bs] = compute_net(lr1 = lr1, lr2 = lr2, batch_size = bs)#,nb_node_layer1 = nbl1, nb_node_layer2 = nbl2)\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\llesoil\\Documents\\Anaconda3\\envs\\tensor_gpu\\lib\\site-packages\\tensorflow\\python\\data\\ops\\dataset_ops.py:1419: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From C:\\Users\\llesoil\\Documents\\Anaconda3\\envs\\tensor_gpu\\lib\\site-packages\\tensorflow\\python\\ops\\math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{(0.25, 0.025, 50): 6.469745023433979,\n",
       " (0.25, 0.05, 50): 6.543176416250375,\n",
       " (0.25, 0.0075, 50): 6.792832026114831,\n",
       " (0.5, 0.025, 50): 5.791097765702467,\n",
       " (0.5, 0.05, 50): 5.865269492222713,\n",
       " (0.5, 0.0075, 50): 6.051237762891329,\n",
       " (0.75, 0.025, 50): 5.994227405694815,\n",
       " (0.75, 0.05, 50): 6.2046061038970945,\n",
       " (0.75, 0.0075, 50): 6.284441397740291}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr1_values = [0.25, 0.5, 0.75]\n",
    "lr2_values = [0.025, 0.05, 0.0075]\n",
    "batch_size = 50\n",
    "\n",
    "res = dict()\n",
    "\n",
    "for lr1 in lr1_values:\n",
    "    for lr2 in lr2_values:\n",
    "        res[lr1, lr2, batch_size] = compute_net(lr1 = lr1, lr2 = lr2, batch_size = batch_size)\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.944322465016292\n",
      "6.056905442017776\n",
      "6.001912755232591\n",
      "6.062779107460609\n",
      "5.9570104598999025\n"
     ]
    }
   ],
   "source": [
    "for i in range(5):\n",
    "    print(compute_net(lr1 = 0.5, lr2 = 0.025, batch_size = 50))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I guess there is more optimization to do in the initialization weights than in our hyperparameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{(0.75, 0.0075, 50, 200, 100): 6.045812485768245,\n",
       " (0.75, 0.0075, 50, 200, 200): 6.0127736421731806,\n",
       " (0.75, 0.0075, 50, 200, 300): 6.077524845416729,\n",
       " (0.75, 0.0075, 50, 300, 100): 6.190631213554969,\n",
       " (0.75, 0.0075, 50, 300, 200): 6.088517420108502,\n",
       " (0.75, 0.0075, 50, 300, 300): 6.019588349415706,\n",
       " (0.75, 0.0075, 50, 400, 100): 6.295459662950956,\n",
       " (0.75, 0.0075, 50, 400, 200): 6.401230922112098,\n",
       " (0.75, 0.0075, 50, 400, 300): 6.162542775961069}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr1_values = 0.5\n",
    "lr2_values = 0.025\n",
    "batch_size = 50\n",
    "nb_node_layer1 = [200, 300, 400]\n",
    "nb_node_layer2 = [100, 200, 300]\n",
    "\n",
    "res = dict()\n",
    "\n",
    "for node1 in nb_node_layer1:\n",
    "    for node2 in nb_node_layer2:\n",
    "        res[lr1, lr2, batch_size, node1, node2] = compute_net(lr1 = lr1, lr2 = lr2, batch_size = batch_size,\n",
    "                                                             nb_node_layer1 = node1, nb_node_layer2 = node2)\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{(0.5, 0.025, 50, 200, 100): 5.866956404539255,\n",
       " (0.5, 0.025, 50, 200, 200): 5.94318119929387,\n",
       " (0.5, 0.025, 50, 200, 300): 6.0386133597447325,\n",
       " (0.5, 0.025, 50, 300, 100): 5.981153865960928,\n",
       " (0.5, 0.025, 50, 300, 200): 5.940192593061007,\n",
       " (0.5, 0.025, 50, 300, 300): 6.243418946632972,\n",
       " (0.5, 0.025, 50, 400, 100): 6.01619873046875,\n",
       " (0.5, 0.025, 50, 400, 200): 6.003289969150837,\n",
       " (0.5, 0.025, 50, 400, 300): 5.883027010697585}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#to run this night\n",
    "\n",
    "lr1_values = 0.5\n",
    "lr2_values = 0.025\n",
    "batch_size = 50\n",
    "nb_node_layer1 = [200, 300, 400]\n",
    "nb_node_layer2 = [100, 200, 300]\n",
    "\n",
    "res = dict()\n",
    "\n",
    "for node1 in nb_node_layer1:\n",
    "    for node2 in nb_node_layer2:\n",
    "        res[lr1_values, lr2_values, batch_size, node1, node2] = compute_net(lr1 = lr1_values, \n",
    "                                                                            lr2 = lr2_values, \n",
    "                                                                            batch_size = batch_size,\n",
    "                                                                            nb_node_layer1 = node1,\n",
    "                                                                            nb_node_layer2 = node2)\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\llesoil\\Documents\\Anaconda3\\envs\\tensor_gpu\\lib\\site-packages\\tensorflow\\python\\data\\ops\\dataset_ops.py:1419: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From C:\\Users\\llesoil\\Documents\\Anaconda3\\envs\\tensor_gpu\\lib\\site-packages\\tensorflow\\python\\ops\\math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{(0.5, 0.025, 50, 200, 100): 6.007373765798715,\n",
       " (0.5, 0.025, 50, 200, 200): 5.819840312004089,\n",
       " (0.5, 0.025, 50, 200, 300): 5.861549256398128,\n",
       " (0.5, 0.025, 50, 300, 100): 5.878635898003211,\n",
       " (0.5, 0.025, 50, 300, 200): 5.930829114180344,\n",
       " (0.5, 0.025, 50, 300, 300): 6.105181063138522,\n",
       " (0.5, 0.025, 50, 400, 100): 5.934586646006657,\n",
       " (0.5, 0.025, 50, 400, 200): 5.895523650829609,\n",
       " (0.5, 0.025, 50, 400, 300): 5.8445775875678425}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#to run this night\n",
    "\n",
    "lr1_values = 0.5\n",
    "lr2_values = 0.025\n",
    "batch_size = 50\n",
    "nb_node_layer1 = [200, 300, 400]\n",
    "nb_node_layer2 = [100, 200, 300]\n",
    "\n",
    "res = dict()\n",
    "\n",
    "for node1 in nb_node_layer1:\n",
    "    for node2 in nb_node_layer2:\n",
    "        res[lr1_values, lr2_values, batch_size, node1, node2] = compute_net(lr1 = lr1_values, \n",
    "                                                                            lr2 = lr2_values, \n",
    "                                                                            batch_size = batch_size,\n",
    "                                                                            nb_node_layer1 = node1,\n",
    "                                                                            nb_node_layer2 = node2)\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{(0.5, 0.025, 50, 200, 100): 5.960509806412917,\n",
       " (0.5, 0.025, 50, 200, 200): 6.067634362440843,\n",
       " (0.5, 0.025, 50, 200, 300): 5.730486352627094,\n",
       " (0.5, 0.025, 50, 300, 100): 5.900232065640963,\n",
       " (0.5, 0.025, 50, 300, 200): 5.833492433107816,\n",
       " (0.5, 0.025, 50, 300, 300): 5.984292393464308,\n",
       " (0.5, 0.025, 50, 400, 100): 6.043863296508789,\n",
       " (0.5, 0.025, 50, 400, 200): 5.981327731792743,\n",
       " (0.5, 0.025, 50, 400, 300): 5.959470316079947}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#to run this night\n",
    "\n",
    "lr1_values = 0.5\n",
    "lr2_values = 0.025\n",
    "batch_size = 50\n",
    "nb_node_layer1 = [200, 300, 400]\n",
    "nb_node_layer2 = [100, 200, 300]\n",
    "\n",
    "res = dict()\n",
    "\n",
    "for node1 in nb_node_layer1:\n",
    "    for node2 in nb_node_layer2:\n",
    "        res[lr1_values, lr2_values, batch_size, node1, node2] = compute_net(lr1 = lr1_values, \n",
    "                                                                            lr2 = lr2_values, \n",
    "                                                                            batch_size = batch_size,\n",
    "                                                                            nb_node_layer1 = node1,\n",
    "                                                                            nb_node_layer2 = node2)\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}