net_best_features_selection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Goal : Select the best set of features to get the lowest test MAPE with nets\n",
    "\n",
    "--\n",
    "\n",
    "### Results => \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "from sklearn.model_selection import train_test_split\n",
    "import tuxml"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## [BEST] Original features (tree) => 5,7-6,3%\n",
    "\n",
    "Could it be because we adjust the first hyperparameters to these features?\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = tuxml.load_dataset()\n",
    "selection = pd.read_csv(\"feature_net.csv\", index_col = 0, skiprows = 1, names = ['feat'])['feat']\n",
    "selection[len(selection)] = 'vmlinux'\n",
    "features = features[selection].replace([2,-1],0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Forest => 6,5-7%\n",
    "\n",
    "with the new sampling (23/07/19)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "925 features kept for 98.5 % of the predictive power\n"
     ]
    }
   ],
   "source": [
    "# we download the dataset using the tuxml script\n",
    "features = tuxml.load_dataset()\n",
    "\n",
    "# then we select the features based on random forest importance\n",
    "df_selection = pd.read_csv('feature_importanceRF.csv', names = ['features', 'imp'], skiprows = 0)\n",
    "selection = df_selection['features']\n",
    "imp = df_selection['imp']\n",
    "\n",
    "# we select the more important features until we get more than alpha % of the 'predictive power'\n",
    "alpha = 0.985\n",
    "sorted_imp = sorted(imp, reverse = True)\n",
    "sum_imp = 0\n",
    "i = 0\n",
    "\n",
    "while sum_imp < alpha:\n",
    "    sum_imp+=sorted_imp[i]\n",
    "    i+=1\n",
    "threshold_imp = sorted_imp[i]\n",
    "\n",
    "final = selection[imp>threshold_imp]\n",
    "final[len(final)] = 'vmlinux'\n",
    "\n",
    "# selection\n",
    "features = features[final]\n",
    "\n",
    "# normalization of the nb yes/no/module counters\n",
    "col = features.columns\n",
    "# I assumed 12611 was (around) the number of real options we can change to compile the linux kernel => cf option_columns.json\n",
    "nb_features = 12611\n",
    "to_normalize = ['nbyes', 'nbno', 'nbmodule', 'nbyesmodule']\n",
    "for feat in to_normalize:\n",
    "    if feat in col:\n",
    "        features[feat] = features[feat]/nb_features\n",
    "print(len(final), \"features kept for\", 100*alpha, \"% of the predictive power\")\n",
    "#features.to_pickle(\"feature_rf_98.pkl\")\n",
    "features = features.replace([2,-1],0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## features DT => 16%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = tuxml.load_dataset()\n",
    "selection = pd.read_csv(\"feature_importanceDT.csv\", skiprows = 1, names = ['features','imp'])['features']\n",
    "selection[len(selection)] = 'vmlinux'\n",
    "features = features[selection].replace([2,-1],0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## features EN => 11%\n",
    "\n",
    "I selected the 1200 more important features, to be around the number of features we used for the tree/RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = tuxml.load_dataset()\n",
    "selection = pd.read_csv(\"feature_importanceEN.csv\", skiprows = 1, names = ['features','imp'])['features'][0:1200]\n",
    "selection[len(selection)] = 'vmlinux'\n",
    "features = features[selection].replace([2,-1], 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## features GB => 12%\n",
    "\n",
    "Impressive results for only 150 features + no major difference between train/test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = tuxml.load_dataset()\n",
    "selection = pd.read_csv(\"feature_importanceGB.csv\", skiprows = 1, names = ['features','imp'])['features']\n",
    "selection[len(selection)] = 'vmlinux'\n",
    "features = features[selection].replace([2,-1], 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## features Lasso => 12%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = tuxml.load_dataset()\n",
    "selection = pd.read_csv(\"feature_importanceLasso.csv\", skiprows = 1, names = ['features','imp'])['features'][0:1200]\n",
    "selection[len(selection)] = 'vmlinux'\n",
    "features = features[selection].replace([2,-1], 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## features LR => 26%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = tuxml.load_dataset()\n",
    "selection = pd.read_csv(\"feature_importanceLR.csv\", skiprows = 1, names = ['features','imp'])['features'][0:1200]\n",
    "selection[len(selection)] = 'vmlinux'\n",
    "features = features[selection].replace([2,-1], 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## features Ridge => 19%\n",
    "\n",
    "training MAPE around 11-12%, which is a huge difference between train/test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = tuxml.load_dataset()\n",
    "selection = pd.read_csv(\"feature_importanceRidge.csv\", skiprows = 1, names = ['features','imp'])['features'][0:1200]\n",
    "selection[len(selection)] = 'vmlinux'\n",
    "features = features[selection].replace([2,-1], 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## linux correlation => 15%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "fi_correlation = pd.read_csv(\"correlations_vmlinux.csv\", skiprows= 1, names = ['features', 'imp'])\n",
    "fi_correlation['imp'] =  np.abs(fi_correlation['imp'])\n",
    "fi_corr = np.array(fi_correlation.sort_values(by = 'imp', ascending = False)['features'][0:1000])\n",
    "features = tuxml.load_dataset()\n",
    "selection = [f for f in fi_corr]\n",
    "selection.append('vmlinux')\n",
    "features = features[selection].replace([2,-1], 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## features welch test => 11%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = tuxml.load_dataset()\n",
    "selection = pd.read_csv(\"welch_test_output.csv\", skiprows = 1, names = ['features','imp'])['features']\n",
    "selection[len(selection)] = 'vmlinux'\n",
    "features = features[selection].replace([2,-1], 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test cell"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cout entrainement epoch n° 1 : 16.271944\n",
      "Cout entrainement epoch n° 2 : 14.176077\n",
      "Cout entrainement epoch n° 3 : 12.891419\n",
      "Cout entrainement epoch n° 4 : 11.615289\n",
      "Cout entrainement epoch n° 5 : 10.719225\n",
      "Cout entrainement epoch n° 6 : 11.36039\n",
      "Cout entrainement epoch n° 7 : 10.266355\n",
      "Cout entrainement epoch n° 8 : 10.920399\n",
      "Cout entrainement epoch n° 9 : 8.88579\n",
      "Cout entrainement epoch n° 10 : 11.544925\n",
      "Cout entrainement epoch n° 11 : 10.53098\n",
      "Cout entrainement epoch n° 12 : 10.526862\n",
      "Cout entrainement epoch n° 13 : 9.490666\n",
      "Cout entrainement epoch n° 14 : 10.370634\n",
      "Cout entrainement epoch n° 15 : 9.957453\n",
      "Cout entrainement epoch n° 16 : 12.372447\n",
      "Cout entrainement epoch n° 17 : 12.370262\n",
      "Cout entrainement epoch n° 18 : 11.229739\n",
      "Cout entrainement epoch n° 19 : 10.180439\n",
      "Cout entrainement epoch n° 20 : 12.368385\n",
      "Cout entrainement epoch n° 21 : 8.746283\n",
      "Cout entrainement epoch n° 22 : 8.622522\n",
      "Cout entrainement epoch n° 23 : 8.453609\n",
      "Cout entrainement epoch n° 24 : 8.530566\n",
      "Cout entrainement epoch n° 25 : 8.479752\n",
      "Cout entrainement epoch n° 26 : 8.402679\n",
      "Cout entrainement epoch n° 27 : 8.4105835\n",
      "Cout entrainement epoch n° 28 : 8.473532\n",
      "Cout entrainement epoch n° 29 : 8.444162\n",
      "Cout entrainement epoch n° 30 : 8.389452\n",
      "Cout entrainement final = 3.7236387115752625\n",
      "Cout test final = 6.960322871574989\n"
     ]
    }
   ],
   "source": [
    "n = 65000\n",
    "sizes = np.array(features[0:n]['vmlinux'])\n",
    "x_train, x_test, y_train, y_test = train_test_split(features.drop('vmlinux', axis=1)[0:n], sizes, test_size = 0.1)\n",
    "\n",
    "nbCol = len(features.columns)\n",
    "\n",
    "x_train = np.array(x_train, dtype = np.float32)\n",
    "x_test =  np.array(x_test, dtype = np.float32)\n",
    "\n",
    "y_train = np.array(y_train, dtype = np.float32)\n",
    "y_test = np.array(y_test, dtype = np.float32)\n",
    "\n",
    "nb_features = x_train.shape[1]\n",
    "batch_size = 50\n",
    "nb_epochs = 30\n",
    "nb_batch_train = int(len(x_train)/batch_size)-1\n",
    "nb_batch_test = int(len(x_test)/batch_size)\n",
    "nb_node_layer1 = 200\n",
    "nb_node_layer2 = 300\n",
    "\n",
    "# slice the datasets => feed_dict was very slow, so I choose an iterator solution\n",
    "dataset_train = tf.data.Dataset.from_tensor_slices((x_train, y_train)).batch(batch_size)\n",
    "iterator_train = tf.compat.v1.data.make_initializable_iterator(dataset_train)\n",
    "xtr, ytr = iterator_train.get_next()\n",
    "\n",
    "dataset_test = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(batch_size)\n",
    "iterator_test = tf.compat.v1.data.make_initializable_iterator(dataset_test)\n",
    "xte, yte = iterator_test.get_next()\n",
    "\n",
    "with tf.device(\"/gpu:0\"):\n",
    "    # Layers training\n",
    "    w_h1_tr = tf.Variable(tf.glorot_uniform_initializer()((nb_features, nb_node_layer1)), name = \"w_h1_tr\")\n",
    "    mat_h1_tr = tf.matmul(xtr, w_h1_tr)\n",
    "    b_h1_tr = tf.Variable(tf.zeros(nb_node_layer1), name=\"b_h1_tr\")\n",
    "    out_h1_tr = tf.nn.relu(tf.add(mat_h1_tr, b_h1_tr))\n",
    "\n",
    "    w_h2_tr = tf.Variable(tf.glorot_uniform_initializer()((nb_node_layer1, nb_node_layer2)), name = \"w_h2_tr\")\n",
    "    mat_h2_tr = tf.matmul(out_h1_tr, w_h2_tr)\n",
    "    b_h2_tr = tf.Variable(tf.zeros(nb_node_layer2), name=\"b_h2_tr\")\n",
    "    out_h2_tr = tf.nn.relu(tf.add(mat_h2_tr, b_h2_tr))\n",
    "\n",
    "    w_final_tr = tf.Variable(tf.glorot_uniform_initializer()((nb_node_layer2, 1)), name = \"w_final_tr\")\n",
    "    outputs_tr =  tf.reshape(tf.matmul(out_h2_tr, w_final_tr), shape=[batch_size])\n",
    "    ytr = tf.reshape(ytr, [batch_size])\n",
    "\n",
    "    # Layers test\n",
    "    mat_h1_te = tf.matmul(xte, w_h1_tr)\n",
    "    out_h1_te = tf.nn.relu(tf.add(mat_h1_te, b_h1_tr))\n",
    "\n",
    "    mat_h2_te = tf.matmul(out_h1_te, w_h2_tr)\n",
    "    out_h2_te = tf.nn.relu(tf.add(mat_h2_te, b_h2_tr))\n",
    "\n",
    "    outputs_te =  tf.reshape(tf.matmul(out_h2_te, w_final_tr), shape=[batch_size])\n",
    "    yte = tf.reshape(yte, [batch_size])\n",
    "\n",
    "    # Cost => MAPE\n",
    "    train_cost = tf.keras.losses.MAPE(ytr, outputs_tr)\n",
    "    test_cost = tf.keras.losses.MAPE(yte, outputs_te)\n",
    "\n",
    "    # Convergence function => AdamOptimizer\n",
    "    train_step = tf.train.AdamOptimizer(learning_rate=0.5).minimize(train_cost)\n",
    "\n",
    "    # train step with a lower learning rate => gain few % at the end\n",
    "    tiny_train_step = tf.train.AdamOptimizer(learning_rate=0.025).minimize(train_cost)\n",
    "    \n",
    "    # allocate memory for tensors\n",
    "    init = tf.global_variables_initializer()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    for j in range(nb_epochs):\n",
    "        if j < 20:\n",
    "            sess.run(iterator_train.initializer)\n",
    "            for i in range(nb_batch_train):\n",
    "                val = sess.run(train_step)\n",
    "            print(\"Cout entrainement epoch n°\", j+1, \":\",  sess.run(train_cost))\n",
    "        else:\n",
    "            sess.run(iterator_train.initializer)\n",
    "            for i in range(nb_batch_train):\n",
    "                val = sess.run(tiny_train_step)\n",
    "            print(\"Cout entrainement epoch n°\", j+1, \":\",  sess.run(train_cost))\n",
    "    sess.run(iterator_train.initializer)\n",
    "    mape = 0\n",
    "    for i in range(nb_batch_train):\n",
    "        mape+=sess.run(train_cost)\n",
    "    print(\"Cout entrainement final =\", mape/nb_batch_train)\n",
    "    sess.run(iterator_test.initializer)\n",
    "    mape = 0\n",
    "    for i in range(nb_batch_test):\n",
    "        mape+=sess.run(test_cost)\n",
    "    print(\"Cout test final =\", mape/nb_batch_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}