craigslist-cars-lgbm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>condition</th>\n",
       "      <th>drive</th>\n",
       "      <th>fuel</th>\n",
       "      <th>location</th>\n",
       "      <th>odometer</th>\n",
       "      <th>paint color</th>\n",
       "      <th>price</th>\n",
       "      <th>sale type</th>\n",
       "      <th>title status</th>\n",
       "      <th>transmission</th>\n",
       "      <th>...</th>\n",
       "      <th>year</th>\n",
       "      <th>age</th>\n",
       "      <th>make_</th>\n",
       "      <th>model</th>\n",
       "      <th>pos_words</th>\n",
       "      <th>neg_words</th>\n",
       "      <th>low_text</th>\n",
       "      <th>model0</th>\n",
       "      <th>model1</th>\n",
       "      <th>model2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>good</td>\n",
       "      <td>4wd</td>\n",
       "      <td>gas</td>\n",
       "      <td>vancouver</td>\n",
       "      <td>281000</td>\n",
       "      <td>white</td>\n",
       "      <td>5500</td>\n",
       "      <td>owner</td>\n",
       "      <td>rebuilt</td>\n",
       "      <td>automatic</td>\n",
       "      <td>...</td>\n",
       "      <td>2007</td>\n",
       "      <td>14</td>\n",
       "      <td>bmw</td>\n",
       "      <td>x3</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>x3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>unknown</td>\n",
       "      <td>fwd</td>\n",
       "      <td>gas</td>\n",
       "      <td>vancouver</td>\n",
       "      <td>140000</td>\n",
       "      <td>unknown</td>\n",
       "      <td>600</td>\n",
       "      <td>owner</td>\n",
       "      <td>clean</td>\n",
       "      <td>automatic</td>\n",
       "      <td>...</td>\n",
       "      <td>1984</td>\n",
       "      <td>37</td>\n",
       "      <td>honda</td>\n",
       "      <td>accord</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>accord</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>excellent</td>\n",
       "      <td>rwd</td>\n",
       "      <td>diesel</td>\n",
       "      <td>vancouver</td>\n",
       "      <td>110000</td>\n",
       "      <td>white</td>\n",
       "      <td>67000</td>\n",
       "      <td>owner</td>\n",
       "      <td>clean</td>\n",
       "      <td>automatic</td>\n",
       "      <td>...</td>\n",
       "      <td>2008</td>\n",
       "      <td>13</td>\n",
       "      <td>gmc</td>\n",
       "      <td>c7500</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>c7500</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>unknown</td>\n",
       "      <td>fwd</td>\n",
       "      <td>gas</td>\n",
       "      <td>vancouver</td>\n",
       "      <td>243000</td>\n",
       "      <td>unknown</td>\n",
       "      <td>4000</td>\n",
       "      <td>owner</td>\n",
       "      <td>clean</td>\n",
       "      <td>automatic</td>\n",
       "      <td>...</td>\n",
       "      <td>2003</td>\n",
       "      <td>18</td>\n",
       "      <td>acura</td>\n",
       "      <td>rsx</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>rsx</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>salvage</td>\n",
       "      <td>fwd</td>\n",
       "      <td>gas</td>\n",
       "      <td>vancouver</td>\n",
       "      <td>225000</td>\n",
       "      <td>black</td>\n",
       "      <td>1500</td>\n",
       "      <td>owner</td>\n",
       "      <td>salvage</td>\n",
       "      <td>automatic</td>\n",
       "      <td>...</td>\n",
       "      <td>2005</td>\n",
       "      <td>16</td>\n",
       "      <td>acura</td>\n",
       "      <td>rsx</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>rsx</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37336</th>\n",
       "      <td>excellent</td>\n",
       "      <td>fwd</td>\n",
       "      <td>gas</td>\n",
       "      <td>port coquitlam</td>\n",
       "      <td>28356</td>\n",
       "      <td>black</td>\n",
       "      <td>28995</td>\n",
       "      <td>dealer</td>\n",
       "      <td>clean</td>\n",
       "      <td>automatic</td>\n",
       "      <td>...</td>\n",
       "      <td>2020</td>\n",
       "      <td>1</td>\n",
       "      <td>mazda</td>\n",
       "      <td>mazda6</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>mazda6</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37337</th>\n",
       "      <td>excellent</td>\n",
       "      <td>fwd</td>\n",
       "      <td>gas</td>\n",
       "      <td>port coquitlam</td>\n",
       "      <td>23820</td>\n",
       "      <td>black</td>\n",
       "      <td>27995</td>\n",
       "      <td>dealer</td>\n",
       "      <td>clean</td>\n",
       "      <td>automatic</td>\n",
       "      <td>...</td>\n",
       "      <td>2020</td>\n",
       "      <td>1</td>\n",
       "      <td>toyota</td>\n",
       "      <td>camry</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>camry</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37338</th>\n",
       "      <td>excellent</td>\n",
       "      <td>4wd</td>\n",
       "      <td>gas</td>\n",
       "      <td>port coquitlam</td>\n",
       "      <td>129449</td>\n",
       "      <td>white</td>\n",
       "      <td>29800</td>\n",
       "      <td>dealer</td>\n",
       "      <td>clean</td>\n",
       "      <td>automatic</td>\n",
       "      <td>...</td>\n",
       "      <td>2016</td>\n",
       "      <td>5</td>\n",
       "      <td>ford</td>\n",
       "      <td>f-150</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>f-150</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37339</th>\n",
       "      <td>excellent</td>\n",
       "      <td>4wd</td>\n",
       "      <td>gas</td>\n",
       "      <td>port coquitlam</td>\n",
       "      <td>31512</td>\n",
       "      <td>silver</td>\n",
       "      <td>28995</td>\n",
       "      <td>dealer</td>\n",
       "      <td>clean</td>\n",
       "      <td>automatic</td>\n",
       "      <td>...</td>\n",
       "      <td>2020</td>\n",
       "      <td>1</td>\n",
       "      <td>ford</td>\n",
       "      <td>escape</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>escape</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37340</th>\n",
       "      <td>excellent</td>\n",
       "      <td>4wd</td>\n",
       "      <td>gas</td>\n",
       "      <td>port coquitlam</td>\n",
       "      <td>108275</td>\n",
       "      <td>white</td>\n",
       "      <td>18995</td>\n",
       "      <td>dealer</td>\n",
       "      <td>clean</td>\n",
       "      <td>automatic</td>\n",
       "      <td>...</td>\n",
       "      <td>2016</td>\n",
       "      <td>5</td>\n",
       "      <td>chevrolet</td>\n",
       "      <td>equinox</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>equinox</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>36929 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        condition drive     fuel        location  odometer paint color  price  \\\n",
       "0            good   4wd      gas       vancouver    281000       white   5500   \n",
       "1         unknown   fwd      gas       vancouver    140000     unknown    600   \n",
       "2       excellent   rwd   diesel       vancouver    110000       white  67000   \n",
       "3         unknown   fwd      gas       vancouver    243000     unknown   4000   \n",
       "4         salvage   fwd      gas       vancouver    225000       black   1500   \n",
       "...           ...   ...      ...             ...       ...         ...    ...   \n",
       "37336   excellent   fwd      gas  port coquitlam     28356       black  28995   \n",
       "37337   excellent   fwd      gas  port coquitlam     23820       black  27995   \n",
       "37338   excellent   4wd      gas  port coquitlam    129449       white  29800   \n",
       "37339   excellent   4wd      gas  port coquitlam     31512      silver  28995   \n",
       "37340   excellent   4wd      gas  port coquitlam    108275       white  18995   \n",
       "\n",
       "      sale type title status transmission  ...  year age      make_    model  \\\n",
       "0         owner      rebuilt    automatic  ...  2007  14        bmw       x3   \n",
       "1         owner        clean    automatic  ...  1984  37      honda   accord   \n",
       "2         owner        clean    automatic  ...  2008  13        gmc    c7500   \n",
       "3         owner        clean    automatic  ...  2003  18      acura      rsx   \n",
       "4         owner      salvage    automatic  ...  2005  16      acura      rsx   \n",
       "...         ...          ...          ...  ...   ...  ..        ...      ...   \n",
       "37336    dealer        clean    automatic  ...  2020   1      mazda   mazda6   \n",
       "37337    dealer        clean    automatic  ...  2020   1     toyota    camry   \n",
       "37338    dealer        clean    automatic  ...  2016   5       ford    f-150   \n",
       "37339    dealer        clean    automatic  ...  2020   1       ford   escape   \n",
       "37340    dealer        clean    automatic  ...  2016   5  chevrolet  equinox   \n",
       "\n",
       "       pos_words neg_words  low_text   model0 model1 model2  \n",
       "0          False     False     False       x3   None   None  \n",
       "1          False     False     False   accord   None   None  \n",
       "2          False     False     False    c7500   None   None  \n",
       "3           True     False     False      rsx   None   None  \n",
       "4          False     False     False      rsx   None   None  \n",
       "...          ...       ...       ...      ...    ...    ...  \n",
       "37336      False     False     False   mazda6   None   None  \n",
       "37337      False     False     False    camry   None   None  \n",
       "37338      False     False     False    f-150   None   None  \n",
       "37339      False     False     False   escape   None   None  \n",
       "37340      False     False     False  equinox   None   None  \n",
       "\n",
       "[36929 rows x 26 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "import lightgbm as lgb\n",
    "from lightgbm import LGBMRegressor\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "#read in data\n",
    "data = pd.read_csv('cars_cleaned.csv')\n",
    "\n",
    "#### missing values #####\n",
    "data.dropna(inplace=True)#there are only a few dozen missing values so drop those rows\n",
    "\n",
    "#cleaning\n",
    "data = data[data.odometer<1e6] #drop rows where the odometer is very high\n",
    "data = data[data.price<1e5] #drop rows where the price is very high\n",
    "\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### create training and validation data ####\n",
    "X = data.copy()\n",
    "y = X.pop('price')\n",
    "\n",
    "#locating categorical and numeric columns\n",
    "cat_cols = X.columns[X.dtypes.values=='O']#categorical columns\n",
    "numeric_cols = [col for col in X.columns if col not in cat_cols] #the rest are numeric\n",
    "\n",
    "for col in cat_cols:\n",
    "    X[col] = X[col].astype('category')#change data type so it can be handled directly by lightgbm\n",
    "\n",
    "#split training and test data\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, y, test_size=0.2, random_state=41)#hold out 20% of data for testing\n",
    "\n",
    "#The parameters for the model were found by a grid search. The code below demonstrates the grid seach for 'num_leaves'\n",
    "model = LGBMRegressor(num_leaves = 200, \n",
    "                      min_child_samples=10,\n",
    "                      random_state = 0, \n",
    "                      n_estimators = 1000, \n",
    "                      metric='l1')#Mean absolute error metric\n",
    "\n",
    "params = {\n",
    "    'num_leaves': [50, 100, 200, 400, 800],#repeat grid search over number of leaves to find optimum number\n",
    "         }\n",
    "\n",
    "searchCV = GridSearchCV(model, scoring='neg_mean_absolute_error', cv=5, param_grid=params, return_train_score=True)\n",
    "print('GridSearchCV fitting...')\n",
    "searchCV.fit(X_train, y_train)\n",
    "scores = -1*searchCV.cv_results_['mean_test_score']\n",
    "train_scores = -1*searchCV.cv_results_['mean_train_score']\n",
    "params_out = searchCV.cv_results_['params']\n",
    "for i in range(0, len(scores)):\n",
    "  print(params_out[i], '->', scores[i])\n",
    "\n",
    "x = params['num_leaves']\n",
    "plt.plot(x, scores, label = 'test score')\n",
    "plt.plot(x, train_scores, label = 'train score')\n",
    "plt.title('Effect of leaf number on model performance')\n",
    "plt.xlabel('Number of Leaves')\n",
    "plt.ylabel('Mean Absolute Error (5-fold cross-validation)')\n",
    "plt.legend()\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#we will now use the paramters found in the last step to fit the model to the training data and observe \n",
    "#the train/test score over the number of boosting iterations\n",
    "evals_result={}\n",
    "\n",
    "params = {#model paramters\n",
    "    'task': 'train',\n",
    "    'num_leaves': 200,\n",
    "    'min_child_samples':10,\n",
    "    'verbose':-1, \n",
    "    'n_estimators':5000,#number of boosting rounds\n",
    "    'metric': 'l1',\n",
    "    'random_state':0,}\n",
    "\n",
    "train = lgb.Dataset(X_train, y_train)#training and testing dataset\n",
    "test = lgb.Dataset(X_test, y_test)\n",
    "\n",
    "#fit on training data with early stopping\n",
    "model = lgb.train(params, train, early_stopping_rounds=25, valid_sets= [train, test], evals_result=evals_result, verbose_eval=False)\n",
    "\n",
    "#final training prediction and errer\n",
    "y_pred = model.predict(X_train)\n",
    "mae = mean_absolute_error(y_train, y_pred)\n",
    "print('train score:', mae)\n",
    "#final testing prediction and error\n",
    "y_pred = model.predict(X_test)\n",
    "mae = mean_absolute_error(y_test, y_pred)\n",
    "print('test score:', mae)\n",
    "\n",
    "#plot the error throughout the fitting process\n",
    "lgb.plot_metric(evals_result, metric='l1')\n",
    "plt.yscale('log')\n",
    "plt.xscale('log')\n",
    "plt.title('Mean absolute error during training')\n",
    "plt.ylabel('Mean Absolute Error of Testing Set')\n",
    "plt.ylim(6e1, 2e4)\n",
    "plt.xlim(1, 10000)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\a_sylvester\\Anaconda3\\envs\\cars\\lib\\site-packages\\lightgbm\\engine.py:151: UserWarning: Found `n_estimators` in params. Will use it instead of argument\n",
      "  warnings.warn(\"Found `{}` in params. Will use it instead of argument\".format(alias))\n"
     ]
    }
   ],
   "source": [
    "#As seen above, after 5000 boosting iterations, the validation score is no longer improving so we can use that in our final model\n",
    "#The final step is to fit our model to ALL of the data with the parameters we have found above.\n",
    "\n",
    "train_final = lgb.Dataset(X, y)#final training dataset is all the data\n",
    "model = lgb.train(params, train_final)#fit the final model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import bz2\n",
    "import pickle\n",
    "import _pickle as cPickle\n",
    "\n",
    "#Compressed pickle functions from: https://betterprogramming.pub/load-fast-load-big-with-compressed-pickles-5f311584507e\n",
    "\n",
    "pkl = {'model': model}\n",
    "\n",
    "# Pickle a file and then compress it into a file with extension \n",
    "def compressed_pickle(title, data):\n",
    "    with bz2.BZ2File(title + '.pbz2', 'w') as f: \n",
    "        cPickle.dump(data, f)\n",
    "\n",
    "# Load any compressed pickle file\n",
    "def decompress_pickle(file):\n",
    "    data = bz2.BZ2File(file, 'rb')\n",
    "    data = cPickle.load(data)\n",
    "    return data\n",
    "\n",
    "compressed_pickle('model_file', pkl)\n",
    "model = decompress_pickle('model_file.pbz2')['model']"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}