data-cleaning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "initial_id",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-22T15:29:12.273845Z",
     "start_time": "2024-12-22T15:29:12.269681Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "\n",
    "sys.path.append(\"./\")  # Adds higher directory to python modules path.\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1e07c47cc005ee46",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-22T15:29:14.359333Z",
     "start_time": "2024-12-22T15:29:12.318734Z"
    }
   },
   "outputs": [],
   "source": [
    "from core.ODEDataset import ODEDataset\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "pd.set_option('future.no_silent_downcasting', True)\n",
    "from utils import cleaning, constants\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d03130217c47f2b3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-22T15:29:14.390933Z",
     "start_time": "2024-12-22T15:29:14.388756Z"
    }
   },
   "outputs": [],
   "source": [
    "src_path = \"./data/combined-datasets.csv\"\n",
    "missing_value = -1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "79f0638b1715b622",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-22T15:29:14.455480Z",
     "start_time": "2024-12-22T15:29:14.436138Z"
    }
   },
   "outputs": [],
   "source": [
    "dataset = ODEDataset(\"combined\")\n",
    "dataset.from_csv(src_path)\n",
    "dataset = dataset.select([\n",
    "                             \"ID\",\n",
    "                         ] + constants.DRIVERS_LIST + constants.PRESENCE_LIST)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "218f1e57",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import numpy as np\n",
    "# from matplotlib import pyplot as plt\n",
    "\n",
    "# df = dataset.to_dataframe()\n",
    "\n",
    "# original = df['Years_of_HHH_in_community']\n",
    "\n",
    "# df['Years_of_HHH_in_community'] = df['Years_of_HHH_in_community'].interpolate()\n",
    "# mean = df['Years_of_HHH_in_community'].mean()\n",
    "# df['Years_of_HHH_in_community'] = df['Years_of_HHH_in_community'].fillna(mean)\n",
    "# inerpolated = df['Years_of_HHH_in_community']\n",
    "# # Create x-values for plotting\n",
    "# x = range(len(df))\n",
    "\n",
    "# plt.figure(figsize=(20, 15))\n",
    "# plt.scatter(x, original, label='Before Interpolation', marker='o', color='r')\n",
    "# plt.scatter(x, inerpolated, label='After Interpolation', marker='+', color='b')\n",
    "\n",
    "# plt.legend()\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "191c976ca2a79bcc",
   "metadata": {},
   "source": [
    "# Drivers"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf863e540f4f71fe",
   "metadata": {},
   "source": [
    "## Years_of_HHH_in_community"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5f55ce2de3b34430",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-22T15:29:14.544442Z",
     "start_time": "2024-12-22T15:29:14.491492Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Years_of_HHH_in_community</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.476160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.476160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.477477</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.477477</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.483108</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Years_of_HHH_in_community\n",
       "0                  -0.476160\n",
       "1                  -0.476160\n",
       "2                  -0.477477\n",
       "3                  -0.477477\n",
       "4                  -0.483108"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "dataset = dataset.apply(cleaning.astype(\"Years_of_HHH_in_community\", np.float32))\n",
    "dataset = dataset.apply(cleaning.interpolate(\"Years_of_HHH_in_community\"))\n",
    "dataset = dataset.apply(cleaning.fillna_with_mean(\"Years_of_HHH_in_community\"))\n",
    "\n",
    "dataset = dataset.apply(cleaning.standardize(\"Years_of_HHH_in_community\"))\n",
    "dataset.to_dataframe()[[\"Years_of_HHH_in_community\"]].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c5210735a30e773",
   "metadata": {},
   "source": [
    "## Dwelling_quality_index\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c40bcfc9d983bb09",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-22T15:29:14.649781Z",
     "start_time": "2024-12-22T15:29:14.605632Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dwelling_quality_index\n",
       "-0.500000    5222\n",
       "-0.364865    1332\n",
       "-0.391892    1112\n",
       "-0.418919     930\n",
       "-0.445946     720\n",
       "             ... \n",
       "-0.475430       1\n",
       "-0.480344       1\n",
       "-0.485258       1\n",
       "-0.490172       1\n",
       " 0.013514       1\n",
       "Name: count, Length: 62, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "dataset = dataset.apply(cleaning.astype(\"Dwelling_quality_index\", np.float32))\n",
    "dataset = dataset.apply(cleaning.interpolate(\"Dwelling_quality_index\"))\n",
    "dataset = dataset.apply(cleaning.fillna_with_mean(\"Dwelling_quality_index\"))\n",
    "\n",
    "dataset = dataset.apply(cleaning.standardize(\"Dwelling_quality_index\"))\n",
    "\n",
    "dataset.value_counts(\"Dwelling_quality_index\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d65228128f0106b7",
   "metadata": {},
   "source": [
    "## Hours_available_electricity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b65b79fa63cf0727",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-22T15:29:14.815714Z",
     "start_time": "2024-12-22T15:29:14.728644Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Hours_available_electricity\n",
       "-0.464286    2753\n",
       "-0.468750    1817\n",
       "-0.477679     516\n",
       "-0.491071     346\n",
       "-0.473214     255\n",
       "             ... \n",
       "-0.464379       1\n",
       "-0.464410       1\n",
       "-0.464441       1\n",
       "-0.464472       1\n",
       " 0.013393       1\n",
       "Name: count, Length: 1889, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "\n",
    "dataset = dataset.apply(cleaning.astype(\"Hours_available_electricity\", np.float32))\n",
    "dataset = dataset.apply(cleaning.interpolate(\"Hours_available_electricity\"))\n",
    "dataset = dataset.apply(cleaning.fillna_with_mean(\"Hours_available_electricity\"))\n",
    "\n",
    "dataset = dataset.apply(cleaning.standardize(\"Hours_available_electricity\"))\n",
    "\n",
    "dataset.value_counts(\"Hours_available_electricity\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bd3ba5884dce0110",
   "metadata": {},
   "source": [
    "## Measurement_age"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "79cc4a11f5f02eb2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-22T15:29:14.915219Z",
     "start_time": "2024-12-22T15:29:14.865837Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Measurement_age\n",
       "-0.398683    1372\n",
       "-0.395643     927\n",
       "-0.396657     902\n",
       "-0.397670     599\n",
       "-0.389564     231\n",
       "             ... \n",
       "-0.396518       1\n",
       "-0.396564       1\n",
       "-0.396610       1\n",
       "-0.396698       1\n",
       "-0.389058       1\n",
       "Name: count, Length: 3403, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = dataset.apply(cleaning.astype(\"Measurement_age\", np.float32))\n",
    "dataset = dataset.apply(cleaning.interpolate(\"Measurement_age\"))\n",
    "dataset = dataset.apply(cleaning.fillna_with_mean(\"Measurement_age\"))\n",
    "\n",
    "dataset = dataset.apply(cleaning.standardize(\"Measurement_age\"))\n",
    "\n",
    "dataset.value_counts(\"Measurement_age\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d41d7e873a7f4fb4",
   "metadata": {},
   "source": [
    "## Monthly_expenditure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "9e2775d3d2c2fadf",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-22T15:29:15.014206Z",
     "start_time": "2024-12-22T15:29:14.963354Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Monthly_expenditure\n",
       "-0.500000    389\n",
       "-0.499996     96\n",
       "-0.499992     81\n",
       "-0.499998     70\n",
       "-0.499995     63\n",
       "            ... \n",
       "-0.499942      1\n",
       "-0.499822      1\n",
       "-0.499887      1\n",
       "-0.499881      1\n",
       "-0.499493      1\n",
       "Name: count, Length: 5081, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = dataset.apply(cleaning.astype(\"Monthly_expenditure\", np.float32))\n",
    "dataset = dataset.apply(cleaning.interpolate(\"Monthly_expenditure\"))\n",
    "dataset = dataset.apply(cleaning.fillna_with_mean(\"Monthly_expenditure\"))\n",
    "\n",
    "dataset = dataset.apply(cleaning.standardize(\"Monthly_expenditure\"))\n",
    "\n",
    "dataset.value_counts(\"Monthly_expenditure\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "63927f187164c2a6",
   "metadata": {},
   "source": [
    "## Number_of_rooms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7603a87d2ec70b21",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-22T15:29:15.168404Z",
     "start_time": "2024-12-22T15:29:15.121078Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Number_of_rooms\n",
       "-0.455556    3377\n",
       "-0.433333    2650\n",
       "-0.477778    2456\n",
       "-0.411111    1101\n",
       "-0.388889     455\n",
       "             ... \n",
       "-0.451587       1\n",
       "-0.452381       1\n",
       "-0.453175       1\n",
       "-0.453968       1\n",
       "-0.388109       1\n",
       "Name: count, Length: 175, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = dataset.apply(cleaning.astype(\"Number_of_rooms\", np.float32))\n",
    "dataset = dataset.apply(cleaning.interpolate(\"Number_of_rooms\"))\n",
    "dataset = dataset.apply(cleaning.fillna_with_mean(\"Number_of_rooms\"))\n",
    "\n",
    "dataset = dataset.apply(cleaning.standardize(\"Number_of_rooms\"))\n",
    "\n",
    "dataset.value_counts(\"Number_of_rooms\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "37d5357c66b10d44",
   "metadata": {},
   "source": [
    "## Climate_zone_lev_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "e5f87ff91d9c4bc4",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-22T15:29:15.283677Z",
     "start_time": "2024-12-22T15:29:15.227424Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Climate_zone_lev_1\n",
       " 0.019460    2976\n",
       "-0.396108    1987\n",
       "-0.098379    1089\n",
       "-0.196090     982\n",
       "-0.024006     707\n",
       " 0.208739     624\n",
       "-0.240270     356\n",
       "-0.084432     223\n",
       "-0.276476     204\n",
       " 0.227244     195\n",
       " 0.267749     192\n",
       " 0.199388     180\n",
       " 0.194557     132\n",
       "-0.448054     129\n",
       "-0.344162     124\n",
       " 0.071406     120\n",
       "-0.500000     110\n",
       " 0.057887     104\n",
       " 0.058602      96\n",
       " 0.078367      60\n",
       " 0.011889      48\n",
       " 0.429340      27\n",
       " 0.289359      23\n",
       " 0.490364       5\n",
       " 0.395550       3\n",
       " 0.162961       3\n",
       " 0.450924       2\n",
       " 0.300345       2\n",
       "-0.474027       2\n",
       " 0.500000       1\n",
       "-0.478356       1\n",
       "-0.456712       1\n",
       "-0.461040       1\n",
       "-0.465369       1\n",
       "-0.469698       1\n",
       "-0.495671       1\n",
       "-0.482685       1\n",
       "-0.487013       1\n",
       "-0.491342       1\n",
       " 0.378420       1\n",
       " 0.486612       1\n",
       "-0.452383       1\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "dataset = dataset.apply(cleaning.astype(\"Climate_zone_lev_1\", np.float32))\n",
    "dataset = dataset.apply(cleaning.interpolate(\"Climate_zone_lev_1\"))\n",
    "dataset = dataset.apply(cleaning.fillna_with_mean(\"Climate_zone_lev_1\"))\n",
    "\n",
    "dataset = dataset.apply(cleaning.standardize(\"Climate_zone_lev_1\"))\n",
    "\n",
    "dataset.value_counts(\"Climate_zone_lev_1\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8620d468fed9619",
   "metadata": {},
   "source": [
    "## Climate_zone_lev_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "950e4b4fbcd6219",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-22T15:29:15.495923Z",
     "start_time": "2024-12-22T15:29:15.446750Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Climate_zone_lev_2\n",
       "-0.397514    1095\n",
       " 0.063671     319\n",
       "-0.243786     211\n",
       "-0.090058     191\n",
       "-0.448757     182\n",
       "             ... \n",
       "-0.292938       1\n",
       "-0.292981       1\n",
       "-0.293023       1\n",
       "-0.293066       1\n",
       "-0.434116       1\n",
       "Name: count, Length: 7849, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = dataset.apply(cleaning.astype(\"Climate_zone_lev_2\", np.float32))\n",
    "dataset = dataset.apply(cleaning.interpolate(\"Climate_zone_lev_2\"))\n",
    "dataset = dataset.apply(cleaning.fillna_with_mean(\"Climate_zone_lev_2\"))\n",
    "\n",
    "dataset = dataset.apply(cleaning.standardize(\"Climate_zone_lev_2\"))\n",
    "\n",
    "dataset.value_counts(\"Climate_zone_lev_2\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e41f0d2c11be173f",
   "metadata": {},
   "source": [
    "## Age_HHH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3c4edfa9a34b5714",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-22T15:29:15.604212Z",
     "start_time": "2024-12-22T15:29:15.556486Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Age_HHH\n",
       "-0.157647    1274\n",
       "-0.259036     557\n",
       "-0.198795     480\n",
       "-0.138554     447\n",
       "-0.319277     391\n",
       "             ... \n",
       " 0.475904       1\n",
       "-0.144578       1\n",
       "-0.156626       1\n",
       "-0.289157       1\n",
       "-0.100402       1\n",
       "Name: count, Length: 193, dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = dataset.apply(cleaning.astype(\"Age_HHH\", np.float32))\n",
    "dataset = dataset.apply(cleaning.interpolate(\"Age_HHH\"))\n",
    "dataset = dataset.apply(cleaning.fillna_with_mean(\"Age_HHH\"))\n",
    "\n",
    "dataset = dataset.apply(cleaning.standardize(\"Age_HHH\"))\n",
    "\n",
    "dataset.value_counts(\"Age_HHH\")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "66b966b107cdb15d",
   "metadata": {},
   "source": [
    "## HH_with_home_business"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "cfb6707e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def interpolate_and_fillna_with_mean_and_standarize(dataset, column):\n",
    "    \n",
    "    cols = [c for c in dataset.get_columns() if column in c]\n",
    "    for c in cols:\n",
    "        dataset = dataset.apply(cleaning.astype(c, np.float32))\n",
    "        dataset = dataset.apply(cleaning.interpolate(c))\n",
    "        dataset = dataset.apply(cleaning.fillna_with_mean(c))\n",
    "        dataset = dataset.apply(cleaning.standardize(c))\n",
    "        \n",
    "        \n",
    "    return dataset\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "dabe1d5674b5a47e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-22T15:29:15.725934Z",
     "start_time": "2024-12-22T15:29:15.715247Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Years_of_HHH_in_community</th>\n",
       "      <th>Dwelling_quality_index</th>\n",
       "      <th>Hours_available_electricity</th>\n",
       "      <th>Measurement_age</th>\n",
       "      <th>Monthly_expenditure</th>\n",
       "      <th>Number_of_rooms</th>\n",
       "      <th>Climate_zone_lev_1</th>\n",
       "      <th>Climate_zone_lev_2</th>\n",
       "      <th>Age_HHH</th>\n",
       "      <th>...</th>\n",
       "      <th>Tariff_payment_frequency</th>\n",
       "      <th>Presence_refrigerator/freezer</th>\n",
       "      <th>Presence_iron</th>\n",
       "      <th>Presence_fan</th>\n",
       "      <th>Presence_DVD_player</th>\n",
       "      <th>Presence_radio/stereo</th>\n",
       "      <th>Presence_phone_charger</th>\n",
       "      <th>Presence_TV</th>\n",
       "      <th>HH_with_home_business_No</th>\n",
       "      <th>HH_with_home_business_Yes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1001_3</td>\n",
       "      <td>-0.476160</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.464103</td>\n",
       "      <td>-0.394716</td>\n",
       "      <td>-0.496128</td>\n",
       "      <td>-0.433333</td>\n",
       "      <td>-0.396108</td>\n",
       "      <td>-0.155227</td>\n",
       "      <td>-0.157647</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1001_3</td>\n",
       "      <td>-0.476160</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.464103</td>\n",
       "      <td>-0.394716</td>\n",
       "      <td>-0.496128</td>\n",
       "      <td>-0.433333</td>\n",
       "      <td>-0.396108</td>\n",
       "      <td>-0.155227</td>\n",
       "      <td>-0.157647</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1026_3</td>\n",
       "      <td>-0.477477</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.464103</td>\n",
       "      <td>-0.394716</td>\n",
       "      <td>-0.496019</td>\n",
       "      <td>-0.433333</td>\n",
       "      <td>-0.396108</td>\n",
       "      <td>-0.397514</td>\n",
       "      <td>-0.157647</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1026_3</td>\n",
       "      <td>-0.477477</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.464103</td>\n",
       "      <td>-0.394716</td>\n",
       "      <td>-0.496019</td>\n",
       "      <td>-0.433333</td>\n",
       "      <td>-0.396108</td>\n",
       "      <td>-0.397514</td>\n",
       "      <td>-0.157647</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1027_2</td>\n",
       "      <td>-0.483108</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.464103</td>\n",
       "      <td>-0.394716</td>\n",
       "      <td>-0.497578</td>\n",
       "      <td>-0.455556</td>\n",
       "      <td>-0.396108</td>\n",
       "      <td>-0.397514</td>\n",
       "      <td>-0.157647</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       ID  Years_of_HHH_in_community  Dwelling_quality_index  \\\n",
       "0  1001_3                  -0.476160                    -0.5   \n",
       "1  1001_3                  -0.476160                    -0.5   \n",
       "2  1026_3                  -0.477477                    -0.5   \n",
       "3  1026_3                  -0.477477                    -0.5   \n",
       "4  1027_2                  -0.483108                    -0.5   \n",
       "\n",
       "   Hours_available_electricity  Measurement_age  Monthly_expenditure  \\\n",
       "0                    -0.464103        -0.394716            -0.496128   \n",
       "1                    -0.464103        -0.394716            -0.496128   \n",
       "2                    -0.464103        -0.394716            -0.496019   \n",
       "3                    -0.464103        -0.394716            -0.496019   \n",
       "4                    -0.464103        -0.394716            -0.497578   \n",
       "\n",
       "   Number_of_rooms  Climate_zone_lev_1  Climate_zone_lev_2   Age_HHH  ...  \\\n",
       "0        -0.433333           -0.396108           -0.155227 -0.157647  ...   \n",
       "1        -0.433333           -0.396108           -0.155227 -0.157647  ...   \n",
       "2        -0.433333           -0.396108           -0.397514 -0.157647  ...   \n",
       "3        -0.433333           -0.396108           -0.397514 -0.157647  ...   \n",
       "4        -0.455556           -0.396108           -0.397514 -0.157647  ...   \n",
       "\n",
       "  Tariff_payment_frequency Presence_refrigerator/freezer  Presence_iron  \\\n",
       "0                      NaN                           0.0            0.0   \n",
       "1                      NaN                           0.0            0.0   \n",
       "2                      NaN                           NaN            NaN   \n",
       "3                      NaN                           NaN            NaN   \n",
       "4                      NaN                           NaN            NaN   \n",
       "\n",
       "  Presence_fan Presence_DVD_player Presence_radio/stereo  \\\n",
       "0          0.0                 0.0                   0.0   \n",
       "1          0.0                 0.0                   0.0   \n",
       "2          NaN                 NaN                   NaN   \n",
       "3          NaN                 NaN                   NaN   \n",
       "4          NaN                 NaN                   NaN   \n",
       "\n",
       "  Presence_phone_charger Presence_TV  HH_with_home_business_No  \\\n",
       "0                    1.0         0.0                      -0.5   \n",
       "1                    1.0         0.0                      -0.5   \n",
       "2                    0.0         0.0                       0.5   \n",
       "3                    0.0         0.0                       0.5   \n",
       "4                    0.0         0.0                      -0.5   \n",
       "\n",
       "   HH_with_home_business_Yes  \n",
       "0                        0.5  \n",
       "1                        0.5  \n",
       "2                       -0.5  \n",
       "3                       -0.5  \n",
       "4                        0.5  \n",
       "\n",
       "[5 rows x 27 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = dataset.apply(cleaning.astype(\"HH_with_home_business\", str))\n",
    "\n",
    "# add_one_hot_encoding\n",
    "dataset = dataset.apply(cleaning.add_one_hot_encoding(\"HH_with_home_business\"))\n",
    "\n",
    "dataset = interpolate_and_fillna_with_mean_and_standarize(dataset, \"HH_with_home_business\")\n",
    "\n",
    "dataset = dataset.drop_columns([\"HH_with_home_business_nan\"])\n",
    "\n",
    "dataset.preview()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d313bfd9b915b92",
   "metadata": {},
   "source": [
    "## Education_level_HHH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "5f0400ff816f9214",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-22T15:29:15.790261Z",
     "start_time": "2024-12-22T15:29:15.745625Z"
    }
   },
   "outputs": [],
   "source": [
    "dataset = dataset.apply(cleaning.fillna(\"Education_level_HHH\", \"Other\"))\n",
    "merged_categories = {\n",
    "    'Other': \"Not classified\",\n",
    "    \"Not elsewhere classified\": \"Not classified\",\n",
    "\n",
    "    \"No schooling\": \"No schooling\",\n",
    "\n",
    "    'Primary education': \"Primary\",\n",
    "\n",
    "    'Upper secondary education': \"Secondary\",\n",
    "    \"Secondary education\": \"Secondary\",\n",
    "    'Lower secondary education': \"Secondary\",\n",
    "\n",
    "    'Post-secondary non-tertiary education': \"Tertiary\",\n",
    "    \"Bachelor's or equivalent level\": \"Tertiary\",\n",
    "    \"Short-cycle tertiary education\": \"Tertiary\",\n",
    "    \"Doctoral or equivalent level\": \"Tertiary\",\n",
    "    \"Master's or equivalent level\": \"Tertiary\",\n",
    "\n",
    "}\n",
    "dataset = dataset.new_feature(\"Education_level_HHH\",cleaning.merge_categories(\"Education_level_HHH\", merged_categories))\n",
    "dataset = dataset.apply(cleaning.add_one_hot_encoding(\"Education_level_HHH\"))\n",
    "\n",
    "dataset = interpolate_and_fillna_with_mean_and_standarize(dataset, \"Education_level_HHH\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "732add2aef779815",
   "metadata": {},
   "source": [
    "## Socio_status_HHH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "45263d2f5330cf06",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-22T15:29:15.911303Z",
     "start_time": "2024-12-22T15:29:15.860707Z"
    }
   },
   "outputs": [],
   "source": [
    "dataset = dataset.apply(cleaning.fillna(\"Socio_status_HHH\", \"Other (not specified in Socio_status)\"))\n",
    "\n",
    "dataset = dataset.apply(cleaning.replace_value(\"Socio_status_HHH\", \"Other (not specified in Socio_status)\", \"Other\"))\n",
    "\n",
    "merged_categories = {\n",
    "    'Own-account worker farm': \"Worker\",\n",
    "    'Employee': \"Worker\",\n",
    "    'Unemployed': \"Unemployed\",\n",
    "    \"Other\": \"Other\",\n",
    "\n",
    "    'Employee non-farm': \"Worker\",\n",
    "    'Own-account worker non-farm': \"Worker\",\n",
    "    'Contributing family worker': \"Worker\",\n",
    "    'Worker not classifiable by status': \"Worker\",\n",
    "    'Employee farm': \"Worker\",\n",
    "\n",
    "    'Employer': \"Employer\",\n",
    "}\n",
    "dataset = dataset.new_feature(\"Socio_status_HHH\",cleaning.merge_categories(\"Socio_status_HHH\", merged_categories))\n",
    "dataset = dataset.apply(cleaning.add_one_hot_encoding(\"Socio_status_HHH\"))\n",
    "\n",
    "dataset = interpolate_and_fillna_with_mean_and_standarize(dataset, \"Socio_status_HHH\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1f20c6f90b6fae6b",
   "metadata": {},
   "source": [
    " ## Number_adults\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "cacbfc4ad3da90db",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-22T15:29:15.999253Z",
     "start_time": "2024-12-22T15:29:15.951930Z"
    }
   },
   "outputs": [],
   "source": [
    "dataset = dataset.apply(cleaning.astype(\"Number_adults\", np.float32))\n",
    "dataset = dataset.apply(cleaning.interpolate(\"Number_adults\"))\n",
    "dataset = dataset.apply(cleaning.fillna_with_mean(\"Number_adults\"))\n",
    "\n",
    "dataset = dataset.apply(cleaning.standardize(\"Number_adults\"))\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5b36d120cdaca549",
   "metadata": {},
   "source": [
    "## Ownership_motorized_vehicle\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "8d265d50dbeafd79",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-22T15:29:16.037758Z",
     "start_time": "2024-12-22T15:29:16.024965Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Years_of_HHH_in_community</th>\n",
       "      <th>Dwelling_quality_index</th>\n",
       "      <th>Hours_available_electricity</th>\n",
       "      <th>Measurement_age</th>\n",
       "      <th>Monthly_expenditure</th>\n",
       "      <th>Number_of_rooms</th>\n",
       "      <th>Climate_zone_lev_1</th>\n",
       "      <th>Climate_zone_lev_2</th>\n",
       "      <th>Age_HHH</th>\n",
       "      <th>...</th>\n",
       "      <th>Education_level_HHH_Not classified</th>\n",
       "      <th>Education_level_HHH_Primary</th>\n",
       "      <th>Education_level_HHH_Secondary</th>\n",
       "      <th>Education_level_HHH_Tertiary</th>\n",
       "      <th>Socio_status_HHH_Employer</th>\n",
       "      <th>Socio_status_HHH_Other</th>\n",
       "      <th>Socio_status_HHH_Unemployed</th>\n",
       "      <th>Socio_status_HHH_Worker</th>\n",
       "      <th>Ownership_motorized_vehicle_No</th>\n",
       "      <th>Ownership_motorized_vehicle_Yes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1001_3</td>\n",
       "      <td>-0.476160</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.464103</td>\n",
       "      <td>-0.394716</td>\n",
       "      <td>-0.496128</td>\n",
       "      <td>-0.433333</td>\n",
       "      <td>-0.396108</td>\n",
       "      <td>-0.155227</td>\n",
       "      <td>-0.157647</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1001_3</td>\n",
       "      <td>-0.476160</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.464103</td>\n",
       "      <td>-0.394716</td>\n",
       "      <td>-0.496128</td>\n",
       "      <td>-0.433333</td>\n",
       "      <td>-0.396108</td>\n",
       "      <td>-0.155227</td>\n",
       "      <td>-0.157647</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1026_3</td>\n",
       "      <td>-0.477477</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.464103</td>\n",
       "      <td>-0.394716</td>\n",
       "      <td>-0.496019</td>\n",
       "      <td>-0.433333</td>\n",
       "      <td>-0.396108</td>\n",
       "      <td>-0.397514</td>\n",
       "      <td>-0.157647</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1026_3</td>\n",
       "      <td>-0.477477</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.464103</td>\n",
       "      <td>-0.394716</td>\n",
       "      <td>-0.496019</td>\n",
       "      <td>-0.433333</td>\n",
       "      <td>-0.396108</td>\n",
       "      <td>-0.397514</td>\n",
       "      <td>-0.157647</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1027_2</td>\n",
       "      <td>-0.483108</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.464103</td>\n",
       "      <td>-0.394716</td>\n",
       "      <td>-0.497578</td>\n",
       "      <td>-0.455556</td>\n",
       "      <td>-0.396108</td>\n",
       "      <td>-0.397514</td>\n",
       "      <td>-0.157647</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 35 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       ID  Years_of_HHH_in_community  Dwelling_quality_index  \\\n",
       "0  1001_3                  -0.476160                    -0.5   \n",
       "1  1001_3                  -0.476160                    -0.5   \n",
       "2  1026_3                  -0.477477                    -0.5   \n",
       "3  1026_3                  -0.477477                    -0.5   \n",
       "4  1027_2                  -0.483108                    -0.5   \n",
       "\n",
       "   Hours_available_electricity  Measurement_age  Monthly_expenditure  \\\n",
       "0                    -0.464103        -0.394716            -0.496128   \n",
       "1                    -0.464103        -0.394716            -0.496128   \n",
       "2                    -0.464103        -0.394716            -0.496019   \n",
       "3                    -0.464103        -0.394716            -0.496019   \n",
       "4                    -0.464103        -0.394716            -0.497578   \n",
       "\n",
       "   Number_of_rooms  Climate_zone_lev_1  Climate_zone_lev_2   Age_HHH  ...  \\\n",
       "0        -0.433333           -0.396108           -0.155227 -0.157647  ...   \n",
       "1        -0.433333           -0.396108           -0.155227 -0.157647  ...   \n",
       "2        -0.433333           -0.396108           -0.397514 -0.157647  ...   \n",
       "3        -0.433333           -0.396108           -0.397514 -0.157647  ...   \n",
       "4        -0.455556           -0.396108           -0.397514 -0.157647  ...   \n",
       "\n",
       "   Education_level_HHH_Not classified Education_level_HHH_Primary  \\\n",
       "0                                -0.5                         0.5   \n",
       "1                                -0.5                         0.5   \n",
       "2                                -0.5                         0.5   \n",
       "3                                -0.5                         0.5   \n",
       "4                                -0.5                        -0.5   \n",
       "\n",
       "  Education_level_HHH_Secondary Education_level_HHH_Tertiary  \\\n",
       "0                          -0.5                         -0.5   \n",
       "1                          -0.5                         -0.5   \n",
       "2                          -0.5                         -0.5   \n",
       "3                          -0.5                         -0.5   \n",
       "4                           0.5                         -0.5   \n",
       "\n",
       "  Socio_status_HHH_Employer  Socio_status_HHH_Other  \\\n",
       "0                      -0.5                    -0.5   \n",
       "1                      -0.5                    -0.5   \n",
       "2                      -0.5                    -0.5   \n",
       "3                      -0.5                    -0.5   \n",
       "4                      -0.5                    -0.5   \n",
       "\n",
       "   Socio_status_HHH_Unemployed  Socio_status_HHH_Worker  \\\n",
       "0                         -0.5                      0.5   \n",
       "1                         -0.5                      0.5   \n",
       "2                          0.5                     -0.5   \n",
       "3                          0.5                     -0.5   \n",
       "4                         -0.5                      0.5   \n",
       "\n",
       "   Ownership_motorized_vehicle_No  Ownership_motorized_vehicle_Yes  \n",
       "0                             0.5                             -0.5  \n",
       "1                             0.5                             -0.5  \n",
       "2                            -0.5                              0.5  \n",
       "3                            -0.5                              0.5  \n",
       "4                            -0.5                              0.5  \n",
       "\n",
       "[5 rows x 35 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = dataset.apply(cleaning.replace_value(\"Ownership_motorized_vehicle\", \"1.0\", 'Yes'))\n",
    "dataset = dataset.apply(cleaning.replace_value(\"Ownership_motorized_vehicle\", \"0.0\", 'No'))\n",
    "\n",
    "dataset = dataset.apply(cleaning.astype(\"Ownership_motorized_vehicle\", str))\n",
    "\n",
    "# add_one_hot_encoding\n",
    "dataset = dataset.apply(cleaning.add_one_hot_encoding(\"Ownership_motorized_vehicle\"))\n",
    "\n",
    "dataset = interpolate_and_fillna_with_mean_and_standarize(dataset, \"Ownership_motorized_vehicle\")\n",
    "\n",
    "dataset = dataset.drop_columns([\"Ownership_motorized_vehicle_nan\"])\n",
    "\n",
    "dataset.preview()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5f929c88ca41b3b6",
   "metadata": {},
   "source": [
    "## Ownership_small_livestock\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "9342fd48fa5ce994",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-22T15:29:16.094840Z",
     "start_time": "2024-12-22T15:29:16.080094Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Years_of_HHH_in_community</th>\n",
       "      <th>Dwelling_quality_index</th>\n",
       "      <th>Hours_available_electricity</th>\n",
       "      <th>Measurement_age</th>\n",
       "      <th>Monthly_expenditure</th>\n",
       "      <th>Number_of_rooms</th>\n",
       "      <th>Climate_zone_lev_1</th>\n",
       "      <th>Climate_zone_lev_2</th>\n",
       "      <th>Age_HHH</th>\n",
       "      <th>...</th>\n",
       "      <th>Education_level_HHH_Secondary</th>\n",
       "      <th>Education_level_HHH_Tertiary</th>\n",
       "      <th>Socio_status_HHH_Employer</th>\n",
       "      <th>Socio_status_HHH_Other</th>\n",
       "      <th>Socio_status_HHH_Unemployed</th>\n",
       "      <th>Socio_status_HHH_Worker</th>\n",
       "      <th>Ownership_motorized_vehicle_No</th>\n",
       "      <th>Ownership_motorized_vehicle_Yes</th>\n",
       "      <th>Ownership_small_livestock_No</th>\n",
       "      <th>Ownership_small_livestock_Yes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1001_3</td>\n",
       "      <td>-0.476160</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.464103</td>\n",
       "      <td>-0.394716</td>\n",
       "      <td>-0.496128</td>\n",
       "      <td>-0.433333</td>\n",
       "      <td>-0.396108</td>\n",
       "      <td>-0.155227</td>\n",
       "      <td>-0.157647</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1001_3</td>\n",
       "      <td>-0.476160</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.464103</td>\n",
       "      <td>-0.394716</td>\n",
       "      <td>-0.496128</td>\n",
       "      <td>-0.433333</td>\n",
       "      <td>-0.396108</td>\n",
       "      <td>-0.155227</td>\n",
       "      <td>-0.157647</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1026_3</td>\n",
       "      <td>-0.477477</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.464103</td>\n",
       "      <td>-0.394716</td>\n",
       "      <td>-0.496019</td>\n",
       "      <td>-0.433333</td>\n",
       "      <td>-0.396108</td>\n",
       "      <td>-0.397514</td>\n",
       "      <td>-0.157647</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1026_3</td>\n",
       "      <td>-0.477477</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.464103</td>\n",
       "      <td>-0.394716</td>\n",
       "      <td>-0.496019</td>\n",
       "      <td>-0.433333</td>\n",
       "      <td>-0.396108</td>\n",
       "      <td>-0.397514</td>\n",
       "      <td>-0.157647</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1027_2</td>\n",
       "      <td>-0.483108</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.464103</td>\n",
       "      <td>-0.394716</td>\n",
       "      <td>-0.497578</td>\n",
       "      <td>-0.455556</td>\n",
       "      <td>-0.396108</td>\n",
       "      <td>-0.397514</td>\n",
       "      <td>-0.157647</td>\n",
       "      <td>...</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 36 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       ID  Years_of_HHH_in_community  Dwelling_quality_index  \\\n",
       "0  1001_3                  -0.476160                    -0.5   \n",
       "1  1001_3                  -0.476160                    -0.5   \n",
       "2  1026_3                  -0.477477                    -0.5   \n",
       "3  1026_3                  -0.477477                    -0.5   \n",
       "4  1027_2                  -0.483108                    -0.5   \n",
       "\n",
       "   Hours_available_electricity  Measurement_age  Monthly_expenditure  \\\n",
       "0                    -0.464103        -0.394716            -0.496128   \n",
       "1                    -0.464103        -0.394716            -0.496128   \n",
       "2                    -0.464103        -0.394716            -0.496019   \n",
       "3                    -0.464103        -0.394716            -0.496019   \n",
       "4                    -0.464103        -0.394716            -0.497578   \n",
       "\n",
       "   Number_of_rooms  Climate_zone_lev_1  Climate_zone_lev_2   Age_HHH  ...  \\\n",
       "0        -0.433333           -0.396108           -0.155227 -0.157647  ...   \n",
       "1        -0.433333           -0.396108           -0.155227 -0.157647  ...   \n",
       "2        -0.433333           -0.396108           -0.397514 -0.157647  ...   \n",
       "3        -0.433333           -0.396108           -0.397514 -0.157647  ...   \n",
       "4        -0.455556           -0.396108           -0.397514 -0.157647  ...   \n",
       "\n",
       "   Education_level_HHH_Secondary Education_level_HHH_Tertiary  \\\n",
       "0                           -0.5                         -0.5   \n",
       "1                           -0.5                         -0.5   \n",
       "2                           -0.5                         -0.5   \n",
       "3                           -0.5                         -0.5   \n",
       "4                            0.5                         -0.5   \n",
       "\n",
       "  Socio_status_HHH_Employer Socio_status_HHH_Other  \\\n",
       "0                      -0.5                   -0.5   \n",
       "1                      -0.5                   -0.5   \n",
       "2                      -0.5                   -0.5   \n",
       "3                      -0.5                   -0.5   \n",
       "4                      -0.5                   -0.5   \n",
       "\n",
       "   Socio_status_HHH_Unemployed  Socio_status_HHH_Worker  \\\n",
       "0                         -0.5                      0.5   \n",
       "1                         -0.5                      0.5   \n",
       "2                          0.5                     -0.5   \n",
       "3                          0.5                     -0.5   \n",
       "4                         -0.5                      0.5   \n",
       "\n",
       "   Ownership_motorized_vehicle_No  Ownership_motorized_vehicle_Yes  \\\n",
       "0                             0.5                             -0.5   \n",
       "1                             0.5                             -0.5   \n",
       "2                            -0.5                              0.5   \n",
       "3                            -0.5                              0.5   \n",
       "4                            -0.5                              0.5   \n",
       "\n",
       "   Ownership_small_livestock_No  Ownership_small_livestock_Yes  \n",
       "0                          -0.5                            0.5  \n",
       "1                          -0.5                            0.5  \n",
       "2                          -0.5                            0.5  \n",
       "3                          -0.5                            0.5  \n",
       "4                          -0.5                            0.5  \n",
       "\n",
       "[5 rows x 36 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = dataset.apply(cleaning.replace_value(\"Ownership_small_livestock\", \"1.0\", 'Yes'))\n",
    "dataset = dataset.apply(cleaning.replace_value(\"Ownership_small_livestock\", \"0.0\", 'No'))\n",
    "dataset = dataset.apply(cleaning.astype(\"Ownership_small_livestock\", str))\n",
    "\n",
    "# add_one_hot_encoding\n",
    "dataset = dataset.apply(cleaning.add_one_hot_encoding(\"Ownership_small_livestock\"))\n",
    "\n",
    "dataset = interpolate_and_fillna_with_mean_and_standarize(dataset, \"Ownership_small_livestock\")\n",
    "\n",
    "dataset = dataset.drop_columns([\"Ownership_small_livestock_nan\"])\n",
    "\n",
    "dataset.preview()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d7a3ddeb8d776ca5",
   "metadata": {},
   "source": [
    "## Ownership_large_livestock\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "e20962f15241dc8c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-22T15:29:16.170927Z",
     "start_time": "2024-12-22T15:29:16.158234Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Years_of_HHH_in_community</th>\n",
       "      <th>Dwelling_quality_index</th>\n",
       "      <th>Hours_available_electricity</th>\n",
       "      <th>Measurement_age</th>\n",
       "      <th>Monthly_expenditure</th>\n",
       "      <th>Number_of_rooms</th>\n",
       "      <th>Climate_zone_lev_1</th>\n",
       "      <th>Climate_zone_lev_2</th>\n",
       "      <th>Age_HHH</th>\n",
       "      <th>...</th>\n",
       "      <th>Socio_status_HHH_Employer</th>\n",
       "      <th>Socio_status_HHH_Other</th>\n",
       "      <th>Socio_status_HHH_Unemployed</th>\n",
       "      <th>Socio_status_HHH_Worker</th>\n",
       "      <th>Ownership_motorized_vehicle_No</th>\n",
       "      <th>Ownership_motorized_vehicle_Yes</th>\n",
       "      <th>Ownership_small_livestock_No</th>\n",
       "      <th>Ownership_small_livestock_Yes</th>\n",
       "      <th>Ownership_large_livestock_No</th>\n",
       "      <th>Ownership_large_livestock_Yes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1001_3</td>\n",
       "      <td>-0.476160</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.464103</td>\n",
       "      <td>-0.394716</td>\n",
       "      <td>-0.496128</td>\n",
       "      <td>-0.433333</td>\n",
       "      <td>-0.396108</td>\n",
       "      <td>-0.155227</td>\n",
       "      <td>-0.157647</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1001_3</td>\n",
       "      <td>-0.476160</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.464103</td>\n",
       "      <td>-0.394716</td>\n",
       "      <td>-0.496128</td>\n",
       "      <td>-0.433333</td>\n",
       "      <td>-0.396108</td>\n",
       "      <td>-0.155227</td>\n",
       "      <td>-0.157647</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1026_3</td>\n",
       "      <td>-0.477477</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.464103</td>\n",
       "      <td>-0.394716</td>\n",
       "      <td>-0.496019</td>\n",
       "      <td>-0.433333</td>\n",
       "      <td>-0.396108</td>\n",
       "      <td>-0.397514</td>\n",
       "      <td>-0.157647</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1026_3</td>\n",
       "      <td>-0.477477</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.464103</td>\n",
       "      <td>-0.394716</td>\n",
       "      <td>-0.496019</td>\n",
       "      <td>-0.433333</td>\n",
       "      <td>-0.396108</td>\n",
       "      <td>-0.397514</td>\n",
       "      <td>-0.157647</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1027_2</td>\n",
       "      <td>-0.483108</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.464103</td>\n",
       "      <td>-0.394716</td>\n",
       "      <td>-0.497578</td>\n",
       "      <td>-0.455556</td>\n",
       "      <td>-0.396108</td>\n",
       "      <td>-0.397514</td>\n",
       "      <td>-0.157647</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 37 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       ID  Years_of_HHH_in_community  Dwelling_quality_index  \\\n",
       "0  1001_3                  -0.476160                    -0.5   \n",
       "1  1001_3                  -0.476160                    -0.5   \n",
       "2  1026_3                  -0.477477                    -0.5   \n",
       "3  1026_3                  -0.477477                    -0.5   \n",
       "4  1027_2                  -0.483108                    -0.5   \n",
       "\n",
       "   Hours_available_electricity  Measurement_age  Monthly_expenditure  \\\n",
       "0                    -0.464103        -0.394716            -0.496128   \n",
       "1                    -0.464103        -0.394716            -0.496128   \n",
       "2                    -0.464103        -0.394716            -0.496019   \n",
       "3                    -0.464103        -0.394716            -0.496019   \n",
       "4                    -0.464103        -0.394716            -0.497578   \n",
       "\n",
       "   Number_of_rooms  Climate_zone_lev_1  Climate_zone_lev_2   Age_HHH  ...  \\\n",
       "0        -0.433333           -0.396108           -0.155227 -0.157647  ...   \n",
       "1        -0.433333           -0.396108           -0.155227 -0.157647  ...   \n",
       "2        -0.433333           -0.396108           -0.397514 -0.157647  ...   \n",
       "3        -0.433333           -0.396108           -0.397514 -0.157647  ...   \n",
       "4        -0.455556           -0.396108           -0.397514 -0.157647  ...   \n",
       "\n",
       "   Socio_status_HHH_Employer Socio_status_HHH_Other  \\\n",
       "0                       -0.5                   -0.5   \n",
       "1                       -0.5                   -0.5   \n",
       "2                       -0.5                   -0.5   \n",
       "3                       -0.5                   -0.5   \n",
       "4                       -0.5                   -0.5   \n",
       "\n",
       "  Socio_status_HHH_Unemployed  Socio_status_HHH_Worker  \\\n",
       "0                        -0.5                      0.5   \n",
       "1                        -0.5                      0.5   \n",
       "2                         0.5                     -0.5   \n",
       "3                         0.5                     -0.5   \n",
       "4                        -0.5                      0.5   \n",
       "\n",
       "   Ownership_motorized_vehicle_No  Ownership_motorized_vehicle_Yes  \\\n",
       "0                             0.5                             -0.5   \n",
       "1                             0.5                             -0.5   \n",
       "2                            -0.5                              0.5   \n",
       "3                            -0.5                              0.5   \n",
       "4                            -0.5                              0.5   \n",
       "\n",
       "   Ownership_small_livestock_No  Ownership_small_livestock_Yes  \\\n",
       "0                          -0.5                            0.5   \n",
       "1                          -0.5                            0.5   \n",
       "2                          -0.5                            0.5   \n",
       "3                          -0.5                            0.5   \n",
       "4                          -0.5                            0.5   \n",
       "\n",
       "   Ownership_large_livestock_No  Ownership_large_livestock_Yes  \n",
       "0                           0.5                           -0.5  \n",
       "1                           0.5                           -0.5  \n",
       "2                          -0.5                            0.5  \n",
       "3                          -0.5                            0.5  \n",
       "4                          -0.5                            0.5  \n",
       "\n",
       "[5 rows x 37 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = dataset.apply(cleaning.replace_value(\"Ownership_large_livestock\", \"1.0\", 'Yes'))\n",
    "dataset = dataset.apply(cleaning.replace_value(\"Ownership_large_livestock\", \"0.0\", 'No'))\n",
    "\n",
    "dataset = dataset.apply(cleaning.astype(\"Ownership_large_livestock\", str))\n",
    "\n",
    "# add_one_hot_encoding\n",
    "dataset = dataset.apply(cleaning.add_one_hot_encoding(\"Ownership_large_livestock\"))\n",
    "\n",
    "dataset = interpolate_and_fillna_with_mean_and_standarize(dataset, \"Ownership_large_livestock\")\n",
    "\n",
    "dataset = dataset.drop_columns([\"Ownership_large_livestock_nan\"])\n",
    "\n",
    "dataset.preview()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d445766e629440bd",
   "metadata": {},
   "source": [
    "## Clean_fuel\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "8c69df0f4d9618c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-22T15:29:16.233300Z",
     "start_time": "2024-12-22T15:29:16.211669Z"
    }
   },
   "outputs": [],
   "source": [
    "dataset = dataset.apply(cleaning.replace_value(\"Clean_fuel\", \"0\", 'No'))\n",
    "dataset = dataset.apply(cleaning.replace_value(\"Clean_fuel\", \"1\", 'Yes'))\n",
    "\n",
    "dataset = dataset.apply(cleaning.astype(\"Clean_fuel\", str))\n",
    "\n",
    "# add_one_hot_encoding\n",
    "dataset = dataset.apply(cleaning.add_one_hot_encoding(\"Clean_fuel\"))\n",
    "\n",
    "dataset = interpolate_and_fillna_with_mean_and_standarize(dataset, \"Clean_fuel\")\n",
    "\n",
    "dataset = dataset.drop_columns([\"Clean_fuel_nan\"])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dff46a500e30d9d4",
   "metadata": {},
   "source": [
    "## Tariff_payment_frequency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "9293c673557e7211",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-22T15:29:16.321237Z",
     "start_time": "2024-12-22T15:29:16.263692Z"
    }
   },
   "outputs": [],
   "source": [
    "dataset = dataset.apply(cleaning.fillna(\"Tariff_payment_frequency\", \"not_classified\"))\n",
    "merged_categories = {\n",
    "    'not_classified': \"other\",\n",
    "    'Other': \"other\",\n",
    "    'Monthly': \"Monthly or More\",\n",
    "    'Every 2 weeks': \"Weekly or More\",\n",
    "    'Weekly': \"Weekly or More\",\n",
    "    'No bill': \"other\",\n",
    "    'Every 6 months': \"Monthly or More\",\n",
    "    'Every 3 months': \"Monthly or More\",\n",
    "}\n",
    "dataset = dataset.new_feature(\"Tariff_payment_frequency\",cleaning.merge_categories(\"Tariff_payment_frequency\", merged_categories))\n",
    "\n",
    "dataset = dataset.apply(cleaning.add_one_hot_encoding(\"Tariff_payment_frequency\"))\n",
    "\n",
    "dataset = interpolate_and_fillna_with_mean_and_standarize(dataset, \"Tariff_payment_frequency\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "9f794fbce6584411",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-22T15:29:16.329956Z",
     "start_time": "2024-12-22T15:29:16.327985Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Years_of_HHH_in_community</th>\n",
       "      <th>Dwelling_quality_index</th>\n",
       "      <th>Hours_available_electricity</th>\n",
       "      <th>Measurement_age</th>\n",
       "      <th>Monthly_expenditure</th>\n",
       "      <th>Number_of_rooms</th>\n",
       "      <th>Climate_zone_lev_1</th>\n",
       "      <th>Climate_zone_lev_2</th>\n",
       "      <th>Age_HHH</th>\n",
       "      <th>...</th>\n",
       "      <th>Ownership_motorized_vehicle_Yes</th>\n",
       "      <th>Ownership_small_livestock_No</th>\n",
       "      <th>Ownership_small_livestock_Yes</th>\n",
       "      <th>Ownership_large_livestock_No</th>\n",
       "      <th>Ownership_large_livestock_Yes</th>\n",
       "      <th>Clean_fuel_No</th>\n",
       "      <th>Clean_fuel_Yes</th>\n",
       "      <th>Tariff_payment_frequency_Monthly or More</th>\n",
       "      <th>Tariff_payment_frequency_Weekly or More</th>\n",
       "      <th>Tariff_payment_frequency_other</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1001_3</td>\n",
       "      <td>-0.476160</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.464103</td>\n",
       "      <td>-0.394716</td>\n",
       "      <td>-0.496128</td>\n",
       "      <td>-0.433333</td>\n",
       "      <td>-0.396108</td>\n",
       "      <td>-0.155227</td>\n",
       "      <td>-0.157647</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1001_3</td>\n",
       "      <td>-0.476160</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.464103</td>\n",
       "      <td>-0.394716</td>\n",
       "      <td>-0.496128</td>\n",
       "      <td>-0.433333</td>\n",
       "      <td>-0.396108</td>\n",
       "      <td>-0.155227</td>\n",
       "      <td>-0.157647</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1026_3</td>\n",
       "      <td>-0.477477</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.464103</td>\n",
       "      <td>-0.394716</td>\n",
       "      <td>-0.496019</td>\n",
       "      <td>-0.433333</td>\n",
       "      <td>-0.396108</td>\n",
       "      <td>-0.397514</td>\n",
       "      <td>-0.157647</td>\n",
       "      <td>...</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1026_3</td>\n",
       "      <td>-0.477477</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.464103</td>\n",
       "      <td>-0.394716</td>\n",
       "      <td>-0.496019</td>\n",
       "      <td>-0.433333</td>\n",
       "      <td>-0.396108</td>\n",
       "      <td>-0.397514</td>\n",
       "      <td>-0.157647</td>\n",
       "      <td>...</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1027_2</td>\n",
       "      <td>-0.483108</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.464103</td>\n",
       "      <td>-0.394716</td>\n",
       "      <td>-0.497578</td>\n",
       "      <td>-0.455556</td>\n",
       "      <td>-0.396108</td>\n",
       "      <td>-0.397514</td>\n",
       "      <td>-0.157647</td>\n",
       "      <td>...</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 40 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       ID  Years_of_HHH_in_community  Dwelling_quality_index  \\\n",
       "0  1001_3                  -0.476160                    -0.5   \n",
       "1  1001_3                  -0.476160                    -0.5   \n",
       "2  1026_3                  -0.477477                    -0.5   \n",
       "3  1026_3                  -0.477477                    -0.5   \n",
       "4  1027_2                  -0.483108                    -0.5   \n",
       "\n",
       "   Hours_available_electricity  Measurement_age  Monthly_expenditure  \\\n",
       "0                    -0.464103        -0.394716            -0.496128   \n",
       "1                    -0.464103        -0.394716            -0.496128   \n",
       "2                    -0.464103        -0.394716            -0.496019   \n",
       "3                    -0.464103        -0.394716            -0.496019   \n",
       "4                    -0.464103        -0.394716            -0.497578   \n",
       "\n",
       "   Number_of_rooms  Climate_zone_lev_1  Climate_zone_lev_2   Age_HHH  ...  \\\n",
       "0        -0.433333           -0.396108           -0.155227 -0.157647  ...   \n",
       "1        -0.433333           -0.396108           -0.155227 -0.157647  ...   \n",
       "2        -0.433333           -0.396108           -0.397514 -0.157647  ...   \n",
       "3        -0.433333           -0.396108           -0.397514 -0.157647  ...   \n",
       "4        -0.455556           -0.396108           -0.397514 -0.157647  ...   \n",
       "\n",
       "   Ownership_motorized_vehicle_Yes  Ownership_small_livestock_No  \\\n",
       "0                             -0.5                          -0.5   \n",
       "1                             -0.5                          -0.5   \n",
       "2                              0.5                          -0.5   \n",
       "3                              0.5                          -0.5   \n",
       "4                              0.5                          -0.5   \n",
       "\n",
       "   Ownership_small_livestock_Yes  Ownership_large_livestock_No  \\\n",
       "0                            0.5                           0.5   \n",
       "1                            0.5                           0.5   \n",
       "2                            0.5                          -0.5   \n",
       "3                            0.5                          -0.5   \n",
       "4                            0.5                          -0.5   \n",
       "\n",
       "   Ownership_large_livestock_Yes  Clean_fuel_No  Clean_fuel_Yes  \\\n",
       "0                           -0.5            0.5            -0.5   \n",
       "1                           -0.5            0.5            -0.5   \n",
       "2                            0.5            0.5            -0.5   \n",
       "3                            0.5            0.5            -0.5   \n",
       "4                            0.5            0.5            -0.5   \n",
       "\n",
       "   Tariff_payment_frequency_Monthly or More  \\\n",
       "0                                      -0.5   \n",
       "1                                      -0.5   \n",
       "2                                      -0.5   \n",
       "3                                      -0.5   \n",
       "4                                      -0.5   \n",
       "\n",
       "   Tariff_payment_frequency_Weekly or More  Tariff_payment_frequency_other  \n",
       "0                                     -0.5                             0.5  \n",
       "1                                     -0.5                             0.5  \n",
       "2                                     -0.5                             0.5  \n",
       "3                                     -0.5                             0.5  \n",
       "4                                     -0.5                             0.5  \n",
       "\n",
       "[5 rows x 40 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.preview()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93e029e61a0d6da9",
   "metadata": {},
   "source": [
    "# Appliances\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "5af668aa19a0facd",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-22T15:29:16.389925Z",
     "start_time": "2024-12-22T15:29:16.376866Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Missing values for Presence_refrigerator/freezer 697\n",
      "Missing values for Presence_iron 1976\n",
      "Missing values for Presence_fan 794\n",
      "Missing values for Presence_DVD_player 794\n",
      "Missing values for Presence_radio/stereo 792\n",
      "Missing values for Presence_phone_charger 1184\n",
      "Missing values for Presence_TV 2\n"
     ]
    }
   ],
   "source": [
    "PRESENCE_LIST = [\n",
    "    'Presence_refrigerator/freezer', 'Presence_iron', 'Presence_fan',\n",
    "    'Presence_DVD_player', 'Presence_radio/stereo',\n",
    "    'Presence_phone_charger',\n",
    "    'Presence_TV'\n",
    "]\n",
    "\n",
    "for p in PRESENCE_LIST:\n",
    "    dataset = dataset.apply(cleaning.fillna(p, -1))\n",
    "    value_counts = dataset.value_counts(p)\n",
    "    print(\"Missing values for\", p, value_counts[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "f9387b715c1b53ed",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-22T15:29:16.442835Z",
     "start_time": "2024-12-22T15:29:16.428494Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Years_of_HHH_in_community</th>\n",
       "      <th>Dwelling_quality_index</th>\n",
       "      <th>Hours_available_electricity</th>\n",
       "      <th>Measurement_age</th>\n",
       "      <th>Monthly_expenditure</th>\n",
       "      <th>Number_of_rooms</th>\n",
       "      <th>Climate_zone_lev_1</th>\n",
       "      <th>Climate_zone_lev_2</th>\n",
       "      <th>Age_HHH</th>\n",
       "      <th>...</th>\n",
       "      <th>Ownership_motorized_vehicle_Yes</th>\n",
       "      <th>Ownership_small_livestock_No</th>\n",
       "      <th>Ownership_small_livestock_Yes</th>\n",
       "      <th>Ownership_large_livestock_No</th>\n",
       "      <th>Ownership_large_livestock_Yes</th>\n",
       "      <th>Clean_fuel_No</th>\n",
       "      <th>Clean_fuel_Yes</th>\n",
       "      <th>Tariff_payment_frequency_Monthly or More</th>\n",
       "      <th>Tariff_payment_frequency_Weekly or More</th>\n",
       "      <th>Tariff_payment_frequency_other</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1001_3</td>\n",
       "      <td>-0.476160</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.464103</td>\n",
       "      <td>-0.394716</td>\n",
       "      <td>-0.496128</td>\n",
       "      <td>-0.433333</td>\n",
       "      <td>-0.396108</td>\n",
       "      <td>-0.155227</td>\n",
       "      <td>-0.157647</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1001_3</td>\n",
       "      <td>-0.476160</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.464103</td>\n",
       "      <td>-0.394716</td>\n",
       "      <td>-0.496128</td>\n",
       "      <td>-0.433333</td>\n",
       "      <td>-0.396108</td>\n",
       "      <td>-0.155227</td>\n",
       "      <td>-0.157647</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1026_3</td>\n",
       "      <td>-0.477477</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.464103</td>\n",
       "      <td>-0.394716</td>\n",
       "      <td>-0.496019</td>\n",
       "      <td>-0.433333</td>\n",
       "      <td>-0.396108</td>\n",
       "      <td>-0.397514</td>\n",
       "      <td>-0.157647</td>\n",
       "      <td>...</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1026_3</td>\n",
       "      <td>-0.477477</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.464103</td>\n",
       "      <td>-0.394716</td>\n",
       "      <td>-0.496019</td>\n",
       "      <td>-0.433333</td>\n",
       "      <td>-0.396108</td>\n",
       "      <td>-0.397514</td>\n",
       "      <td>-0.157647</td>\n",
       "      <td>...</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1027_2</td>\n",
       "      <td>-0.483108</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.464103</td>\n",
       "      <td>-0.394716</td>\n",
       "      <td>-0.497578</td>\n",
       "      <td>-0.455556</td>\n",
       "      <td>-0.396108</td>\n",
       "      <td>-0.397514</td>\n",
       "      <td>-0.157647</td>\n",
       "      <td>...</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 40 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       ID  Years_of_HHH_in_community  Dwelling_quality_index  \\\n",
       "0  1001_3                  -0.476160                    -0.5   \n",
       "1  1001_3                  -0.476160                    -0.5   \n",
       "2  1026_3                  -0.477477                    -0.5   \n",
       "3  1026_3                  -0.477477                    -0.5   \n",
       "4  1027_2                  -0.483108                    -0.5   \n",
       "\n",
       "   Hours_available_electricity  Measurement_age  Monthly_expenditure  \\\n",
       "0                    -0.464103        -0.394716            -0.496128   \n",
       "1                    -0.464103        -0.394716            -0.496128   \n",
       "2                    -0.464103        -0.394716            -0.496019   \n",
       "3                    -0.464103        -0.394716            -0.496019   \n",
       "4                    -0.464103        -0.394716            -0.497578   \n",
       "\n",
       "   Number_of_rooms  Climate_zone_lev_1  Climate_zone_lev_2   Age_HHH  ...  \\\n",
       "0        -0.433333           -0.396108           -0.155227 -0.157647  ...   \n",
       "1        -0.433333           -0.396108           -0.155227 -0.157647  ...   \n",
       "2        -0.433333           -0.396108           -0.397514 -0.157647  ...   \n",
       "3        -0.433333           -0.396108           -0.397514 -0.157647  ...   \n",
       "4        -0.455556           -0.396108           -0.397514 -0.157647  ...   \n",
       "\n",
       "   Ownership_motorized_vehicle_Yes  Ownership_small_livestock_No  \\\n",
       "0                             -0.5                          -0.5   \n",
       "1                             -0.5                          -0.5   \n",
       "2                              0.5                          -0.5   \n",
       "3                              0.5                          -0.5   \n",
       "4                              0.5                          -0.5   \n",
       "\n",
       "   Ownership_small_livestock_Yes  Ownership_large_livestock_No  \\\n",
       "0                            0.5                           0.5   \n",
       "1                            0.5                           0.5   \n",
       "2                            0.5                          -0.5   \n",
       "3                            0.5                          -0.5   \n",
       "4                            0.5                          -0.5   \n",
       "\n",
       "   Ownership_large_livestock_Yes  Clean_fuel_No  Clean_fuel_Yes  \\\n",
       "0                           -0.5            0.5            -0.5   \n",
       "1                           -0.5            0.5            -0.5   \n",
       "2                            0.5            0.5            -0.5   \n",
       "3                            0.5            0.5            -0.5   \n",
       "4                            0.5            0.5            -0.5   \n",
       "\n",
       "   Tariff_payment_frequency_Monthly or More  \\\n",
       "0                                      -0.5   \n",
       "1                                      -0.5   \n",
       "2                                      -0.5   \n",
       "3                                      -0.5   \n",
       "4                                      -0.5   \n",
       "\n",
       "   Tariff_payment_frequency_Weekly or More  Tariff_payment_frequency_other  \n",
       "0                                     -0.5                             0.5  \n",
       "1                                     -0.5                             0.5  \n",
       "2                                     -0.5                             0.5  \n",
       "3                                     -0.5                             0.5  \n",
       "4                                     -0.5                             0.5  \n",
       "\n",
       "[5 rows x 40 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.preview()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e5f806ecf2fd3397",
   "metadata": {},
   "source": [
    "# Save the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "2d972792b8243da3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-22T15:34:35.224559Z",
     "start_time": "2024-12-22T15:34:35.217546Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "df = dataset.to_dataframe()\n",
    "df = df.drop(columns=[\"ID\"])\n",
    "train_df, temp_df = train_test_split(df, test_size=0.4, random_state=412)\n",
    "test_df, val_df = train_test_split(temp_df, test_size=0.5, random_state=412)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "6cb2f1a01725f840",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-22T15:34:35.454124Z",
     "start_time": "2024-12-22T15:34:35.441060Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training\n",
      "Presence_refrigerator/freezer\n",
      " 0.0    5207\n",
      " 1.0     814\n",
      "-1.0     409\n",
      "Name: count, dtype: int64\n",
      "Presence_iron\n",
      " 0.0    4051\n",
      " 1.0    1221\n",
      "-1.0    1158\n",
      "Name: count, dtype: int64\n",
      "Presence_fan\n",
      " 0.0    4511\n",
      " 1.0    1453\n",
      "-1.0     466\n",
      "Name: count, dtype: int64\n",
      "Presence_DVD_player\n",
      " 0.0    5266\n",
      " 1.0     698\n",
      "-1.0     466\n",
      "Name: count, dtype: int64\n",
      "Presence_radio/stereo\n",
      " 0.0    4546\n",
      " 1.0    1419\n",
      "-1.0     465\n",
      "Name: count, dtype: int64\n",
      "Presence_phone_charger\n",
      " 0.0    4495\n",
      " 1.0    1242\n",
      "-1.0     693\n",
      "Name: count, dtype: int64\n",
      "Presence_TV\n",
      " 0.0    5296\n",
      " 1.0    1133\n",
      "-1.0       1\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(\"Training\")\n",
    "for p in PRESENCE_LIST:\n",
    "    value_counts = train_df.value_counts(p)\n",
    "    print( value_counts)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "6f30363cef2d181d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-22T15:34:37.271193Z",
     "start_time": "2024-12-22T15:34:37.260540Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation\n",
      "Presence_refrigerator/freezer\n",
      " 0.0    1736\n",
      " 1.0     286\n",
      "-1.0     122\n",
      "Name: count, dtype: int64\n",
      "Presence_iron\n",
      " 0.0    1322\n",
      " 1.0     420\n",
      "-1.0     402\n",
      "Name: count, dtype: int64\n",
      "Presence_fan\n",
      " 0.0    1506\n",
      " 1.0     500\n",
      "-1.0     138\n",
      "Name: count, dtype: int64\n",
      "Presence_DVD_player\n",
      " 0.0    1758\n",
      " 1.0     248\n",
      "-1.0     138\n",
      "Name: count, dtype: int64\n",
      "Presence_radio/stereo\n",
      " 0.0    1535\n",
      " 1.0     471\n",
      "-1.0     138\n",
      "Name: count, dtype: int64\n",
      "Presence_phone_charger\n",
      " 0.0    1442\n",
      " 1.0     438\n",
      "-1.0     264\n",
      "Name: count, dtype: int64\n",
      "Presence_TV\n",
      "0.0    1735\n",
      "1.0     409\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(\"Validation\")\n",
    "for p in PRESENCE_LIST:\n",
    "    value_counts = val_df.value_counts(p)\n",
    "    print(value_counts)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "ee8b9d6a14df15bf",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-22T15:34:39.672761Z",
     "start_time": "2024-12-22T15:34:39.664762Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing\n",
      "Presence_refrigerator/freezer\n",
      " 0.0    1707\n",
      " 1.0     271\n",
      "-1.0     166\n",
      "Name: count, dtype: int64\n",
      "Presence_iron\n",
      " 0.0    1319\n",
      "-1.0     416\n",
      " 1.0     409\n",
      "Name: count, dtype: int64\n",
      "Presence_fan\n",
      " 0.0    1448\n",
      " 1.0     506\n",
      "-1.0     190\n",
      "Name: count, dtype: int64\n",
      "Presence_DVD_player\n",
      " 0.0    1742\n",
      " 1.0     212\n",
      "-1.0     190\n",
      "Name: count, dtype: int64\n",
      "Presence_radio/stereo\n",
      " 0.0    1487\n",
      " 1.0     468\n",
      "-1.0     189\n",
      "Name: count, dtype: int64\n",
      "Presence_phone_charger\n",
      " 0.0    1517\n",
      " 1.0     400\n",
      "-1.0     227\n",
      "Name: count, dtype: int64\n",
      "Presence_TV\n",
      " 0.0    1778\n",
      " 1.0     365\n",
      "-1.0       1\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(\"Testing\")\n",
    "for p in PRESENCE_LIST:\n",
    "    value_counts = test_df.value_counts(p)\n",
    "    print(value_counts)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "7d1246f674f8bdb1",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-22T15:35:41.945427Z",
     "start_time": "2024-12-22T15:35:41.841429Z"
    }
   },
   "outputs": [],
   "source": [
    "train_df.to_csv(\"./data/normalized_train_set.csv\", index=False)\n",
    "val_df.to_csv(\"./data/normalized_val_set.csv\", index=False)\n",
    "test_df.to_csv(\"./data/normalized_test_set.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a0c8beadcefc70aa",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-22T15:29:16.972282Z",
     "start_time": "2024-12-22T15:29:16.970488Z"
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nlp",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}