EQI Demographic Screen

equity_index/01_hcd_income_limit.ipynb

@@ -0,0 +1,169 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "0ff0cf22-9eb0-470a-a2ae-b60728907ed2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import re\n",
+    "\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "import pdfplumber"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "2d73318a-d91f-41ae-87f9-d71839c6e5f4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pd.options.display.max_columns = 100\n",
+    "pd.options.display.float_format = \"{:.2f}\".format\n",
+    "pd.set_option(\"display.max_rows\", None)\n",
+    "pd.set_option(\"display.max_colwidth\", None)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 47,
+   "id": "efce4e7b-1ac5-49f3-acba-be4f9ea99d0a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "def extract_tables(pdf_path: str, start_page: int = 6, end_page: int = 14) -> pd.DataFrame:\n",
+    "    \"\"\"\n",
+    "    Extract income tables for each county from pages start_page to end_page of the PDF.\n",
+    "    Returns a combined DataFrame with all rows from all tables.\n",
+    "    \"\"\"\n",
+    "    all_tables = []\n",
+    "\n",
+    "    with pdfplumber.open(pdf_path) as pdf:\n",
+    "        for page_num in range(start_page - 1, end_page):  # pdfplumber pages are 0-indexed\n",
+    "            page = pdf.pages[page_num]\n",
+    "            tables = page.extract_tables()\n",
+    "\n",
+    "            for table in tables:\n",
+    "                # Convert table to DataFrame\n",
+    "                df = pd.DataFrame(table)\n",
+    "                all_tables.append(df)\n",
+    "\n",
+    "    # Combine all tables into one DataFrame\n",
+    "    combined_df = pd.concat(all_tables, ignore_index=True)\n",
+    "    return combined_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "e493255b-26a6-4951-addd-bada6200c59d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Usage\n",
+    "pdf_path = \"./hcd_income_limits_2024.pdf\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 51,
+   "id": "f9d7f03f-3149-44fd-9343-ac98c077c598",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'hcd_income_limits_2024.csv'"
+      ]
+     },
+     "execution_count": 51,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 54,
+   "id": "76be7343-a001-490b-bc87-ba8120121a22",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def create_low_income_hcd(pdf_path:str, GCS_PATH:str)->pd.DataFrame:\n",
+    "    df = extract_income_tables(pdf_path)\n",
+    "    df = df.ffill()\n",
+    "\n",
+    "    \n",
+    "    df = df.rename(columns = \n",
+    "                  {0: \"county\",\n",
+    "                  1: \"category\",\n",
+    "                  2: \"one\",\n",
+    "                  3: \"two\",\n",
+    "                  4: \"three\",\n",
+    "                  5: \"four\",\n",
+    "                  6: \"five\",\n",
+    "                  7: \"six\",\n",
+    "                  8: \"seven\",\n",
+    "                  9: \"eight\"})\n",
+    "\n",
+    "    df['county'] = df['county'].str.split('Area', n=1).str.get(0).str.strip()\n",
+    "    \n",
+    "    df2 = df.loc[df.category == \"Low Income\"].reset_index(drop = True)\n",
+    "    csv_name = pdf_path.replace(\"pdf\",\"csv\").replace(\"./\",\"\")\n",
+    "    df2.to_csv(f\"{GCS_PATH}{csv_name}\")\n",
+    "    return df2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 55,
+   "id": "83590da8-afef-409b-9486-7c7972a24580",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "low_income_df = create_low_income_hcd(pdf_path = pdf_path,\n",
+    "                                     GCS_PATH = \"gs://calitp-analytics-data/data-analyses/equity_index/\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
+   "id": "cf47c33e-abb2-4774-8631-d1ee46762014",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "df_income= df_income.rename(columns = {0:\"county\", 1:\"category\", 2: \"one\",\n",
+    "                                      3: \"two\", 4: \"three\", 5: \"four\", 6: \"five\",\n",
+    "                                      7:\"six\", 8: \"seven\", 9:\"eight\"})"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}