ran open data for jan

Author: amandaha8

.bashrc

@@ -0,0 +1,2 @@
+source .profile
+source .profile

_shared_utils/shared_utils/rt_dates.py

@@ -75,6 +75,7 @@
     "nov2024": "2024-11-13",
     "dec2024": "2024-12-11",
     "jan2025": "2025-01-15",
+    "feb2025": "2025-02-12"
 }
 
 y2023_dates = [

gtfs_digest/.bash_profile

@@ -0,0 +1,2 @@
+source .profile
+source .profile

gtfs_digest/03_report.ipynb

@@ -52,8 +52,8 @@
    "outputs": [],
    "source": [
     "# Comment out and leave this cell right below pandas\n",
-    "organization_name = \"Marin County Transit District\"\n",
-    "# organization_name = \"Monterey-Salinas Transit\"\n",
+    "# organization_name = \"Capitol Corridor Joint Powers Authority\"\n",
+    "organization_name = \"Monterey-Salinas Transit\"\n",
     "# organization_name = \"City of Santa Maria\"\n",
     "# organization_name = \"City of Montebello\""
    ]
@@ -63,7 +63,9 @@
    "execution_count": null,
    "id": "d870c492-ef2c-45f6-ab47-8d46eda7f344",
    "metadata": {
-    "tags": []
+    "tags": [
+     "parameters"
+    ]
    },
    "outputs": [],
    "source": [
@@ -540,14 +542,6 @@
     "except:\n",
     "    display(Markdown(f\"\"\"{organization_name} only has schedule data.\"\"\"))"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "f8f9f2cb-5464-4080-9fda-f342c040d1b9",
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {

gtfs_digest/52_feb_run.ipynb

@@ -0,0 +1,182 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "cbb6e02b-aa46-4143-96fc-1d3c9df1000d",
+   "metadata": {},
+   "source": [
+    "## Error when running `gtfs_digest/merge_segment_data.py`\n",
+    "`Traceback (most recent call last):\n",
+    "  File \"/opt/conda/lib/python3.11/site-packages/dask/dataframe/utils.py\", line 195, in raise_on_meta_error\n",
+    "    yield\n",
+    "  File \"/opt/conda/lib/python3.11/site-packages/dask/dataframe/core.py\", line 7175, in _emulate\n",
+    "    return func(*_extract_meta(args, True), **_extract_meta(kwargs, True))\n",
+    "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+    "  File \"/home/jovyan/data-analyses/_shared_utils/shared_utils/dask_utils.py\", line 134, in import_df_func\n",
+    "    df = pd.read_parquet(\n",
+    "         ^^^^^^^^^^^^^^^^\n",
+    "  File \"/opt/conda/lib/python3.11/site-packages/pandas/io/parquet.py\", line 503, in read_parquet\n",
+    "    return impl.read(\n",
+    "           ^^^^^^^^^^\n",
+    "  File \"/opt/conda/lib/python3.11/site-packages/pandas/io/parquet.py\", line 251, in read\n",
+    "    result = self.api.parquet.read_table(\n",
+    "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+    "  File \"/opt/conda/lib/python3.11/site-packages/pyarrow/parquet/core.py\", line 1793, in read_table\n",
+    "    dataset = ParquetDataset(\n",
+    "              ^^^^^^^^^^^^^^^\n",
+    "  File \"/opt/conda/lib/python3.11/site-packages/pyarrow/parquet/core.py\", line 1371, in __init__\n",
+    "    self._dataset = ds.dataset(path_or_paths, filesystem=filesystem,\n",
+    "                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+    "  File \"/opt/conda/lib/python3.11/site-packages/pyarrow/dataset.py\", line 794, in dataset\n",
+    "    return _filesystem_dataset(source, **kwargs)\n",
+    "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+    "  File \"/opt/conda/lib/python3.11/site-packages/pyarrow/dataset.py\", line 476, in _filesystem_dataset\n",
+    "    fs, paths_or_selector = _ensure_single_source(source, filesystem)\n",
+    "                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+    "  File \"/opt/conda/lib/python3.11/site-packages/pyarrow/dataset.py\", line 441, in _ensure_single_source\n",
+    "    raise FileNotFoundError(path)\n",
+    "FileNotFoundError: calitp-analytics-data/data-analyses/rt_segment_speeds/rollup_singleday/speeds_route_dir_segments_2024-01-17.parquet\n",
+    "\n",
+    "The above exception was the direct cause of the following exception:\n",
+    "\n",
+    "Traceback (most recent call last):\n",
+    "  File \"/home/jovyan/data-analyses/gtfs_digest/merge_segment_data.py\", line 93, in <module>\n",
+    "    segment_speeds = concatenate_segment_speeds_by_route_direction(\n",
+    "                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+    "  File \"/home/jovyan/data-analyses/gtfs_digest/merge_segment_data.py\", line 40, in concatenate_segment_speeds_by_route_direction\n",
+    "    df = time_series_utils.concatenate_datasets_across_dates(\n",
+    "         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+    "  File \"/home/jovyan/data-analyses/rt_segment_speeds/segment_speed_utils/time_series_utils.py\", line 30, in concatenate_datasets_across_dates\n",
+    "    df = dask_utils.get_ddf(\n",
+    "         ^^^^^^^^^^^^^^^^^^^\n",
+    "  File \"/home/jovyan/data-analyses/_shared_utils/shared_utils/dask_utils.py\", line 183, in get_ddf\n",
+    "    ddf = dd.from_map(import_df_func, paths, date_list, data_type=data_type, **kwargs).drop_duplicates()\n",
+    "          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+    "  File \"/opt/conda/lib/python3.11/site-packages/dask/dataframe/io/io.py\", line 1028, in from_map\n",
+    "    meta = _emulate(\n",
+    "           ^^^^^^^^^\n",
+    "  File \"/opt/conda/lib/python3.11/site-packages/dask/dataframe/core.py\", line 7174, in _emulate\n",
+    "    with raise_on_meta_error(funcname(func), udf=udf), check_numeric_only_deprecation():\n",
+    "  File \"/opt/conda/lib/python3.11/contextlib.py\", line 155, in __exit__\n",
+    "    self.gen.throw(typ, value, traceback)\n",
+    "  File \"/opt/conda/lib/python3.11/site-packages/dask/dataframe/utils.py\", line 216, in raise_on_meta_error\n",
+    "    raise ValueError(msg) from e\n",
+    "ValueError: Metadata inference failed in `import_df_func`.\n",
+    "\n",
+    "You have supplied a custom function and Dask is unable to \n",
+    "determine the type of output that that function returns. \n",
+    "\n",
+    "To resolve this please provide a meta= keyword.\n",
+    "The docstring of the Dask function you ran should have more information.\n",
+    "\n",
+    "Original error is below:\n",
+    "------------------------\n",
+    "FileNotFoundError('calitp-analytics-data/data-analyses/rt_segment_speeds/rollup_singleday/speeds_route_dir_segments_2024-01-17.parquet')\n",
+    "\n",
+    "Traceback:\n",
+    "---------\n",
+    "  File \"/opt/conda/lib/python3.11/site-packages/dask/dataframe/utils.py\", line 195, in raise_on_meta_error\n",
+    "    yield\n",
+    "  File \"/opt/conda/lib/python3.11/site-packages/dask/dataframe/core.py\", line 7175, in _emulate\n",
+    "    return func(*_extract_meta(args, True), **_extract_meta(kwargs, True))\n",
+    "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+    "  File \"/home/jovyan/data-analyses/_shared_utils/shared_utils/dask_utils.py\", line 134, in import_df_func\n",
+    "    df = pd.read_parquet(\n",
+    "         ^^^^^^^^^^^^^^^^\n",
+    "  File \"/opt/conda/lib/python3.11/site-packages/pandas/io/parquet.py\", line 503, in read_parquet\n",
+    "    return impl.read(\n",
+    "           ^^^^^^^^^^\n",
+    "  File \"/opt/conda/lib/python3.11/site-packages/pandas/io/parquet.py\", line 251, in read\n",
+    "    result = self.api.parquet.read_table(\n",
+    "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+    "  File \"/opt/conda/lib/python3.11/site-packages/pyarrow/parquet/core.py\", line 1793, in read_table\n",
+    "    dataset = ParquetDataset(\n",
+    "              ^^^^^^^^^^^^^^^\n",
+    "  File \"/opt/conda/lib/python3.11/site-packages/pyarrow/parquet/core.py\", line 1371, in __init__\n",
+    "    self._dataset = ds.dataset(path_or_paths, filesystem=filesystem,\n",
+    "                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+    "  File \"/opt/conda/lib/python3.11/site-packages/pyarrow/dataset.py\", line 794, in dataset\n",
+    "    return _filesystem_dataset(source, **kwargs)\n",
+    "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+    "  File \"/opt/conda/lib/python3.11/site-packages/pyarrow/dataset.py\", line 476, in _filesystem_dataset\n",
+    "    fs, paths_or_selector = _ensure_single_source(source, filesystem)\n",
+    "                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+    "  File \"/opt/conda/lib/python3.11/site-packages/pyarrow/dataset.py\", line 441, in _ensure_single_source\n",
+    "    raise FileNotFoundError(path)\n",
+    "\n",
+    "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n",
+    "E0000 00:00:1739983694.449132     625 init.cc:232] grpc_wait_for_shutdown_with_timeout() timed out.`"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "3d70f4c3-3572-4d2e-b588-226709d48e25",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import _section1_utils as section1\n",
+    "import geopandas as gpd\n",
+    "import merge_data\n",
+    "import merge_operator_data\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "from segment_speed_utils import gtfs_schedule_wrangling, helpers\n",
+    "from shared_utils import portfolio_utils\n",
+    "from update_vars import GTFS_DATA_DICT, RT_SCHED_GCS, SCHED_GCS, SEGMENT_GCS"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "adfc13e3-098e-41e8-883a-497f85afbea5",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "pd.options.display.max_columns = 100\n",
+    "pd.options.display.float_format = \"{:.2f}\".format\n",
+    "pd.set_option(\"display.max_rows\", None)\n",
+    "pd.set_option(\"display.max_colwidth\", None)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "58095cec-ca10-40e4-9605-51626170f32a",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# FileNotFoundError: \n",
+    "# calitp-analytics-data/data-analyses/rt_segment_speeds/rollup_singleday/speeds_route_dir_segments_2024-01-17.parquet\n",
+    "# gs://calitp-analytics-data/data-analyses/rt_segment_speeds/rollup_singleday/speeds_route_dir_2024-01-17.parquet"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

gtfs_digest/Makefile

@@ -2,7 +2,6 @@ assemble_data:
 	python merge_data.py
 	python merge_operator_data.py
 	python merge_operator_service.py
-	python merge_segment_data.py
 	python publish_public_data.py
 
 

gtfs_funnel/update_vars.py

@@ -11,10 +11,7 @@
     apr2024_week + rt_dates.y2025_dates
 )
 
-all_dates2 = (
-    rt_dates.y2023_dates + 
-    oct2023_week + apr2023_week + 
-    apr2024_week + rt_dates.y2025_dates
+all_dates2 = (rt_dates.y2025_dates
 )
 
 # analysis_date_list = [rt_dates.DATES["dec2024"]] + [rt_dates.DATES['nov2024']]

portfolio/gtfs_digest/README.md

@@ -1,9 +1,11 @@
 # General Transit Feed Specification (GTFS) Digest
-The goal of this website is to give you an overview of transit operators that produce GTFS schedule and/or real-time data. We use data from the [National Transit Database](https://www.transit.dot.gov/ntd), [National Association of City Transportation Officials's Transit Route Types](https://nacto.org/publication/transit-street-design-guide/introduction/service-context/transit-route-types/), and [GTFS feeds](https://gtfs.org/) to deliver key insights. You can find details such as the types of routes and the total scheduled hours of public transit service for which an operator runs.
+The goal of this website is to give you an overview of transit operators that produce GTFS schedule and/or real-time data either on the individual operator, Caltrans district, or legislative district level. 
 
-For operators who produce real-time data, we also calculate additional performance metrics for all of their routes. Examples include displaying the number of on-time, early, and late trips, the average speed, and the headway for a route.
+We use data from the [National Transit Database](https://www.transit.dot.gov/ntd), [National Association of City Transportation Official’s Transit Route Types](https://nacto.org/publication/transit-street-design-guide/introduction/service-context/transit-route-types/), and [GTFS feeds](https://gtfs.org/) to deliver key insights. You can find details such as the types of routes and the total scheduled hours of public transit service for which an operator runs.
 
-GTFS Digest will continue to evolve as we dive into our own data warehouse!
+For operators who produce real-time data, we also calculate additional performance metrics for all their routes. Examples include displaying the number of on-time, early, and late trips, the average speed, and the headway for a route.
+
+GTFS Digest will continue to evolve as we dive into our own data warehouse! 
 
 ## Definitions and Methodology
 To read about the methodology behind and the definitions of terms used throughout our work, please visit [here](https://github.com/cal-itp/data-analyses/blob/main/gtfs_digest/methodology.md).

portfolio/gtfs_digest/_toc.yml

@@ -23,6 +23,9 @@ parts:
     -   file: district_07-los-angeles.md
         sections:
         -   glob: district_07-los-angeles/*
+    -   file: district_07-los-angeles-ventura.md
+        sections:
+        -   glob: district_07-los-angeles-ventura/*
     -   file: district_08-san-bernardino.md
         sections:
         -   glob: district_08-san-bernardino/*

portfolio/gtfs_digest/district_01-eureka/00__03_report__district_01-eureka__organization_name_blue-lake-rancheria.ipynb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a33fd48f931c6c72e9059e48144c049a82e86177c97685597eb2133796fcd3ed
-size 2902492
+oid sha256:18b3526815a4b77006b0c004e24aff15bd255596f731f9f31d37d7a45d7bb418
+size 2898415

portfolio/gtfs_digest/district_01-eureka/01__03_report__district_01-eureka__organization_name_city-of-arcata.ipynb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:93f951fb01f3da032a30cde48cde15d390ff8ffa42ecc89447f3d9c8816582bc
-size 5331630
+oid sha256:f87afbfc904e9261eee89ca20805b6233fb1ced9ddfe7699b41c72a238a8e417
+size 79690