Merge branch 'main' into ah_gtfs

Author: amandaha8

_shared_utils/requirements.txt

@@ -1,11 +1,7 @@
 -e .
-altair==5.3.0
 altair-transform==0.2.0
-gtfs-segments==2.1.7
-pyairtable==2.2.2
-great_tables==0.14.0
+great_tables==0.16.1
 omegaconf==2.3.0 # better yaml configuration
-polars==0.20.29
-quarto-cli==1.4.554
+polars==1.22.0
+quarto-cli==1.6.40
 quarto==0.1.0
-typing_extensions==4.12.2

_shared_utils/setup.py

@@ -4,7 +4,7 @@
 setup(
     name="shared_utils",
     packages=find_packages(),
-    version="3.0",
+    version="4.0",
     description="Shared utility functions for data analyses",
     author="Cal-ITP",
     license="Apache",

_shared_utils/shared_utils/dask_utils.py

@@ -142,6 +142,35 @@ def import_df_func(
     return df
 
 
+def import_ddf_func(path, date_list, data_type, **kwargs):
+    """
+    Equivalent to improt_df_func, except uses dask to read in the dataframe
+    instead of pandas.
+    Concatenates the various dates.
+    """
+    if data_type == "df":
+        ddf = dd.multi.concat(
+            [
+                dd.read_parquet(f"{path}_{one_date}.parquet", **kwargs).assign(service_date=one_date)
+                for one_date in date_list
+            ],
+            axis=0,
+            ignore_index=True,
+        )
+
+    elif data_type == "gdf":
+        ddf = dd.multi.concat(
+            [
+                dg.read_parquet(f"{path}_{one_date}.parquet", **kwargs).assign(service_date=one_date)
+                for one_date in date_list
+            ],
+            axis=0,
+            ignore_index=True,
+        )
+
+    return ddf
+
+
 def get_ddf(paths, date_list, data_type, get_pandas: bool = False, **kwargs):
     """
     Set up function with little modifications based on

_shared_utils/shared_utils/gtfs_analytics_data.yml

@@ -80,10 +80,14 @@ stop_segments:
   stop_pair_cols: ["stop_pair", "stop_pair_name"]
   route_dir_cols: ["route_id", "direction_id"]
   segment_cols: ["route_id", "direction_id", "stop_pair", "geometry"]
+  segment_timeofday: "rollup_singleday/speeds_route_dir_timeofday_segments"
+  # segment_peakoffpeak
+  # segment_weekday_timeofday
+  # -- cache segment_timeofday first and use this to build other layers? other keys to make peak/offpeak, weekday/weekend grains clear?
   #shape_stop_single_segment: "rollup_singleday/speeds_shape_stop_segments" #-- stop after Oct 2024
   route_dir_single_segment: "rollup_singleday/speeds_route_dir_segments"
   route_dir_single_segment_detail: "rollup_singleday/speeds_route_dir_segments_detail" # interim for speedmaps
-  route_dir_multi_segment: "rollup_multiday/speeds_route_dir_segments"
+  route_dir_multi_segment: "rollup_multiday/speeds_route_dir_segments" # -- this one should be replaced with weekday/weekend, make clear the grain
   segments_file: "segment_options/shape_stop_segments"
   max_speed: ${speed_vars.max_speed}
   route_dir_quarter_segment: "rollup_multiday/quarter_speeds_route_dir_segments"

_shared_utils/shared_utils/rt_dates.py

@@ -80,7 +80,7 @@
     v for k, v in DATES.items() if k.endswith("2023") and not any(substring in k for substring in ["jan", "feb"])
 ]
 
-y2024_dates = [v for k, v in DATES.items() if k.endswith("2024")]
+y2024_dates = [v for k, v in DATES.items() if k.endswith("2024") and k not in ["oct2024g"]]
 
 
 valid_weeks = ["apr2023", "oct2023", "apr2024", "oct2024"]
@@ -96,7 +96,7 @@ def get_week(month: Literal[[*valid_weeks]], exclude_wed: bool) -> list:
 apr2023_week = get_week(month="apr2023", exclude_wed=False)
 oct2023_week = get_week(month="oct2023", exclude_wed=False)
 apr2024_week = get_week(month="apr2024", exclude_wed=False)
-oct2024_week = get_week(month="oct2024", exclude_wed=False)
+oct2024_week = [d for d in get_week(month="oct2024", exclude_wed=False) if d != DATES["oct2024g"]]
 
 MONTH_DICT = {
     1: "January",

_shared_utils/shared_utils/rt_utils.py

@@ -402,7 +402,7 @@ def get_vehicle_positions(ix_df: pd.DataFrame) -> gpd.GeoDataFrame:
         vp_all = gpd.read_parquet(f"{VP_FILE_PATH}vp_{date_str}.parquet")
         org_vp = vp_all >> filter(_.gtfs_dataset_key.isin(ix_df.vehicle_positions_gtfs_dataset_key))
         org_vp = org_vp >> select(-_.location_timestamp, -_.service_date, -_.activity_date)
-        org_vp = org_vp.to_crs(geography_utils.CA_NAD83Albers)
+        org_vp = org_vp.to_crs(geography_utils.CA_NAD83Albers_m)
         utils.geoparquet_gcs_export(org_vp, GCS_FILE_PATH + V2_SUBFOLDER, filename)
 
     return org_vp
@@ -459,7 +459,9 @@ def get_stops(ix_df: pd.DataFrame) -> gpd.GeoDataFrame:
         org_stops = gpd.read_parquet(path)
     else:
         feed_key_list = list(ix_df.feed_key.unique())
-        org_stops = gtfs_utils_v2.get_stops(service_date, feed_key_list, stop_cols, crs=geography_utils.CA_NAD83Albers)
+        org_stops = gtfs_utils_v2.get_stops(
+            service_date, feed_key_list, stop_cols, crs=geography_utils.CA_NAD83Albers_m
+        )
         utils.geoparquet_gcs_export(org_stops, GCS_FILE_PATH + V2_SUBFOLDER, filename)
 
     return org_stops
@@ -478,7 +480,7 @@ def get_shapes(ix_df: pd.DataFrame) -> gpd.GeoDataFrame:
     else:
         feed_key_list = list(ix_df.feed_key.unique())
         org_shapes = gtfs_utils_v2.get_shapes(
-            service_date, feed_key_list, crs=geography_utils.CA_NAD83Albers, shape_cols=shape_cols
+            service_date, feed_key_list, crs=geography_utils.CA_NAD83Albers_m, shape_cols=shape_cols
         )
         # invalid geos are nones in new df...
         org_shapes = org_shapes.dropna(subset=["geometry"])

_shared_utils/shared_utils/shared_data.py

@@ -19,7 +19,7 @@ def make_county_centroids():
     """
     URL = "https://opendata.arcgis.com/datasets/" "8713ced9b78a4abb97dc130a691a8695_0.geojson"
 
-    gdf = gpd.read_file(URL).to_crs(geography_utils.CA_StatePlane)
+    gdf = gpd.read_file(URL).to_crs(geography_utils.CA_NAD83Albers_ft)
     gdf.columns = gdf.columns.str.lower()
 
     gdf = (
@@ -167,7 +167,7 @@ def segment_highway_lines_by_postmile(gdf: gpd.GeoDataFrame):
 
     # Assign segment geometry and overwrite the postmile geometry column
     gdf2 = (
-        gdf.assign(geometry=gpd.GeoSeries(segment_geom, crs=geography_utils.CA_NAD83Albers))
+        gdf.assign(geometry=gpd.GeoSeries(segment_geom, crs=geography_utils.CA_NAD83Albers_m))
         .drop(columns=drop_cols)
         .set_geometry("geometry")
     )
@@ -205,7 +205,7 @@ def create_postmile_segments(
         .explode("geometry")
         .reset_index(drop=True)
         .pipe(round_odometer_values, ["bodometer", "eodometer"], num_decimals=3)
-        .to_crs(geography_utils.CA_NAD83Albers)
+        .to_crs(geography_utils.CA_NAD83Albers_m)
     )
 
     # Have a list accompany the geometry
@@ -222,7 +222,7 @@ def create_postmile_segments(
             f"{GCS_FILE_PATH}state_highway_network_postmiles.parquet", columns=group_cols + ["odometer", "geometry"]
         )
         .pipe(round_odometer_values, ["odometer"], num_decimals=3)
-        .to_crs(geography_utils.CA_NAD83Albers)
+        .to_crs(geography_utils.CA_NAD83Albers_m)
     )
     # Round to 3 digits for odometer. When there are more decimal places, it makes our cutoffs iffy
     # when we use this condition below: odometer >= bodometer & odometer <= eodometer

_shared_utils/shared_utils/time_helpers.py

@@ -38,6 +38,11 @@
     **{k: "weekend" for k in ["Saturday", "Sunday"]},
 }
 
+WEEKDAY_DICT2 = {
+    **{k: "Weekday" for k in ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday"]},
+    **{k: k for k in ["Saturday", "Sunday"]},
+}
+
 
 def time_span_labeling(date_list: list) -> tuple[str]:
     """

bus_service_increase/bus_service_utils/create_parallel_corridors.py

@@ -31,8 +31,8 @@ def process_transit_routes(
     ## Clean transit routes
     df = df.assign(
         route_length = df.to_crs(
-            geography_utils.CA_StatePlane).geometry.length
-    ).to_crs(geography_utils.CA_StatePlane)
+            geography_utils.CA_NAD83Albers_ft).geometry.length
+    ).to_crs(geography_utils.CA_NAD83Albers_ft)
 
     # Get it down to route_id and pick longest shape
     df2 = (df.sort_values(operator_cols + ["route_id", "route_length"], 
@@ -63,7 +63,7 @@ def prep_highway_directions_for_dissolve(
     '''
     df = (gpd.read_parquet("gs://calitp-analytics-data/data-analyses/"
                            "shared_data/state_highway_network.parquet")
-          .to_crs(geography_utils.CA_StatePlane))
+          .to_crs(geography_utils.CA_NAD83Albers_ft))
 
     # Get dummies for direction
     # Can make data wide instead of long

bus_service_increase/create_analysis_data.py

@@ -134,7 +134,7 @@ def get_shapes(selected_date: str) -> gpd.GeoDataFrame:
         selected_date,
         columns = ["shape_array_key", "geometry"],
         get_pandas = True,
-        crs = geography_utils.CA_NAD83Albers
+        crs = geography_utils.CA_NAD83Albers_m
     ).pipe(
         helpers.remove_shapes_outside_ca
     ).merge(
@@ -151,7 +151,7 @@ def get_shapes(selected_date: str) -> gpd.GeoDataFrame:
 
 
 def dissolve_census_tracts(
-    crs: str = geography_utils.CA_NAD83Albers
+    crs: str = geography_utils.CA_NAD83Albers_m
 ) -> gpd.GeoDataFrame:
     census_tracts = (
         catalog.calenviroscreen_lehd_by_tract.read()

bus_service_increase/highways-existing-transit.ipynb

@@ -78,7 +78,7 @@
     "plot_df = gdf[\n",
     "        gdf.route_length >= geography_utils.FEET_PER_MI * 0.5\n",
     "    ].assign(\n",
-    "        geometry = (gdf.geometry.to_crs(geography_utils.CA_StatePlane)\n",
+    "        geometry = (gdf.geometry.to_crs(geography_utils.CA_NAD83Albers_ft)\n",
     "                .buffer(300)\n",
     "                .to_crs(geography_utils.WGS84)\n",
     "               )\n",