final debug, run speed pipeline for jan

edasmalchi · edasmalchi · commit 3b9a3fa364b2 · 2026-02-03T23:47:37.000Z
diff --git a/rt_segment_speeds/logs/avg_speeds.log b/rt_segment_speeds/logs/avg_speeds.log
@@ -77,3 +77,7 @@
 2025-11-17 13:02:18.627 | INFO     | __main__:summary_speeds_by_peak_offpeak:161 - rt_stop_times summary speed averaging by peak/offpeak for 2025-11-05 execution time: 0:00:14.538236
 2025-11-17 15:31:09.297 | INFO     | __main__:trip_summary_speeds_by_time_of_day:92 - rt_stop_times summary speed averaging by time-of-day 2025-11-05 execution time: 0:00:07.154874
 2025-11-17 15:31:23.477 | INFO     | __main__:summary_speeds_by_peak_offpeak:161 - rt_stop_times summary speed averaging by peak/offpeak for 2025-11-05 execution time: 0:00:14.178745
+2026-02-03 12:40:41.180 | INFO     | __main__:trip_summary_speeds_by_time_of_day:99 - rt_stop_times summary speed averaging by time-of-day 2025-12-17 execution time: 0:00:07.917585
+2026-02-03 12:40:57.166 | INFO     | __main__:summary_speeds_by_peak_offpeak:168 - rt_stop_times summary speed averaging by peak/offpeak for 2025-12-17 execution time: 0:00:15.984873
+2026-02-03 15:18:13.379 | INFO     | __main__:trip_summary_speeds_by_time_of_day:99 - rt_stop_times summary speed averaging by time-of-day 2026-01-14 execution time: 0:00:08.800196
+2026-02-03 15:18:27.839 | INFO     | __main__:summary_speeds_by_peak_offpeak:168 - rt_stop_times summary speed averaging by peak/offpeak for 2026-01-14 execution time: 0:00:14.427996
diff --git a/rt_segment_speeds/logs/cut_stop_segments.log b/rt_segment_speeds/logs/cut_stop_segments.log
@@ -55,3 +55,5 @@
 2025-11-17 14:33:52.902 | INFO     | __main__:<module>:246 - speedmap segments and proxy_stop_times 2025-11-05: 0:05:35.727054
 2026-01-23 09:57:31.886 | INFO     | __main__:<module>:178 - cut segments 2025-12-17: 0:24:22.626277
 2026-01-23 10:10:50.955 | INFO     | __main__:<module>:259 - speedmap segments and proxy_stop_times 2025-12-17: 0:05:32.498813
+2026-02-03 14:10:56.471 | INFO     | __main__:<module>:178 - cut segments 2026-01-14: 0:27:55.775283
+2026-02-03 14:17:44.292 | INFO     | __main__:<module>:188 - speedmap segments and proxy_stop_times 2026-01-14: 0:06:07.308326
diff --git a/rt_segment_speeds/logs/interpolate_stop_arrival.log b/rt_segment_speeds/logs/interpolate_stop_arrival.log
@@ -73,3 +73,10 @@ interpolate arrivals for stop_segments 2024-12-11:  2024-12-11: 0:14:09.409333
 2025-11-17 15:24:22.306 | INFO     | interpolate_stop_arrival:interpolate_stop_arrivals:236 - interpolate arrivals for rt_stop_times 2025-11-05:  2025-11-05: 0:10:51.471830
 2025-11-17 15:43:37.574 | INFO     | interpolate_stop_arrival:interpolate_stop_arrivals:236 - interpolate arrivals for speedmap_segments 2025-11-05:  2025-11-05: 0:02:35.503949
 2026-02-02 18:13:27.813 | INFO     | interpolate_stop_arrival:interpolate_stop_arrivals:181 - interpolate arrivals for rt_stop_times 2025-12-17:  2025-12-17: 0:09:55.360755
+2026-02-02 22:35:58.958 | INFO     | interpolate_stop_arrival:interpolate_stop_arrivals:181 - interpolate arrivals for rt_stop_times 2025-12-17:  2025-12-17: 0:09:58.914735
+2026-02-03 10:03:59.171 | INFO     | interpolate_stop_arrival:interpolate_stop_arrivals:181 - interpolate arrivals for rt_stop_times 2025-12-17:  2025-12-17: 0:09:04.426576
+2026-02-03 11:51:30.117 | INFO     | interpolate_stop_arrival:interpolate_stop_arrivals:181 - interpolate arrivals for rt_stop_times 2025-12-17:  2025-12-17: 0:08:49.456770
+2026-02-03 12:55:54.249 | INFO     | interpolate_stop_arrival:interpolate_stop_arrivals:181 - interpolate arrivals for speedmap_segments 2025-12-17:  2025-12-17: 0:02:31.763735
+2026-02-03 13:14:46.925 | INFO     | interpolate_stop_arrival:interpolate_stop_arrivals:181 - interpolate arrivals for speedmap_segments 2025-12-17:  2025-12-17: 0:02:31.517210
+2026-02-03 15:11:01.692 | INFO     | interpolate_stop_arrival:interpolate_stop_arrivals:181 - interpolate arrivals for rt_stop_times 2026-01-14:  2026-01-14: 0:09:35.006378
+2026-02-03 15:32:27.473 | INFO     | interpolate_stop_arrival:interpolate_stop_arrivals:181 - interpolate arrivals for speedmap_segments 2026-01-14:  2026-01-14: 0:02:53.380369
diff --git a/rt_segment_speeds/logs/nearest_vp.log b/rt_segment_speeds/logs/nearest_vp.log
@@ -79,3 +79,10 @@
 2026-02-02 16:13:13.661 | INFO     | nearest_vp_to_stop:nearest_neighbor_for_stop:149 - nearest neighbor for rt_stop_times 2025-12-17: 0:41:58.668677
 2026-02-02 17:13:09.915 | INFO     | nearest_vp_to_stop:nearest_neighbor_for_stop:149 - nearest neighbor for rt_stop_times 2025-12-17: 0:43:15.982834
 2026-02-02 18:03:32.371 | INFO     | nearest_vp_to_stop:nearest_neighbor_for_stop:149 - nearest neighbor for rt_stop_times 2025-12-17: 0:42:34.619619
+2026-02-02 22:25:59.979 | INFO     | nearest_vp_to_stop:nearest_neighbor_for_stop:149 - nearest neighbor for rt_stop_times 2025-12-17: 0:41:02.484268
+2026-02-03 09:54:54.613 | INFO     | nearest_vp_to_stop:nearest_neighbor_for_stop:149 - nearest neighbor for rt_stop_times 2025-12-17: 0:40:50.178978
+2026-02-03 11:42:40.537 | INFO     | nearest_vp_to_stop:nearest_neighbor_for_stop:149 - nearest neighbor for rt_stop_times 2025-12-17: 0:42:00.180649
+2026-02-03 12:53:22.459 | INFO     | nearest_vp_to_stop:nearest_neighbor_for_stop:149 - nearest neighbor for speedmap_segments 2025-12-17: 0:10:00.773214
+2026-02-03 13:12:15.378 | INFO     | nearest_vp_to_stop:nearest_neighbor_for_stop:149 - nearest neighbor for speedmap_segments 2025-12-17: 0:09:49.639091
+2026-02-03 15:01:26.620 | INFO     | nearest_vp_to_stop:nearest_neighbor_for_stop:149 - nearest neighbor for rt_stop_times 2026-01-14: 0:43:25.044377
+2026-02-03 15:29:34.063 | INFO     | nearest_vp_to_stop:nearest_neighbor_for_stop:149 - nearest neighbor for speedmap_segments 2026-01-14: 0:10:52.328854
diff --git a/rt_segment_speeds/logs/speeds_by_segment_trip.log b/rt_segment_speeds/logs/speeds_by_segment_trip.log
@@ -84,3 +84,7 @@
 2025-11-17 13:17:41.461 | INFO     | stop_arrivals_to_speed:calculate_speed_from_stop_arrivals:154 - speeds by segment for speedmap_segments 2025-11-05: 0:02:33.109613
 2025-11-17 15:26:29.604 | INFO     | stop_arrivals_to_speed:calculate_speed_from_stop_arrivals:154 - speeds by segment for rt_stop_times 2025-11-05: 0:02:07.254360
 2025-11-17 15:45:46.585 | INFO     | stop_arrivals_to_speed:calculate_speed_from_stop_arrivals:154 - speeds by segment for speedmap_segments 2025-11-05: 0:02:00.259152
+2026-02-03 11:53:12.912 | INFO     | stop_arrivals_to_speed:calculate_speed_from_stop_arrivals:124 - speeds by segment for rt_stop_times 2025-12-17: 0:01:42.747516
+2026-02-03 13:16:56.294 | INFO     | stop_arrivals_to_speed:calculate_speed_from_stop_arrivals:124 - speeds by segment for speedmap_segments 2025-12-17: 0:02:02.048248
+2026-02-03 15:12:54.353 | INFO     | stop_arrivals_to_speed:calculate_speed_from_stop_arrivals:124 - speeds by segment for rt_stop_times 2026-01-14: 0:01:52.610136
+2026-02-03 15:34:46.792 | INFO     | stop_arrivals_to_speed:calculate_speed_from_stop_arrivals:124 - speeds by segment for speedmap_segments 2026-01-14: 0:02:10.417459
diff --git a/rt_segment_speeds/logs/speeds_timeofday.log b/rt_segment_speeds/logs/speeds_timeofday.log
@@ -54,3 +54,7 @@
 2025-11-17 13:23:54.285 | INFO     | speeds_by_time_of_day:aggregate_by_time_of_day:151 - speedmap_segments: time-of-day averages for 2025-11-05 execution time: 0:06:12.772068
 2025-11-17 15:30:35.072 | INFO     | speeds_by_time_of_day:aggregate_by_time_of_day:151 - rt_stop_times: time-of-day averages for 2025-11-05 execution time: 0:04:05.451189
 2025-11-17 15:51:31.053 | INFO     | speeds_by_time_of_day:aggregate_by_time_of_day:151 - speedmap_segments: time-of-day averages for 2025-11-05 execution time: 0:05:44.398517
+2026-02-03 11:57:45.856 | INFO     | speeds_by_time_of_day:aggregate_by_time_of_day:127 - rt_stop_times: time-of-day averages for 2025-12-17 execution time: 0:04:32.927050
+2026-02-03 13:23:25.389 | INFO     | speeds_by_time_of_day:aggregate_by_time_of_day:127 - speedmap_segments: time-of-day averages for 2025-12-17 execution time: 0:06:29.078839
+2026-02-03 15:17:45.055 | INFO     | speeds_by_time_of_day:aggregate_by_time_of_day:127 - rt_stop_times: time-of-day averages for 2026-01-14 execution time: 0:04:50.670633
+2026-02-03 15:41:11.752 | INFO     | speeds_by_time_of_day:aggregate_by_time_of_day:127 - speedmap_segments: time-of-day averages for 2026-01-14 execution time: 0:06:24.933008
diff --git a/rt_segment_speeds/scripts/Makefile b/rt_segment_speeds/scripts/Makefile
@@ -5,7 +5,7 @@ segmentize:
 	python cut_speedmap_segments.py
 
 rt_stop_times_pipeline:
-#	make segmentize
+	make segmentize
 	python pipeline_rt_stop_times.py
 	python average_summary_speeds.py
 	#python average_segment_speeds.py  # Now multi-day averages
diff --git a/rt_segment_speeds/scripts/average_summary_speeds.py b/rt_segment_speeds/scripts/average_summary_speeds.py
@@ -92,7 +92,7 @@ def trip_summary_speeds_by_time_of_day(
     )
 
     trip_avg = compute(df)[0]
-    trip_avg.to_parquet(f"{SEGMENT_GCS}{TRIP_FILE}_{analysis_date}.parquet")
+    gcs_pandas().data_frame_to_parquet(trip_avg, f"{SEGMENT_GCS}{TRIP_FILE}_{analysis_date}.parquet")
 
     end = datetime.datetime.now()
 
diff --git a/rt_segment_speeds/scripts/pipeline_speedmap.py b/rt_segment_speeds/scripts/pipeline_speedmap.py
@@ -56,7 +56,7 @@ def concatenate_speedmap_proxy_arrivals_with_remaining(analysis_date: str, confi
         .reset_index(drop=True)
     )
 
-    gcs_pandas.data_frame_to_parquet(df, f"{SEGMENT_GCS}{SPEEDMAP_STOP_ARRIVALS}_{analysis_date}.parquet")
+    gcs_pandas().data_frame_to_parquet(df, f"{SEGMENT_GCS}{SPEEDMAP_STOP_ARRIVALS}_{analysis_date}.parquet")
 
     del df, proxy_arrivals, other_arrivals
 
diff --git a/rt_segment_speeds/segment_speed_utils/project_vars.py b/rt_segment_speeds/segment_speed_utils/project_vars.py
@@ -11,7 +11,7 @@
 SHARED_GCS = GTFS_DATA_DICT.gcs_paths.SHARED_GCS
 PUBLIC_GCS = GTFS_DATA_DICT.gcs_paths.PUBLIC_GCS
 
-analysis_date = rt_dates.DATES["dec2025"]
+analysis_date = rt_dates.DATES["jan2026"]
 analysis_date_list = [analysis_date]
 
 oct2023_week = rt_dates.get_week("oct2023", exclude_wed=True)
diff --git a/rt_segment_speeds/segment_speed_utils/segment_calcs.py b/rt_segment_speeds/segment_speed_utils/segment_calcs.py
@@ -37,7 +37,7 @@ def derive_speed(
 
     df = df.assign(meters_elapsed=(df[max_dist] - df[min_dist]).abs())
 
-    if np.issubdtype(df[min_time].dtype, np.numeric):
+    if np.issubdtype(df[min_time].dtype, np.number):
         # If 2 time cols are already converted to seconds, just take difference
         df = df.assign(sec_elapsed=(df[max_time] - df[min_time]).abs())
     else:

Original file line number	Diff line number	Diff line change
`@@ -92,7 +92,7 @@ def trip_summary_speeds_by_time_of_day(`
`92`	`92`	`)`
`93`	`93`
`94`	`94`	`trip_avg = compute(df)[0]`
`95`		`- trip_avg.to_parquet(f"{SEGMENT_GCS}{TRIP_FILE}_{analysis_date}.parquet")`
	`95`	`+ gcs_pandas().data_frame_to_parquet(trip_avg, f"{SEGMENT_GCS}{TRIP_FILE}_{analysis_date}.parquet")`
`96`	`96`
`97`	`97`	`end = datetime.datetime.now()`
`98`	`98`
Original file line number	Diff line number	Diff line change
`@@ -56,7 +56,7 @@ def concatenate_speedmap_proxy_arrivals_with_remaining(analysis_date: str, confi`
`56`	`56`	`.reset_index(drop=True)`
`57`	`57`	`)`
`58`	`58`
`59`		`- gcs_pandas.data_frame_to_parquet(df, f"{SEGMENT_GCS}{SPEEDMAP_STOP_ARRIVALS}_{analysis_date}.parquet")`
	`59`	`+ gcs_pandas().data_frame_to_parquet(df, f"{SEGMENT_GCS}{SPEEDMAP_STOP_ARRIVALS}_{analysis_date}.parquet")`
`60`	`60`
`61`	`61`	`del df, proxy_arrivals, other_arrivals`
`62`	`62`