Add the code

Author: ThomasAFink

map_spatial_integration_haversine_mapping.py

@@ -0,0 +1,112 @@
+import folium
+import numpy as np
+import pandas as pd
+import os
+
+# Load the datasets (assuming they are in the same directory as the script)
+OS_PATH = os.path.dirname(os.path.realpath('__file__'))
+
+weather_sensors_df = pd.read_csv(os.path.join(OS_PATH, 'data/metr-la/sensors/metr_la_sensors_weather.csv'))
+traffic_sensors_df = pd.read_csv(os.path.join(OS_PATH, 'data/metr-la/sensors/metr_la_sensors_traffic.csv'))
+
+traffic_speed_df = pd.read_csv(os.path.join(OS_PATH, 'data/metr-la/traffic/speed.csv'))
+air_temp_df = pd.read_csv(os.path.join(OS_PATH, 'data/metr-la/weather/air_temp_set_1_fahrenheit.csv'))
+
+# Haversine formula to calculate the distance between two geographical points
+def haversine(lat1, lon1, lat2, lon2):
+    lat1, lon1, lat2, lon2 = map(np.radians, [lat1, lon1, lat2, lon2])
+    dlat = lat2 - lat1 
+    dlon = lon2 - lon1 
+    a = np.sin(dlat/2)**2 + np.cos(lat1) * np.cos(lat2) * np.sin(dlon/2)**2
+    c = 2 * np.arcsin(np.sqrt(a)) 
+    r = 6371  # Radius of Earth in kilometers
+    return c * r
+
+def find_nearest_weather_sensor(traffic_lat, traffic_lon, weather_df):
+    distances = weather_df.apply(lambda row: haversine(traffic_lat, traffic_lon, row['lat'], row['long']), axis=1)
+    return weather_df.iloc[distances.idxmin()]['detid']
+
+# Map each traffic sensor to its nearest weather sensor
+traffic_sensors_df['nearest_weather_sensor'] = traffic_sensors_df.apply(
+    lambda row: find_nearest_weather_sensor(row['lat'], row['long'], weather_sensors_df),
+    axis=1
+)
+
+# Dictionary mapping of traffic sensor to its nearest weather sensor
+sensor_to_weather_mapping = dict(zip(traffic_sensors_df['detid'], traffic_sensors_df['nearest_weather_sensor']))
+
+def calculate_centroid(df):
+    """
+    Calculate the centroid (geometric center) of a set of points.
+    
+    Parameters:
+    - df: DataFrame containing lat and long columns
+    
+    Returns:
+    - A tuple (centroid_lat, centroid_lon).
+    """
+    centroid_lat = df['lat'].mean()
+    centroid_lon = df['long'].mean()
+    
+    return centroid_lat, centroid_lon
+
+def generate_sensor_map(traffic_sensors_df, weather_sensors_df, sensor_to_weather_mapping):
+    """
+    Generate a map showing the traffic sensors and weather sensors.
+    
+    Parameters:
+    - traffic_sensors_df: DataFrame containing traffic sensor locations.
+    - weather_sensors_df: DataFrame containing weather sensor locations.
+    - sensor_to_weather_mapping: Dictionary mapping traffic sensors to their nearest weather sensor.
+    
+    Returns:
+    - A folium map object.
+    """
+
+    centroid_lat, centroid_lon = calculate_centroid(traffic_sensors_df)
+    
+    # Create a base map
+    m = folium.Map(location=[centroid_lat, centroid_lon], zoom_start=12)  # Centered around San Francisco
+
+    # Define distinct colors for weather sensors
+    colors = [
+        'red', 'blue', 'green', 'purple', 'orange', 'darkred',
+        'lightred', 'beige', 'darkblue', 'darkgreen', 'cadetblue',
+        'darkpurple', 'pink', 'lightblue', 'lightgreen',
+        'gray', 'black', 'lightgray'
+    ]
+
+    # Create a mapping from weather sensor to color
+    weather_sensor_to_color = {detid: colors[i % len(colors)] for i, detid in enumerate(weather_sensors_df['detid'].unique())}
+
+    # Plot the weather sensors on the map
+    for idx, row in weather_sensors_df.iterrows():
+        folium.CircleMarker(
+            location=(row['lat'], row['long']),
+            radius=8,
+            color=weather_sensor_to_color[row['detid']],
+            fill=True,
+            fill_color=weather_sensor_to_color[row['detid']]
+        ).add_to(m)
+
+    # Plot the traffic sensors on the map
+    for idx, row in traffic_sensors_df.iterrows():
+        nearest_weather_sensor = sensor_to_weather_mapping[row['detid']]
+        folium.CircleMarker(
+            location=(row['lat'], row['long']),
+            radius=4,
+            color=weather_sensor_to_color[nearest_weather_sensor],
+            fill=True,
+            fill_color=weather_sensor_to_color[nearest_weather_sensor]
+        ).add_to(m)
+
+    return m
+
+# Generate the map using the function
+# Save the merged dataframe to a CSV file
+output_folder = os.path.join(OS_PATH, 'output/metr-la')
+os.makedirs(output_folder, exist_ok=True)  # Create output folder if it doesn't exist
+
+map_output = generate_sensor_map(traffic_sensors_df, weather_sensors_df, sensor_to_weather_mapping)
+map_output.save(os.path.join(output_folder, 'sensor_map.html'))
+

spatial_integration_haversine_mapping.py

@@ -4,13 +4,12 @@
 
 # Load the datasets (assuming they are in the same directory as the script)
 OS_PATH = os.path.dirname(os.path.realpath('__file__'))
-weather_sensors_df = os.path.join(OS_PATH, 'data/pems-bay/sensors/pems_bay_sensors_weather.csv')
-traffic_sensors_df = os.path.join(OS_PATH, 'data/pems-bay/sensors/pems_bay_sensors_traffic.csv')
 
+weather_sensors_df = pd.read_csv(os.path.join(OS_PATH, 'data/metr-la/sensors/metr_la_sensors_weather.csv'))
+traffic_sensors_df = pd.read_csv(os.path.join(OS_PATH, 'data/metr-la/sensors/metr_la_sensors_traffic.csv'))
 
-traffic_speed_df = os.path.join(OS_PATH, 'data/pems-bay/traffic/speed.csv')
-air_temp_df = os.path.join(OS_PATH, 'data/pems-bay/weather/air_temp_set_1_fahrenheit.csv')
-
+traffic_speed_df = pd.read_csv(os.path.join(OS_PATH, 'data/metr-la/traffic/speed.csv'))
+air_temp_df = pd.read_csv(os.path.join(OS_PATH, 'data/metr-la/weather/air_temp_set_1_fahrenheit.csv'))
 
 # Haversine formula to calculate the distance between two geographical points
 def haversine(lat1, lon1, lat2, lon2):
@@ -35,32 +34,51 @@ def find_nearest_weather_sensor(traffic_lat, traffic_lon, weather_df):
 # Dictionary mapping of traffic sensor to its nearest weather sensor
 sensor_to_weather_mapping = dict(zip(traffic_sensors_df['detid'], traffic_sensors_df['nearest_weather_sensor']))
 
-# Merge weather data with traffic data in chunks
-chunk_size = 5000
+# Function to merge data based on timestamps
+def merge_data_on_timestamps(traffic_speed_df, air_temp_df, sensor_to_weather_mapping):
+    # Initialize merged dataframe with DATETIMESTAMP column
+    merged_df = pd.DataFrame()
+    merged_df["DATETIMESTAMP"] = traffic_speed_df["DATETIMESTAMP"]
+    
+    # Iterate through each sensor in the traffic_speed_df
+    for sensor in traffic_speed_df.columns[1:]:
+        # Copy the speed data
+        merged_df[sensor] = traffic_speed_df[sensor]
+        
+        # Find corresponding weather sensor
+        weather_sensor = sensor_to_weather_mapping[int(sensor)]
+        
+        # Find corresponding temperature data
+        if f"{weather_sensor}" in air_temp_df.columns:
+            merged_df[f"{sensor}_temp"] = air_temp_df[f"{weather_sensor}"].reindex_like(traffic_speed_df)
+    
+    return merged_df
 
-def merge_weather_with_traffic(traffic_data, weather_data, sensor_to_weather_mapping):
-    merged_data = traffic_data[['DATETIMESTAMP']].copy()
-    for sensor in traffic_data.columns:
-        if sensor != "DATETIMESTAMP":
-            nearest_weather_sensor = sensor_to_weather_mapping.get(sensor, None)
-            if nearest_weather_sensor:
-                weather_chunk = weather_data[['Date_Time', nearest_weather_sensor]].rename(columns={nearest_weather_sensor: sensor})
-                merged_data = merged_data.merge(weather_chunk, left_on="DATETIMESTAMP", right_on="Date_Time", how="left").drop(columns="Date_Time")
-    return merged_data
+# Reorder columns function
+def reorder_columns(merged_df):
+    # Create a list for the reordered columns
+    speed_cols = [col for col in merged_df.columns if '_temp' not in col]
+    temp_cols = [col for col in merged_df.columns if '_temp' in col]
+    # Reorder the columns
+    final_order = speed_cols + temp_cols
+    merged_df = merged_df[final_order]
+    return merged_df
 
-# Example of merging air temperature data with traffic data in chunks
-merged_chunks = []
-for start_row in range(0, traffic_speed_df.shape[0], chunk_size):
-    end_row = start_row + chunk_size
-    traffic_chunk = traffic_speed_df.iloc[start_row:end_row]
-    merged_chunk = merge_weather_with_traffic(traffic_chunk, air_temp_df, sensor_to_weather_mapping)
-    merged_chunks.append(merged_chunk)
+# Rename speed columns with "_speed" suffix
+def rename_speed_columns(merged_df):
+    speed_cols = [col for col in merged_df.columns if '_temp' not in col and col != "DATETIMESTAMP"]
+    speed_columns_renamed = {col: f"{col}_speed" for col in speed_cols}
+    merged_df.rename(columns=speed_columns_renamed, inplace=True)
+    return merged_df
 
+# Merge, reorder, rename and save the dataframe
+merged_df = merge_data_on_timestamps(traffic_speed_df, air_temp_df, sensor_to_weather_mapping)
+merged_df = reorder_columns(merged_df)
+merged_df = rename_speed_columns(merged_df)
 
-final_merged_df = pd.concat(merged_chunks, axis=0)
 
-# Define output path and save the final merged dataframe
-output_folder = os.path.join(OS_PATH, 'output')
+# Save the merged dataframe to a CSV file
+output_folder = os.path.join(OS_PATH, 'output/metr-la')
 os.makedirs(output_folder, exist_ok=True)  # Create output folder if it doesn't exist
-output_file_path = os.path.join(output_folder, 'merged_traffic_weather_data.csv')
-final_merged_df.to_csv(output_file_path, index=False)
+merged_file_path = os.path.join(output_folder, 'merged_speed_traffic_and_air_temperature_data.csv')
+merged_df.to_csv(merged_file_path, index=False)