Merge pull request #1421 from cal-itp/ah_gtfs

GTFS Digest

Author: amandaha8

_shared_utils/shared_utils/schedule_gtfs_dataset_key_multi_operator.yml

@@ -1,121 +1,40 @@
-directory: ./gtfs_digest/
-notebook: ./gtfs_digest/03_report.ipynb
-parts:
-- chapters:
-  - caption: 01 - Eureka (a253a8d7acd57657bb98050f37dd6b0f)
-    params:
-      District/Key: 01 - Eureka (a253a8d7acd57657bb98050f37dd6b0f)
-    sections:
-    - organization_name: City of Arcata
-    - organization_name: City of Eureka
-    - organization_name: Humboldt Transit Authority
-  - caption: 02 - Redding (73c79ccbfd681df300489226a158b9db)
-    params:
-      District/Key: 02 - Redding (73c79ccbfd681df300489226a158b9db)
-    sections:
-    - organization_name: Tehama County
-    - organization_name: Susanville Indian Rancheria
-  - caption: 02 - Redding (91af7482fde58c6261f386b732404e11)
-    params:
-      District/Key: 02 - Redding (91af7482fde58c6261f386b732404e11)
-    sections:
-    - organization_name: Shasta County
-    - organization_name: Redding Area Bus Authority
-  - caption: 03 - Marysville (6fda78099793184fe08dd78945d188c0)
-    params:
-      District/Key: 03 - Marysville (6fda78099793184fe08dd78945d188c0)
-    sections:
-    - organization_name: North Lake Tahoe Express
-    - organization_name: Tahoe Truckee Area Regional Transportation
-  - caption: 03 - Marysville (70c8a8b71c815224299523bf2115924a)
-    params:
-      District/Key: 03 - Marysville (70c8a8b71c815224299523bf2115924a)
-    sections:
-    - organization_name: Sacramento Regional Transit District
-    - organization_name: City of Rancho Cordova
-  - caption: 04 - Oakland (09e16227fc42c4fe90204a9d11581034)
-    params:
-      District/Key: 04 - Oakland (09e16227fc42c4fe90204a9d11581034)
-    sections:
-    - organization_name: Sonoma County
-    - organization_name: Cloverdale Transit
-  - caption: 04 - Oakland (1ebafaca8716652559b2017b6eedc4ef)
-    params:
-      District/Key: 04 - Oakland (1ebafaca8716652559b2017b6eedc4ef)
-    sections:
-    - organization_name: Solano County Transit
-    - organization_name: Solano Transportation Authority
-  - caption: 04 - Oakland (82f30e22dafe8156367297eb9a316c57)
-    params:
-      District/Key: 04 - Oakland (82f30e22dafe8156367297eb9a316c57)
-    sections:
-    - organization_name: City of Alameda
-    - organization_name: San Francisco Bay Area Water Emergency Transit Authority
-  - caption: 04 - Oakland (c2a40ce92e76ec5beb88c40df3cd3a67)
-    params:
-      District/Key: 04 - Oakland (c2a40ce92e76ec5beb88c40df3cd3a67)
-    sections:
-    - organization_name: City of Menlo Park
-    - organization_name: Commute.org
-  - caption: 05 - San Luis Obispo (b34f8d2270968f55f23f80b267df1d5f)
-    params:
-      District/Key: 05 - San Luis Obispo (b34f8d2270968f55f23f80b267df1d5f)
-    sections:
-    - organization_name: City of Santa Cruz
-    - organization_name: University of California, Santa Cruz
-  - caption: 07 - Los Angeles (1770249a5a2e770ca90628434d4934b1)
-    params:
-      District/Key: 07 - Los Angeles (1770249a5a2e770ca90628434d4934b1)
-    sections:
-    - organization_name: City of Ojai
-    - organization_name: Ventura County Transportation Commission
-    - organization_name: Gold Coast Transit District
-    - organization_name: City of Simi Valley
-    - organization_name: City of Moorpark
-    - organization_name: City of Thousand Oaks
-    - organization_name: City of Camarillo
-  - caption: 07 - Los Angeles (a37760dde6b9fdcb76b82e57afab7274)
-    params:
-      District/Key: 07 - Los Angeles (a37760dde6b9fdcb76b82e57afab7274)
-    sections:
-    - organization_name: Greyhound
-    - organization_name: FlixBus
-  - caption: 07 - Los Angeles (f74424acf8c41e4c1e9fd42838c4875c)
-    params:
-      District/Key: 07 - Los Angeles (f74424acf8c41e4c1e9fd42838c4875c)
-    sections:
-    - organization_name: City of Duarte
-    - organization_name: Foothill Transit
-  - caption: 07 - Los Angeles / Ventura (1770249a5a2e770ca90628434d4934b1)
-    params:
-      District/Key: 07 - Los Angeles / Ventura (1770249a5a2e770ca90628434d4934b1)
-    sections:
-    - organization_name: City of Camarillo
-    - organization_name: Gold Coast Transit District
-    - organization_name: City of Moorpark
-    - organization_name: City of Ojai
-    - organization_name: City of Simi Valley
-    - organization_name: City of Thousand Oaks
-    - organization_name: Ventura County Transportation Commission
-  - caption: 07 - Los Angeles / Ventura (a37760dde6b9fdcb76b82e57afab7274)
-    params:
-      District/Key: 07 - Los Angeles / Ventura (a37760dde6b9fdcb76b82e57afab7274)
-    sections:
-    - organization_name: Greyhound
-    - organization_name: FlixBus
-  - caption: 07 - Los Angeles / Ventura (f74424acf8c41e4c1e9fd42838c4875c)
-    params:
-      District/Key: 07 - Los Angeles / Ventura (f74424acf8c41e4c1e9fd42838c4875c)
-    sections:
-    - organization_name: City of Duarte
-    - organization_name: Foothill Transit
-  - caption: 11 - San Diego (baeeb157e85a901e47b828ef9fe75091)
-    params:
-      District/Key: 11 - San Diego (baeeb157e85a901e47b828ef9fe75091)
-    sections:
-    - organization_name: San Diego Metropolitan Transit System
-    - organization_name: Flagship Cruises and Events Inc.
-    - organization_name: San Diego International Airport
-readme: ./gtfs_digest/README.md
-title: This file contains the Schedule GTFS Datset Keys with multiple operators associated
-  with it.
+City of Eureka:
+- City of Arcata
+City of Moorpark:
+- City of Camarillo
+City of Ojai:
+- City of Camarillo
+City of Simi Valley:
+- City of Camarillo
+City of Thousand Oaks:
+- City of Camarillo
+Commute.org:
+- City of Menlo Park
+Foothill Transit:
+- City of Duarte
+Gold Coast Transit District:
+- City of Camarillo
+Greyhound:
+- FlixBus
+Humboldt Transit Authority:
+- City of Arcata
+Sacramento Regional Transit District:
+- City of Rancho Cordova
+San Diego International Airport:
+- Flagship Cruises and Events Inc.
+San Diego Metropolitan Transit System:
+- Flagship Cruises and Events Inc.
+Shasta County:
+- Redding Area Bus Authority
+Solano Transportation Authority:
+- Solano County Transit
+Sonoma County:
+- Cloverdale Transit
+Tahoe Truckee Area Regional Transportation:
+- North Lake Tahoe Express
+Tehama County:
+- Susanville Indian Rancheria
+University of California, Santa Cruz:
+- City of Santa Cruz
+Ventura County Transportation Commission:
+- City of Camarillo

_shared_utils/shared_utils/schedule_gtfs_keys_multi_orgs.py

@@ -11,7 +11,7 @@
 from update_vars import GTFS_DATA_DICT
 
 sys.path.append("../../gtfs_digest/")
-
+import _operators_prep
 
 def count_orgs(df: pd.DataFrame) -> list:
     """
@@ -23,96 +23,77 @@ def count_orgs(df: pd.DataFrame) -> list:
     """
     agg1 = (
         df.groupby(["caltrans_district", "schedule_gtfs_dataset_key"])
-        .agg({"organization_name": "nunique"})
+        .agg({"repeated_organization_name": "nunique"})
         .reset_index()
     )
 
     # Filter out rows with more than 1 organization_name
-    agg1 = agg1.loc[agg1.organization_name > 1].reset_index(drop=True)
+    agg1 = agg1.loc[agg1.repeated_organization_name > 1].reset_index(drop=True)
     # Grab schedule_gtfs_datset_key into a list
     multi_org_list = list(agg1.schedule_gtfs_dataset_key.unique())
     return multi_org_list
 
-
-def find_schd_keys_multi_ops() -> pd.DataFrame:
+def find_schd_keys_multi_ops() -> dict:
     """
     Return a dataframe with all the schedule_gtfs_dataset_keys
     that have more than one organization_name that corresponds to it.
     This way, we won't include duplicate organizations when publishing
     our GTFS products.
     """
-    schd_vp_url = f"{GTFS_DATA_DICT.digest_tables.dir}{GTFS_DATA_DICT.digest_tables.route_schedule_vp}.parquet"
-
-    subset = [
-        "caltrans_district",
-        "schedule_gtfs_dataset_key",
-        "organization_name",
-        "service_date",
-    ]
-
-    sort_cols = [
-        "caltrans_district",
-        "service_date",
-        "schedule_gtfs_dataset_key",
-    ]
-
-    schd_vp_df = pd.read_parquet(
-        schd_vp_url,
-        filters=[[("sched_rt_category", "in", ["schedule_and_vp", "schedule_only"])]],
-        columns=subset,
-    )
-
-    # Sort dataframe to keep the  row for district/gtfs_key for the most
-    # current date
-    schd_vp_df2 = schd_vp_df.dropna(subset="caltrans_district").sort_values(by=sort_cols, ascending=[True, False, True])
-    schd_vp_df3 = schd_vp_df2.drop_duplicates(
-        subset=[
-            "organization_name",
-            "schedule_gtfs_dataset_key",
-            "caltrans_district",
-        ]
-    )
-
-    # Aggregate the dataframe to find schedule_gtfs_dataset_keys
-    # With multiple organization_names.
-    multi_orgs_list = count_orgs(schd_vp_df3)
-
-    # Filter out the dataframe to only include schedule_gtfs_keys with multiple orgs
-    schd_vp_df4 = schd_vp_df3.loc[schd_vp_df3.schedule_gtfs_dataset_key.isin(multi_orgs_list)].reset_index(drop=True)
-
-    # Drop duplicates for organization_name
-    schd_vp_df5 = schd_vp_df4.drop_duplicates(subset=["caltrans_district", "organization_name"]).reset_index(drop=True)
-
-    # Aggregate the dataframe to find schedule_gtfs_dataset_keys
-    # with multiple organization_names once more.
-    multi_orgs_list2 = count_orgs(schd_vp_df5)
-
-    # Filter one last time to only include schedule_gtfs_keys with multiple orgs
-    schd_vp_df6 = schd_vp_df5.loc[schd_vp_df5.schedule_gtfs_dataset_key.isin(multi_orgs_list2)].reset_index(drop=True)
-
-    # Clean
-    schd_vp_df6 = schd_vp_df6.drop(columns=["service_date"])
-    schd_vp_df6["combo"] = schd_vp_df6.caltrans_district + " (" + schd_vp_df6.schedule_gtfs_dataset_key + ")"
-
-    return schd_vp_df6
 
+    # Load in the various dataframes that create the GTFS Digest portfolio site yaml
+    one_to_many_df, one_to_one_df, final = _operators_prep.operators_schd_vp_rt()
+    
+    # Subset and clean the dataframes
+    subset_cols = ["schedule_gtfs_dataset_key", "caltrans_district", "organization_name"]
+    
+    # This dataframe displays the relationship of 1 schedule dataset key to many
+    # organization names
+    one_to_many_df = one_to_many_df[subset_cols]
+    one_to_many_df = one_to_many_df.rename(
+    columns={"organization_name": "repeated_organization_name"}
+)
+    
+    # This dataframe displays the relationship of 1 schedule dataset key to 1
+    # organization name
+    one_to_one_df = one_to_one_df[subset_cols]
+    one_to_one_df = one_to_one_df.rename(
+    columns={"organization_name": "kept_organization_name"}
+)
+    # Merge the two dataframes 
+    m1 = pd.merge(
+    one_to_one_df,
+    one_to_many_df,
+    on=["schedule_gtfs_dataset_key", "caltrans_district"],
+)
+    
+    # Find the schedule_dataset_keys with more than one organization_name
+    # and filter out any rows that don't meet this criteria. 
+    multiple_organizations_list = count_orgs(m1)
+    m2 = m1.loc[m1.schedule_gtfs_dataset_key.isin(multiple_organizations_list)]
+    
+    # Delete the rows that house the organization name we use for the portfolio
+    m2["kept_name_bool"] = m2.kept_organization_name == m2.repeated_organization_name
+    m3 = m2.loc[m2.kept_name_bool == False]
+    
+    # Clean and sort 
+    final_cols = ["kept_organization_name", "repeated_organization_name"]
+    m3 = m3.sort_values(by=final_cols)[final_cols]
+    
+    # Turn it into a dictionary
+    my_dict = m3.set_index("repeated_organization_name").T.to_dict("list")
+    return my_dict
 
 SITE_YML = "./schedule_gtfs_dataset_key_multi_operator.yml"
 
 if __name__ == "__main__":
-    df = find_schd_keys_multi_ops()
-
-    portfolio_utils.create_portfolio_yaml_chapters_with_sections(
-        SITE_YML,
-        df,
-        chapter_info={
-            "column": "combo",
-            "name": "District/Key",
-            "caption_prefix": "",
-            "caption_suffix": "",
-        },
-        section_info={
-            "column": "organization_name",
-            "name": "organization_name",
-        },
-    )
+    my_dict = find_schd_keys_multi_ops()
+    
+    with open(SITE_YML) as f:
+        site_yaml_dict = yaml.load(f, yaml.Loader)
+        
+    output = yaml.dump(my_dict)
+    
+    with open(SITE_YML, "w") as f:
+        f.write(output)
+