finished updating rk locodes

Author: amandaha8

dla/iija/_data_utils.py

@@ -194,7 +194,7 @@ def add_new_codes(df):
     #new_codes = update_program_code_list2()
 
     ## adding updated program codes 1/30/25
-    new_codes = update_program_code_list_2025
+    new_codes = update_program_code_list_2025()
     code_map = dict(new_codes[['iija_program_code', 'program_name']].values)
 
     df['program_code_description'] = df.program_code.map(code_map)

dla/iija/_script_utils.py

@@ -147,6 +147,109 @@ def identify_agency(df, identifier_col):
 
     return full_df
 
+def identify_agency2(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Fill in locodes, using the column rk_locode first
+    then using the original function from Natalie.
+    """
+    # Load dataframe with locodes
+    locodes_df = to_snakecase(
+        pd.read_excel(
+            f"gs://calitp-analytics-data/data-analyses/dla/e-76Obligated/locodes_updated7122021.xlsx"
+        )
+    ).rename(
+        columns={
+            "agency_name": "implementing_agency",
+        }
+    )
+
+    # Filter out for rows in which rk_locode is filled
+    filled_locode_df = df.loc[df.rk_locode.notna()].reset_index(drop=True)
+
+    # Merge the two dataframes
+    filled_locode_df2 = pd.merge(
+        filled_locode_df,
+        locodes_df,
+        left_on="rk_locode",
+        right_on="agency_locode",
+        how="left",
+        indicator=True,
+    )
+    display("Rows with locodes filled")
+    display(filled_locode_df2._merge.value_counts())
+
+    # Clean
+    filled_locode_df2 = filled_locode_df2.rename(
+        columns={
+            "agency_name": "implementing_agency",
+            "rk_locode": "implementing_agency_locode",
+        }
+    ).drop(
+        columns=[
+            "active_e76s______7_12_2021_",
+            "mpo_locode_fads",
+            "agency_locode",
+            "_merge",
+        ]
+    )
+
+    # Filter out for rows with missing locodes
+    missing_locode_df = (df.loc[(df.rk_locode.isna())].reset_index(drop=True)).drop(
+        columns=["rk_locode"]
+    )
+
+    # Fill in summary_recipient_defined_text_field_1_value
+    missing_locode_df.summary_recipient_defined_text_field_1_value = (
+        missing_locode_df.summary_recipient_defined_text_field_1_value.fillna("None")
+    )
+
+    # Try add_name_from_locode from _data_utils
+    missing_locode_df2 = _data_utils.add_name_from_locode(
+        missing_locode_df, "summary_recipient_defined_text_field_1_value"
+    )
+
+    # Concat all the dataframes
+    final_df = pd.concat([filled_locode_df2, missing_locode_df2])
+    display("Do the # of rows match?")
+    display(len(final_df) == len(df))
+
+    # More cleaning
+    county_base = to_snakecase(pd.read_excel(f"{GCS_FILE_PATH}/Copy of County.xlsx", sheet_name='County', header=[1]))
+    county_base.drop(columns =['unnamed:_0', 'unnamed:_4'], axis=1, inplace=True)
+    county_base['county_description'] = county_base['county_description'] + " County"
+    
+    county_district = (
+        locodes_df
+        >> group_by(_.district, _.county_name)
+        >> count(_.county_name)
+        >> select(_.district, _.county_name)
+        >> filter(_.county_name != "Multi-County", _.district != 53)
+    )
+    county_info = pd.merge(
+        county_base,
+        county_district,
+        how="left",
+        left_on="county_description",
+        right_on="county_name",
+    ).drop(columns=["county_name"])
+    mapping1 = dict(county_info[["county_code", "county_description"]].values)
+    mapping2 = dict(county_info[["county_code", "recipient_name"]].values)
+    mapping3 = dict(county_info[["county_code", "district"]].values)
+    
+    final_df["county_description"] = final_df.county_code.map(mapping1)
+    final_df["recipient_name"] = final_df.county_code.map(mapping2)
+    final_df["district"] = final_df.county_code.map(mapping3)
+    
+    final_df.loc[
+    final_df.county_name == "Statewide County", "county_name"] = "Statewide"
+
+    final_df["implementing_agency"] = final_df[
+        "implementing_agency"
+    ].fillna(value="Unknown")
+    final_df["county_name"] = final_df["county_name"].fillna(
+        value="Unknown"
+    )
+    return final_df
 
 def condense_df(df):
     """
@@ -536,7 +639,23 @@ def run_script(file_name, recipient_column, df_agg_level):
 
     return agg
 
+def run_script2(file_name, recipient_column, df_agg_level):
+    
+    ### Read in data
+    proj_list = to_snakecase(pd.read_excel(f"{GCS_FILE_PATH}/{file_name}"))
+    
+    ### run function to get new program codes
+    proj_cleaned = _data_utils.add_new_codes(proj_list)
+    
+    ## function that adds known agency name to df 
+    df = identify_agency2(proj_cleaned)
+    
+    ### run the data through the rest of the script
+    ### return a dataset that is aggregated at the project and program code
+    agg = get_clean_data(df, full_or_agg = df_agg_level)
 
+    return agg
+
 def export_to_gcs(df, export_date):
 
     ### pretty print the column names