From 5209b331f2a8e3646eb6f974edc8200577f8947f Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Fri, 1 Sep 2023 12:11:31 +0100 Subject: [PATCH 001/118] TODO: some modules use equipment when talking about consumables --- src/tlo/methods/care_of_women_during_pregnancy.py | 2 ++ src/tlo/methods/labour.py | 2 ++ src/tlo/methods/newborn_outcomes.py | 2 ++ src/tlo/methods/rti.py | 2 ++ 4 files changed, 8 insertions(+) diff --git a/src/tlo/methods/care_of_women_during_pregnancy.py b/src/tlo/methods/care_of_women_during_pregnancy.py index 42e026478c..9fe9eb4215 100644 --- a/src/tlo/methods/care_of_women_during_pregnancy.py +++ b/src/tlo/methods/care_of_women_during_pregnancy.py @@ -205,6 +205,8 @@ def get_and_store_pregnancy_item_codes(self): get_list_of_items = pregnancy_helper_functions.get_list_of_items # ---------------------------------- BLOOD TEST EQUIPMENT --------------------------------------------------- + # TODO: As we now consider both consumables and equipment, using 'equipment' when meaning consumables is + # confusing self.item_codes_preg_consumables['blood_test_equipment'] = \ get_list_of_items(self, ['Blood collecting tube, 5 ml', 'Cannula iv (winged with injection pot) 18_each_CMST', diff --git a/src/tlo/methods/labour.py b/src/tlo/methods/labour.py index 0f6d7d134e..3fa400f2f0 100644 --- a/src/tlo/methods/labour.py +++ b/src/tlo/methods/labour.py @@ -686,6 +686,8 @@ def get_and_store_labour_item_codes(self): get_list_of_items = pregnancy_helper_functions.get_list_of_items # ---------------------------------- IV DRUG ADMIN EQUIPMENT ------------------------------------------------- + # TODO: As we now consider both consumables and equipment, using 'equipment' when meaning consumables is + # confusing self.item_codes_lab_consumables['iv_drug_equipment'] = \ get_list_of_items(self, ['Cannula iv (winged with injection pot) 18_each_CMST', 'Giving set iv administration + needle 15 drops/ml_each_CMST', diff --git a/src/tlo/methods/newborn_outcomes.py b/src/tlo/methods/newborn_outcomes.py index 513b644746..7c240f8bc7 100644 --- a/src/tlo/methods/newborn_outcomes.py +++ b/src/tlo/methods/newborn_outcomes.py @@ -380,6 +380,8 @@ def get_and_store_newborn_item_codes(self): get_list_of_items = pregnancy_helper_functions.get_list_of_items # ---------------------------------- IV DRUG ADMIN EQUIPMENT ------------------------------------------------- + # TODO: As we now consider both consumables and equipment, using 'equipment' when meaning consumables is + # confusing self.item_codes_nb_consumables['iv_drug_equipment'] = \ get_list_of_items(self, ['Cannula iv (winged with injection pot) 18_each_CMST', 'Giving set iv administration + needle 15 drops/ml_each_CMST', diff --git a/src/tlo/methods/rti.py b/src/tlo/methods/rti.py index aac0129cf6..0d788aaf6c 100644 --- a/src/tlo/methods/rti.py +++ b/src/tlo/methods/rti.py @@ -4665,6 +4665,8 @@ def apply(self, person_id, squeeze_factor): # administer antibiotic get_item_code("Ampicillin injection 500mg, PFR_each_CMST"): 1, # equipment used by surgeon, gloves and facemask + # TODO: As we now consider both consumables and equipment, using 'equipment' when meaning consumables is + # confusing get_item_code('Disposables gloves, powder free, 100 pieces per box'): 1, get_item_code('surgical face mask, disp., with metal nose piece_50_IDA'): 1, # request syringe From 5158aa22a6c3cb848f352c05b7e52fdbc1530574 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Tue, 5 Sep 2023 00:11:12 +0100 Subject: [PATCH 002/118] breast_cancer: dummy used_equipment added where Andrew requested diff --git src/tlo/methods/breast_cancer.py src/tlo/methods/breast_cancer.py index 1ce9ad2bf..56c935fba 100644 --- src/tlo/methods/breast_cancer.py +++ src/tlo/methods/breast_cancer.py @@ -666,6 +666,8 @@ class HSI_BreastCancer_Investigation_Following_breast_lump_discernible(HSI_Event # Use a biopsy to diagnose whether the person has breast Cancer: # todo: request consumables needed for this + self.used_equipment = {'Slice Master sample processing Unit', 'Paraffin Dispense', 'Whatever used with biopsy', + 'Mammograph maybe?'} dx_result = hs.dx_manager.run_dx_test( dx_tests_to_run='biopsy_for_breast_cancer_given_breast_lump_discernible', @@ -759,6 +761,9 @@ class HSI_BreastCancer_StartTreatment(HSI_Event, IndividualScopeEventMixin): df.at[person_id, "brc_date_treatment"] = self.sim.date df.at[person_id, "brc_stage_at_which_treatment_given"] = df.at[person_id, "brc_status"] + # Record used equipment + self.used_equipment = 'Anything used for mastectomy as I guess this is about' + # Schedule a post-treatment check for 12 months: hs.schedule_hsi_event( hsi_event=HSI_BreastCancer_PostTreatmentCheck( --- src/tlo/methods/breast_cancer.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/tlo/methods/breast_cancer.py b/src/tlo/methods/breast_cancer.py index 1ce9ad2bf6..56c935fba2 100644 --- a/src/tlo/methods/breast_cancer.py +++ b/src/tlo/methods/breast_cancer.py @@ -666,6 +666,8 @@ def apply(self, person_id, squeeze_factor): # Use a biopsy to diagnose whether the person has breast Cancer: # todo: request consumables needed for this + self.used_equipment = {'Slice Master sample processing Unit', 'Paraffin Dispense', 'Whatever used with biopsy', + 'Mammograph maybe?'} dx_result = hs.dx_manager.run_dx_test( dx_tests_to_run='biopsy_for_breast_cancer_given_breast_lump_discernible', @@ -759,6 +761,9 @@ def apply(self, person_id, squeeze_factor): df.at[person_id, "brc_date_treatment"] = self.sim.date df.at[person_id, "brc_stage_at_which_treatment_given"] = df.at[person_id, "brc_status"] + # Record used equipment + self.used_equipment = 'Anything used for mastectomy as I guess this is about' + # Schedule a post-treatment check for 12 months: hs.schedule_hsi_event( hsi_event=HSI_BreastCancer_PostTreatmentCheck( From e554dd74bb40e466e8a93bb33039951cbe4db8e0 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Tue, 5 Sep 2023 00:15:40 +0100 Subject: [PATCH 003/118] co: dummy used_equipment added for methods where Emi listed some --- src/tlo/methods/contraception.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/tlo/methods/contraception.py b/src/tlo/methods/contraception.py index 9ddbc6aac9..4def5a649d 100644 --- a/src/tlo/methods/contraception.py +++ b/src/tlo/methods/contraception.py @@ -1170,7 +1170,8 @@ def apply(self, person_id, squeeze_factor): items_all = {**items_essential, **items_optional} # Determine whether the contraception is administrated (ie all essential items are available), - # if so do log the availability of all items, if not set the contraception to "not_using": + # if so do log the availability of all items and record used equipment if any, if not set the contraception to + # "not_using": co_administrated = all(v for k, v in cons_available.items() if k in items_essential) if co_administrated: @@ -1194,6 +1195,13 @@ def apply(self, person_id, squeeze_factor): ) _new_contraceptive = self.new_contraceptive + + # Record used equipment when needed + if _new_contraceptive == 'female_sterilization': + self.used_equipment = {'Smt used to sterilize a woman'} + elif _new_contraceptive == 'IUD': + self.used_equipment = {'Equipment used when performing IUD'} + else: _new_contraceptive = "not_using" From e4935e36ec29225a71d7ebe2deca931ed31fb561 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Tue, 5 Sep 2023 00:24:02 +0100 Subject: [PATCH 004/118] healthsystem: annual equipment summary log by fac. level --- src/tlo/methods/healthsystem.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 6838f56705..4fd39fcb25 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -182,6 +182,7 @@ def __init__(self, module, *args, **kwargs): self._received_info_about_bed_days = None self.expected_time_requests = {} self.facility_info = None + self.used_equipment = set() @property def bed_days_allocated_to_this_event(self): @@ -1810,6 +1811,7 @@ def record_hsi_event(self, hsi_event, actual_appt_footprint=None, squeeze_factor squeeze_factor=_squeeze_factor, did_run=did_run, priority=priority, + equipment=hsi_event.used_equipment, ) def write_to_hsi_log( @@ -1820,6 +1822,7 @@ def write_to_hsi_log( squeeze_factor: float, did_run: bool, priority: int, + equipment: set, ): """Write the log `HSI_Event` and add to the summary counter.""" logger.debug( @@ -1849,6 +1852,7 @@ def write_to_hsi_log( squeeze_factor=squeeze_factor, appt_footprint=event_details.appt_footprint, level=event_details.facility_level, + equipment=equipment, ) def call_and_record_never_ran_hsi_event(self, hsi_event, priority=None): @@ -2671,6 +2675,7 @@ def apply(self, population): squeeze_factor=0.0, priority=-1, did_run=True, + equipment=set() # TODO: explore more, should it be non-emtpy in some cases? ) # Restart the total footprint of all calls today, beginning with those due to existing in-patients. @@ -2725,6 +2730,7 @@ def _reset_internal_stores(self) -> None: self._appts = defaultdict(int) # Running record of the Appointments of `HSI_Event`s that have run self._appts_by_level = {_level: defaultdict(int) for _level in ('0', '1a', '1b', '2', '3', '4')} # <--Same as `self._appts` but also split by facility_level + self._equip_by_level = {_level: set() for _level in ('0', '1a', '1b', '2', '3', '4')} # Log HSI_Events that never ran to monitor shortcoming of Health System self._never_ran_treatment_ids = defaultdict(int) # As above, but for `HSI_Event`s that never ran @@ -2741,7 +2747,8 @@ def record_hsi_event(self, hsi_event_name: str, squeeze_factor: float, appt_footprint: Counter, - level: str + level: str, + equipment: set ) -> None: """Add information about an `HSI_Event` to the running summaries.""" @@ -2758,6 +2765,9 @@ def record_hsi_event(self, self._appts[appt_type] += number self._appts_by_level[level][appt_type] += number + # Update used equipment by level + self._equip_by_level[level].update(equipment) + def record_never_ran_hsi_event(self, treatment_id: str, hsi_event_name: str, @@ -2820,6 +2830,14 @@ def write_to_log_and_reset_counters(self): }, ) + logger_summary.info( + key="Equipment", + description="Sets of used equipment for each facility level in this calendar year.", + data={ + "Equipment_By_Level": self._equip_by_level, + }, + ) + self._reset_internal_stores() From 8b455bd6726d6b59e196aae3773a566cb0b00381 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Tue, 5 Sep 2023 16:15:18 +0100 Subject: [PATCH 005/118] breast_cancer: mastectomy dummy equipment fixed --- src/tlo/methods/breast_cancer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tlo/methods/breast_cancer.py b/src/tlo/methods/breast_cancer.py index 56c935fba2..5d8fabfcb2 100644 --- a/src/tlo/methods/breast_cancer.py +++ b/src/tlo/methods/breast_cancer.py @@ -762,7 +762,7 @@ def apply(self, person_id, squeeze_factor): df.at[person_id, "brc_stage_at_which_treatment_given"] = df.at[person_id, "brc_status"] # Record used equipment - self.used_equipment = 'Anything used for mastectomy as I guess this is about' + self.used_equipment = {'Anything used for mastectomy as I guess this is about'} # Schedule a post-treatment check for 12 months: hs.schedule_hsi_event( From 9f44e8560c8a151953f1f3cdb02ace3203bf2f06 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Wed, 6 Sep 2023 21:01:35 +0100 Subject: [PATCH 006/118] equipment_catalogue & utils: new script + a change in utils.py - to create equip. catalogue --- .../equipment/equipment_catalogue.py | 101 ++++++++++++++++++ src/tlo/analysis/utils.py | 8 +- 2 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 src/scripts/healthsystem/equipment/equipment_catalogue.py diff --git a/src/scripts/healthsystem/equipment/equipment_catalogue.py b/src/scripts/healthsystem/equipment/equipment_catalogue.py new file mode 100644 index 0000000000..c88a469c2f --- /dev/null +++ b/src/scripts/healthsystem/equipment/equipment_catalogue.py @@ -0,0 +1,101 @@ +import argparse +import pandas as pd +from pathlib import Path +from tlo.analysis.utils import extract_results +from functools import reduce + + +def get_annual_equipment_declarations_by_levels(results_folder: Path) -> pd.DataFrame: + """Return pd.DataFrame gives the simulated annual equipment declaration by facility levels for each simulated + year.""" + + def get_equipment_declaration_by_levels(_df): + """Get the equipment declaration by facility levels for the year.""" + + def unpack_dict_in_series(_raw: pd.Series): + # Create an empty DataFrame to store the data + df = pd.DataFrame() + + # Iterate through the dictionary items + for col_name, mydict in _raw.items(): + for date, inner_dict in mydict.items(): + # Convert the inner_dict to a list of dictionaries with 'date' + data = [{'date': date, 'fac_level': inner_dict_key, 'value': inner_dict_set} for + inner_dict_key, inner_dict_set in inner_dict.items()] + # Create a DataFrame from the list with date & fac_level as indexes + temp_df = pd.DataFrame(data) + temp_df.set_index(['date', 'fac_level'], inplace=True) + temp_df.columns = [None] + + # Concatenate the temporary DataFrame to the result DataFrame + df = pd.concat([df, temp_df]) + + # print(f"\ndf\n {df}") + df.columns = [None] + + return df + + return _df \ + .set_index('date') \ + .pipe(unpack_dict_in_series) \ + .stack() \ + .droplevel(level=2) + + return extract_results( + results_folder, + module='tlo.methods.healthsystem.summary', + key='Equipment', + custom_generate_series=get_equipment_declaration_by_levels + ) + + +def create_equipment_catalogue(results_folder: Path, output_folder: Path): + # Declare path for output file from this script + output_file_name = 'equipment_catalogue_by_level.csv' + output_detailed_file_name = 'equipment_catalogue_by_date_level_sim.csv' + + sim_equipment = get_annual_equipment_declarations_by_levels(results_folder) + sim_equipment_df = pd.DataFrame(sim_equipment) + sim_equipment_df.index.names = ['date', 'fac_level'] + + # Save the detailed CSV + sim_equipment_df.to_csv(output_folder / output_detailed_file_name) + print('equipment_catalogue_by_date_level_sim.csv saved.') + + # Prepare a catalogue only by facility levels + # Define a custom aggregation function to combine sets in columns for each row + def combine_sets(row): + combined_set = set() + for col in row: + combined_set.update(col) + return combined_set + + # Apply the custom aggregation function to each row + sim_equipment_by_level_df = sim_equipment_df.copy() + sim_equipment_by_level_df['equipment'] = sim_equipment_by_level_df.apply(combine_sets, axis=1) + # Group by 'fac_level' and join rows with the same 'fac_level' into one set + sim_equipment_by_level_df.reset_index(inplace=True) + sim_equipment_by_level_df = sim_equipment_by_level_df.groupby('fac_level')['equipment'].apply( + lambda x: list(set.union(*x)) + ).reset_index() + + # Explode the 'equipment' column to separate elements into rows + sim_equipment_by_level_df = sim_equipment_by_level_df.explode('equipment', ignore_index=True).set_index('fac_level') + + # Save the CSV equipment catalogue + sim_equipment_by_level_df.to_csv(output_folder / output_file_name) + print('equipment_catalogue_by_level.csv saved.') + + return 0 + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("results_folder", type=Path) + args = parser.parse_args() + + create_equipment_catalogue( + results_folder=args.results_folder, + output_folder=args.results_folder, + ) +# NB. Edit run configuration, the Parameters: "./outputs/sejjej5@ucl.ac.uk/long_run_all_diseases-2023-09-04T233551Z" diff --git a/src/tlo/analysis/utils.py b/src/tlo/analysis/utils.py index f3fc29108e..a741eefee3 100644 --- a/src/tlo/analysis/utils.py +++ b/src/tlo/analysis/utils.py @@ -282,6 +282,9 @@ def generate_series(dataframe: pd.DataFrame) -> pd.Series: # get number of draws and numbers of runs info = get_scenario_info(results_folder) + def is_number(element): + return isinstance(element, (int, float)) + # Collect results from each draw/run res = dict() for draw in range(info['number_of_draws']): @@ -293,7 +296,10 @@ def generate_series(dataframe: pd.DataFrame) -> pd.Series: df: pd.DataFrame = load_pickled_dataframes(results_folder, draw, run, module)[module][key] output_from_eval: pd.Series = generate_series(df) assert pd.Series == type(output_from_eval), 'Custom command does not generate a pd.Series' - res[draw_run] = output_from_eval * get_multiplier(draw, run) + if output_from_eval.apply(is_number).all(): + res[draw_run] = output_from_eval * get_multiplier(draw, run) + else: + res[draw_run] = output_from_eval except KeyError: # Some logs could not be found - probably because this run failed. From 0ff003b511901f93026632844ac7829c128c534f Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Thu, 7 Sep 2023 17:12:45 +0100 Subject: [PATCH 007/118] equipment_catalogue: PEP8 --- src/scripts/healthsystem/equipment/equipment_catalogue.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/scripts/healthsystem/equipment/equipment_catalogue.py b/src/scripts/healthsystem/equipment/equipment_catalogue.py index c88a469c2f..fdaca54515 100644 --- a/src/scripts/healthsystem/equipment/equipment_catalogue.py +++ b/src/scripts/healthsystem/equipment/equipment_catalogue.py @@ -1,8 +1,9 @@ import argparse -import pandas as pd from pathlib import Path + +import pandas as pd + from tlo.analysis.utils import extract_results -from functools import reduce def get_annual_equipment_declarations_by_levels(results_folder: Path) -> pd.DataFrame: From f73693913dc226c2bf17dab822325047bbc71681 Mon Sep 17 00:00:00 2001 From: Eva Janouskova <48157464+EvaJanouskova@users.noreply.github.com> Date: Tue, 12 Sep 2023 19:53:10 +0200 Subject: [PATCH 008/118] healthsystem: sort equipment for log Co-authored-by: Tim Hallett <39991060+tbhallett@users.noreply.github.com> --- src/tlo/methods/healthsystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 4fd39fcb25..1a07595d9e 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -2834,7 +2834,7 @@ def write_to_log_and_reset_counters(self): key="Equipment", description="Sets of used equipment for each facility level in this calendar year.", data={ - "Equipment_By_Level": self._equip_by_level, + "Equipment_By_Level": sorted(self._equip_by_level), }, ) From 745d4aba4eae181da569f49b4e6cd99bfa012e28 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Tue, 12 Sep 2023 18:26:35 +0100 Subject: [PATCH 009/118] equipment_catalogue: comment updated --- src/scripts/healthsystem/equipment/equipment_catalogue.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/scripts/healthsystem/equipment/equipment_catalogue.py b/src/scripts/healthsystem/equipment/equipment_catalogue.py index fdaca54515..ecbc206aba 100644 --- a/src/scripts/healthsystem/equipment/equipment_catalogue.py +++ b/src/scripts/healthsystem/equipment/equipment_catalogue.py @@ -8,7 +8,8 @@ def get_annual_equipment_declarations_by_levels(results_folder: Path) -> pd.DataFrame: """Return pd.DataFrame gives the simulated annual equipment declaration by facility levels for each simulated - year.""" + year. + NB. healthsystem.summary logger required to have been set at the level INFO or higher.""" def get_equipment_declaration_by_levels(_df): """Get the equipment declaration by facility levels for the year.""" From e7c52d82136be4457938ab7845fd3d11712f14bb Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Tue, 12 Sep 2023 19:10:01 +0100 Subject: [PATCH 010/118] rti: unified use of consumables/equipment terms --- src/tlo/methods/rti.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/tlo/methods/rti.py b/src/tlo/methods/rti.py index 0d788aaf6c..f92d751e14 100644 --- a/src/tlo/methods/rti.py +++ b/src/tlo/methods/rti.py @@ -4664,9 +4664,7 @@ def apply(self, person_id, squeeze_factor): get_item_code('Pethidine, 50 mg/ml, 2 ml ampoule'): 1, # administer antibiotic get_item_code("Ampicillin injection 500mg, PFR_each_CMST"): 1, - # equipment used by surgeon, gloves and facemask - # TODO: As we now consider both consumables and equipment, using 'equipment' when meaning consumables is - # confusing + # consumables used by surgeon, gloves and facemask get_item_code('Disposables gloves, powder free, 100 pieces per box'): 1, get_item_code('surgical face mask, disp., with metal nose piece_50_IDA'): 1, # request syringe @@ -5002,7 +5000,7 @@ def apply(self, person_id, squeeze_factor): get_item_code('Pethidine, 50 mg/ml, 2 ml ampoule'): 1, # administer antibiotic get_item_code("Ampicillin injection 500mg, PFR_each_CMST"): 1, - # equipment used by surgeon, gloves and facemask + # consumables used by surgeon, gloves and facemask get_item_code('Disposables gloves, powder free, 100 pieces per box'): 1, get_item_code('surgical face mask, disp., with metal nose piece_50_IDA'): 1, # request syringe From b523d584d1134f7b7a6fc71833ef2f2644a8579a Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Tue, 12 Sep 2023 19:15:44 +0100 Subject: [PATCH 011/118] hs, brc, co: used_equipment renamed to EQUIPMENT; if equip always same for HSI - set in __init__, otherwise updated in apply diff --git src/tlo/methods/breast_cancer.py src/tlo/methods/breast_cancer.py index 5d8fabfcb..26155729a 100644 --- src/tlo/methods/breast_cancer.py +++ src/tlo/methods/breast_cancer.py @@ -646,6 +646,8 @@ class HSI_BreastCancer_Investigation_Following_breast_lump_discernible(HSI_Event self.TREATMENT_ID = "BreastCancer_Investigation" self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint({"Over5OPD": 1, "Mammography": 1}) self.ACCEPTED_FACILITY_LEVEL = '3' # Biopsy only available at level 3 and above. + self.EQUIPMENT = {'Slice Master sample processing Unit', 'Paraffin Dispense', 'Whatever used with biopsy'} + # biopsy always performed with this HSI def apply(self, person_id, squeeze_factor): df = self.sim.population.props @@ -666,8 +668,6 @@ class HSI_BreastCancer_Investigation_Following_breast_lump_discernible(HSI_Event # Use a biopsy to diagnose whether the person has breast Cancer: # todo: request consumables needed for this - self.used_equipment = {'Slice Master sample processing Unit', 'Paraffin Dispense', 'Whatever used with biopsy', - 'Mammograph maybe?'} dx_result = hs.dx_manager.run_dx_test( dx_tests_to_run='biopsy_for_breast_cancer_given_breast_lump_discernible', @@ -761,8 +761,9 @@ class HSI_BreastCancer_StartTreatment(HSI_Event, IndividualScopeEventMixin): df.at[person_id, "brc_date_treatment"] = self.sim.date df.at[person_id, "brc_stage_at_which_treatment_given"] = df.at[person_id, "brc_status"] - # Record used equipment - self.used_equipment = {'Anything used for mastectomy as I guess this is about'} + # Update equipment used with treatment + # NB. read only with HSI run and healthsystem.summary logger set at the level INFO or higher + self.EQUIPMENT.update({'Anything used for mastectomy as I guess this is about'}) # Schedule a post-treatment check for 12 months: hs.schedule_hsi_event( diff --git src/tlo/methods/contraception.py src/tlo/methods/contraception.py index 6ffb0ebc6..15851e1b7 100644 --- src/tlo/methods/contraception.py +++ src/tlo/methods/contraception.py @@ -1281,11 +1281,12 @@ class HSI_Contraception_FamilyPlanningAppt(HSI_Event, IndividualScopeEventMixin) _new_contraceptive = self.new_contraceptive - # Record used equipment when needed + # Update equipment when needed + # NB. read only with HSI run and healthsystem.summary logger set at the level of logger.INFO or higher if _new_contraceptive == 'female_sterilization': - self.used_equipment = {'Smt used to sterilize a woman'} + self.EQUIPMENT.update({'Smt used to sterilize a woman'}) elif _new_contraceptive == 'IUD': - self.used_equipment = {'Equipment used when performing IUD'} + self.EQUIPMENT.update({'Equipment used when performing IUD'}) else: _new_contraceptive = "not_using" diff --git src/tlo/methods/healthsystem.py src/tlo/methods/healthsystem.py index c19b0f433..3eb6b9940 100644 --- src/tlo/methods/healthsystem.py +++ src/tlo/methods/healthsystem.py @@ -182,7 +182,7 @@ class HSI_Event: self._received_info_about_bed_days = None self.expected_time_requests = {} self.facility_info = None - self.used_equipment = set() + self.EQUIPMENT = set() @property def bed_days_allocated_to_this_event(self): @@ -1741,7 +1741,7 @@ class HealthSystem(Module): squeeze_factor=_squeeze_factor, did_run=did_run, priority=priority, - equipment=hsi_event.used_equipment, + equipment=hsi_event.EQUIPMENT, ) def write_to_hsi_log( --- src/tlo/methods/breast_cancer.py | 9 +++++---- src/tlo/methods/contraception.py | 7 ++++--- src/tlo/methods/healthsystem.py | 4 ++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/tlo/methods/breast_cancer.py b/src/tlo/methods/breast_cancer.py index 5d8fabfcb2..26155729ab 100644 --- a/src/tlo/methods/breast_cancer.py +++ b/src/tlo/methods/breast_cancer.py @@ -646,6 +646,8 @@ def __init__(self, module, person_id): self.TREATMENT_ID = "BreastCancer_Investigation" self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint({"Over5OPD": 1, "Mammography": 1}) self.ACCEPTED_FACILITY_LEVEL = '3' # Biopsy only available at level 3 and above. + self.EQUIPMENT = {'Slice Master sample processing Unit', 'Paraffin Dispense', 'Whatever used with biopsy'} + # biopsy always performed with this HSI def apply(self, person_id, squeeze_factor): df = self.sim.population.props @@ -666,8 +668,6 @@ def apply(self, person_id, squeeze_factor): # Use a biopsy to diagnose whether the person has breast Cancer: # todo: request consumables needed for this - self.used_equipment = {'Slice Master sample processing Unit', 'Paraffin Dispense', 'Whatever used with biopsy', - 'Mammograph maybe?'} dx_result = hs.dx_manager.run_dx_test( dx_tests_to_run='biopsy_for_breast_cancer_given_breast_lump_discernible', @@ -761,8 +761,9 @@ def apply(self, person_id, squeeze_factor): df.at[person_id, "brc_date_treatment"] = self.sim.date df.at[person_id, "brc_stage_at_which_treatment_given"] = df.at[person_id, "brc_status"] - # Record used equipment - self.used_equipment = {'Anything used for mastectomy as I guess this is about'} + # Update equipment used with treatment + # NB. read only with HSI run and healthsystem.summary logger set at the level INFO or higher + self.EQUIPMENT.update({'Anything used for mastectomy as I guess this is about'}) # Schedule a post-treatment check for 12 months: hs.schedule_hsi_event( diff --git a/src/tlo/methods/contraception.py b/src/tlo/methods/contraception.py index 4def5a649d..1d4d62a929 100644 --- a/src/tlo/methods/contraception.py +++ b/src/tlo/methods/contraception.py @@ -1196,11 +1196,12 @@ def apply(self, person_id, squeeze_factor): _new_contraceptive = self.new_contraceptive - # Record used equipment when needed + # Update equipment when needed + # NB. read only with HSI run and healthsystem.summary logger set at the level of logger.INFO or higher if _new_contraceptive == 'female_sterilization': - self.used_equipment = {'Smt used to sterilize a woman'} + self.EQUIPMENT.update({'Smt used to sterilize a woman'}) elif _new_contraceptive == 'IUD': - self.used_equipment = {'Equipment used when performing IUD'} + self.EQUIPMENT.update({'Equipment used when performing IUD'}) else: _new_contraceptive = "not_using" diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 1a07595d9e..39654911c6 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -182,7 +182,7 @@ def __init__(self, module, *args, **kwargs): self._received_info_about_bed_days = None self.expected_time_requests = {} self.facility_info = None - self.used_equipment = set() + self.EQUIPMENT = set() @property def bed_days_allocated_to_this_event(self): @@ -1811,7 +1811,7 @@ def record_hsi_event(self, hsi_event, actual_appt_footprint=None, squeeze_factor squeeze_factor=_squeeze_factor, did_run=did_run, priority=priority, - equipment=hsi_event.used_equipment, + equipment=hsi_event.EQUIPMENT, ) def write_to_hsi_log( From 539626928ec782ac5ebfe251ac18b0e09dbbb5c6 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Mon, 18 Sep 2023 17:33:16 +0100 Subject: [PATCH 012/118] brc: comment updated diff --git src/tlo/methods/breast_cancer.py src/tlo/methods/breast_cancer.py index 26155729a..aef476c87 100644 --- src/tlo/methods/breast_cancer.py +++ src/tlo/methods/breast_cancer.py @@ -647,7 +647,7 @@ class HSI_BreastCancer_Investigation_Following_breast_lump_discernible(HSI_Event self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint({"Over5OPD": 1, "Mammography": 1}) self.ACCEPTED_FACILITY_LEVEL = '3' # Biopsy only available at level 3 and above. self.EQUIPMENT = {'Slice Master sample processing Unit', 'Paraffin Dispense', 'Whatever used with biopsy'} - # biopsy always performed with this HSI + # biopsy always performed with this HSI, hence always used the same set of equipment def apply(self, person_id, squeeze_factor): df = self.sim.population.props --- src/tlo/methods/breast_cancer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tlo/methods/breast_cancer.py b/src/tlo/methods/breast_cancer.py index 26155729ab..aef476c870 100644 --- a/src/tlo/methods/breast_cancer.py +++ b/src/tlo/methods/breast_cancer.py @@ -647,7 +647,7 @@ def __init__(self, module, person_id): self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint({"Over5OPD": 1, "Mammography": 1}) self.ACCEPTED_FACILITY_LEVEL = '3' # Biopsy only available at level 3 and above. self.EQUIPMENT = {'Slice Master sample processing Unit', 'Paraffin Dispense', 'Whatever used with biopsy'} - # biopsy always performed with this HSI + # biopsy always performed with this HSI, hence always used the same set of equipment def apply(self, person_id, squeeze_factor): df = self.sim.population.props From 2791c18cec1bb562e57dba8ed6edef01e03009cb Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Wed, 20 Sep 2023 17:44:03 +0100 Subject: [PATCH 013/118] brc & co: rm the dummy examples of equipment from modules --- src/tlo/methods/breast_cancer.py | 6 ------ src/tlo/methods/contraception.py | 10 +--------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/src/tlo/methods/breast_cancer.py b/src/tlo/methods/breast_cancer.py index aef476c870..1ce9ad2bf6 100644 --- a/src/tlo/methods/breast_cancer.py +++ b/src/tlo/methods/breast_cancer.py @@ -646,8 +646,6 @@ def __init__(self, module, person_id): self.TREATMENT_ID = "BreastCancer_Investigation" self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint({"Over5OPD": 1, "Mammography": 1}) self.ACCEPTED_FACILITY_LEVEL = '3' # Biopsy only available at level 3 and above. - self.EQUIPMENT = {'Slice Master sample processing Unit', 'Paraffin Dispense', 'Whatever used with biopsy'} - # biopsy always performed with this HSI, hence always used the same set of equipment def apply(self, person_id, squeeze_factor): df = self.sim.population.props @@ -761,10 +759,6 @@ def apply(self, person_id, squeeze_factor): df.at[person_id, "brc_date_treatment"] = self.sim.date df.at[person_id, "brc_stage_at_which_treatment_given"] = df.at[person_id, "brc_status"] - # Update equipment used with treatment - # NB. read only with HSI run and healthsystem.summary logger set at the level INFO or higher - self.EQUIPMENT.update({'Anything used for mastectomy as I guess this is about'}) - # Schedule a post-treatment check for 12 months: hs.schedule_hsi_event( hsi_event=HSI_BreastCancer_PostTreatmentCheck( diff --git a/src/tlo/methods/contraception.py b/src/tlo/methods/contraception.py index 1d4d62a929..0a4884d0f3 100644 --- a/src/tlo/methods/contraception.py +++ b/src/tlo/methods/contraception.py @@ -1170,8 +1170,7 @@ def apply(self, person_id, squeeze_factor): items_all = {**items_essential, **items_optional} # Determine whether the contraception is administrated (ie all essential items are available), - # if so do log the availability of all items and record used equipment if any, if not set the contraception to - # "not_using": + # if so do log the availability of all items, if not set the contraception to "not_using": co_administrated = all(v for k, v in cons_available.items() if k in items_essential) if co_administrated: @@ -1196,13 +1195,6 @@ def apply(self, person_id, squeeze_factor): _new_contraceptive = self.new_contraceptive - # Update equipment when needed - # NB. read only with HSI run and healthsystem.summary logger set at the level of logger.INFO or higher - if _new_contraceptive == 'female_sterilization': - self.EQUIPMENT.update({'Smt used to sterilize a woman'}) - elif _new_contraceptive == 'IUD': - self.EQUIPMENT.update({'Equipment used when performing IUD'}) - else: _new_contraceptive = "not_using" From 07e1ae92321797f3451e4cb089619b0f816ffb3a Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Sun, 24 Sep 2023 16:15:42 +0100 Subject: [PATCH 014/118] RF_Equipment: equipment catalogue - first draft (from Sakshi) --- .../infrastructure_and_equipment/ResourceFile_Equipment.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv diff --git a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv new file mode 100644 index 0000000000..b119a1d503 --- /dev/null +++ b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17a3a35a9c7908c9a291b8906ae1f8f7be6ff89459a183900de142ee29368c53 +size 36731 From 00270c19507deb4a8cdcf686e7e6139ae11c0dd6 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Sun, 24 Sep 2023 23:54:02 +0100 Subject: [PATCH 015/118] RF_Equipment: equipment catalogue - merge duplicates (round 1) --- .../infrastructure_and_equipment/ResourceFile_Equipment.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv index b119a1d503..4e936f4cb9 100644 --- a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv +++ b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:17a3a35a9c7908c9a291b8906ae1f8f7be6ff89459a183900de142ee29368c53 -size 36731 +oid sha256:faf0337415b4ef87fc47dace921c9f9fb4450d88ff1718def0a96e80fa073cb6 +size 34520 From d486c4da2c6f56053db3e492e8c393ae51b891e7 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Wed, 27 Sep 2023 09:56:16 +0100 Subject: [PATCH 016/118] RF_Equipment: equipment catalogue - merge duplicates (round 2) --- .../infrastructure_and_equipment/ResourceFile_Equipment.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv index 4e936f4cb9..22f62d5a0f 100644 --- a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv +++ b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:faf0337415b4ef87fc47dace921c9f9fb4450d88ff1718def0a96e80fa073cb6 -size 34520 +oid sha256:b4f78c47032a3e658fc1d07fd631c2e8aea85b145a7978fcd5d965bf9e0c5fce +size 32543 From 860d9524d0f904271a4efee31a256a9d24d88e77 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Wed, 27 Sep 2023 21:50:20 +0100 Subject: [PATCH 017/118] hs: debugging Equipment log - rm sorted --- src/tlo/methods/healthsystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 39654911c6..ab4bdd0ec1 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -2834,7 +2834,7 @@ def write_to_log_and_reset_counters(self): key="Equipment", description="Sets of used equipment for each facility level in this calendar year.", data={ - "Equipment_By_Level": sorted(self._equip_by_level), + "Equipment_By_Level": self._equip_by_level, }, ) From c573fc8550b0549e885658ae31f100c781501554 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Thu, 28 Sep 2023 00:49:34 +0100 Subject: [PATCH 018/118] hs: fix sorting of _equip_by_level --- src/tlo/methods/healthsystem.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index ab4bdd0ec1..bdddc1d76a 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -2830,6 +2830,9 @@ def write_to_log_and_reset_counters(self): }, ) + # Sort equipment within levels, and log them + for key in self._equip_by_level: + self._equip_by_level[key] = sorted(self._equip_by_level[key]) logger_summary.info( key="Equipment", description="Sets of used equipment for each facility level in this calendar year.", From 373018c63ddf79aa68d1992755c9c5355c0e683a Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Wed, 27 Sep 2023 19:08:35 +0100 Subject: [PATCH 019/118] RF_Equipment: equipment catalogue - merge duplicates (round 3) --- .../infrastructure_and_equipment/ResourceFile_Equipment.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv index 22f62d5a0f..6178c242ba 100644 --- a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv +++ b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b4f78c47032a3e658fc1d07fd631c2e8aea85b145a7978fcd5d965bf9e0c5fce -size 32543 +oid sha256:9f02f434986e8070dacc50405e2b88826be18eb6de61b4e6475e7a4cad948334 +size 32455 From 307efab9a37fe18442578438886c53cc69627f04 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Thu, 28 Sep 2023 00:32:23 +0100 Subject: [PATCH 020/118] RF_Equipment: equipment catalogue - merge duplicates (round 4) --- .../infrastructure_and_equipment/ResourceFile_Equipment.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv index 6178c242ba..aca3040b03 100644 --- a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv +++ b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9f02f434986e8070dacc50405e2b88826be18eb6de61b4e6475e7a4cad948334 -size 32455 +oid sha256:29782fdacd7b13efef9b0b9de23fb7413b329bb9880b87c3b0522e6732130bc1 +size 31872 From 4f2ce6c7775f614d272a787664a37fa364bff73d Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Fri, 29 Sep 2023 10:45:25 +0100 Subject: [PATCH 021/118] RF_Equipment: equipment catalogue - merge duplicates (round 5) --- .../infrastructure_and_equipment/ResourceFile_Equipment.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv index aca3040b03..0055cd8d8e 100644 --- a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv +++ b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:29782fdacd7b13efef9b0b9de23fb7413b329bb9880b87c3b0522e6732130bc1 -size 31872 +oid sha256:f3854096278164e1e3c63c2aa2e7afecbae1c6ee14dabcff829af78c271cd82d +size 31712 From 10b9dde4738dc38df76a666bc3924c6552ee9257 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Wed, 15 Nov 2023 18:09:49 +0000 Subject: [PATCH 022/118] RF_Equipment: equipment catalogue - merge duplicates (round 6) + col names renamed --- .../infrastructure_and_equipment/ResourceFile_Equipment.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv index 0055cd8d8e..c7f2e7e240 100644 --- a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv +++ b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f3854096278164e1e3c63c2aa2e7afecbae1c6ee14dabcff829af78c271cd82d -size 31712 +oid sha256:c1f647adbbd18cbc1460135450d2f0b402fac00f80f050fcb3cd8c138b822f71 +size 31663 From 1f2ff79d5d29c308e21338fe2c245236030dd1d0 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Wed, 15 Nov 2023 18:23:41 +0000 Subject: [PATCH 023/118] RF_Equip: equip item codes added --- .../infrastructure_and_equipment/ResourceFile_Equipment.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv index c7f2e7e240..f0a2b53c9f 100644 --- a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv +++ b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c1f647adbbd18cbc1460135450d2f0b402fac00f80f050fcb3cd8c138b822f71 -size 31663 +oid sha256:cd2c3558a4d30dfb6d913054513e1d6df1b91f4955b60371e762eff05ac99afd +size 32759 From adde374490047539cd19b4768dcc7bda5fc829de Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Wed, 15 Nov 2023 18:21:55 +0000 Subject: [PATCH 024/118] codes_to_items_list: new script created --- src/tlo/analysis/codes_to_items_list.py | 70 +++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 src/tlo/analysis/codes_to_items_list.py diff --git a/src/tlo/analysis/codes_to_items_list.py b/src/tlo/analysis/codes_to_items_list.py new file mode 100644 index 0000000000..365f2e64b8 --- /dev/null +++ b/src/tlo/analysis/codes_to_items_list.py @@ -0,0 +1,70 @@ +""" +(1) Can be used for a list of items without item codes yet saved in a csv file named 'csv_file_to_update_name'. + +This script will assign unique code to each unique item name which has no code assigned yet. The codes are +assigned in order from the sequence 0, 1, 2, .... + +Duplicated items are allowed, the same code will be assigned to the same items. + +(2) Can be used when new items are added later without item codes but some items with codes are already in the list. + +This script will keep the existing codes for items with already assigned code and for items without existing +code will assign new code (continue in sequence, i.e. if the highest code is 5, it assigns new codes from the continuing +sequence 6, 7, 8, ...). + +------ +NB. Make sure the 'csv_file_to_update_name' is the file you want to update. The output will be named +'csv_file_to_update_name' + '_new.csv' to avoid unintentionally losing the previous version. +------ +""" + +import pandas as pd +from pathlib import Path + + +# ## CHANGE THIS IF YOU WANT TO USE DIFFERENT FILE AS INPUT +csv_file_to_update_name = 'ResourceFile_Equipment_withoutEquipmentCodes' + +# Get the path of the current script file +script_path = Path(__file__) +print(script_path) + +# Specify the file path to RF csv file +file_path = script_path.parent.parent.parent.parent / 'resources/healthsystem/infrastructure_and_equipment' + +# Load the CSV RF into a DataFrame +df = pd.read_csv(Path(file_path) / str(csv_file_to_update_name + '.csv')) + +# Find unique values in Equipment that have no code and are not None or empty +unique_values =\ + df.loc[df['Equip_Code'].isna() & df['Equip_Item'].notna() & (df['Equip_Item'] != ''), 'Equip_Item'].unique() + +# Create a mapping of unique values to codes +value_to_code = {} +# Initialize the starting code value +if not df['Equip_Code'].isna().all(): + next_code = int(df['Equip_Code'].max()) + 1 +else: + next_code = 0 + +# Iterate through unique values +for value in unique_values: + # Check if there is at least one existing code for this value + matching_rows = df.loc[df['Equip_Item'] == value, 'Equip_Code'].dropna() + if not matching_rows.empty: + # Use the existing code for this value + existing_code = int(matching_rows.iloc[0]) + # TODO: verify all the codes are the same + else: + # If no existing codes, start with the next available code + existing_code = next_code + next_code += 1 + value_to_code[value] = existing_code + # Update the 'Equip_Code' column for matching rows + df.loc[df['Equip_Item'] == value, 'Equip_Code'] = existing_code + +# Convert 'Equip_Code' column to integers +df['Equip_Code'] = df['Equip_Code'].astype('Int64') # Convert to nullable integer type + +# Save CSV with equipment codes +df.to_csv(Path(file_path) / str(csv_file_to_update_name + '_new.csv'), index=False) From e6013d9540dcb16b9934abcc490c6f827d89a971 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Wed, 15 Nov 2023 18:45:42 +0000 Subject: [PATCH 025/118] codes_to_items_list: script generalised + PEP 8 --- src/tlo/analysis/codes_to_items_list.py | 31 ++++++++++++++----------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/tlo/analysis/codes_to_items_list.py b/src/tlo/analysis/codes_to_items_list.py index 365f2e64b8..da169a06e5 100644 --- a/src/tlo/analysis/codes_to_items_list.py +++ b/src/tlo/analysis/codes_to_items_list.py @@ -18,39 +18,44 @@ ------ """ -import pandas as pd from pathlib import Path - -# ## CHANGE THIS IF YOU WANT TO USE DIFFERENT FILE AS INPUT -csv_file_to_update_name = 'ResourceFile_Equipment_withoutEquipmentCodes' +import pandas as pd # Get the path of the current script file script_path = Path(__file__) print(script_path) -# Specify the file path to RF csv file +# ############################# +# ## CHANGE THIS FOR YOUR FILE +# Specify name of the csv file +csv_file_to_update_name = 'ResourceFile_Equipment_withoutEquipmentCodes' +# Specify the file path to csv file file_path = script_path.parent.parent.parent.parent / 'resources/healthsystem/infrastructure_and_equipment' +# Specify the names of columns containing the item names and item codes +item_col_name = 'Equip_Item' +code_col_name = 'Equip_Code' +# ############################# # Load the CSV RF into a DataFrame df = pd.read_csv(Path(file_path) / str(csv_file_to_update_name + '.csv')) # Find unique values in Equipment that have no code and are not None or empty unique_values =\ - df.loc[df['Equip_Code'].isna() & df['Equip_Item'].notna() & (df['Equip_Item'] != ''), 'Equip_Item'].unique() + df.loc[df[code_col_name].isna() & df[item_col_name].notna() & (df[item_col_name] != ''), item_col_name].unique() # Create a mapping of unique values to codes value_to_code = {} # Initialize the starting code value -if not df['Equip_Code'].isna().all(): - next_code = int(df['Equip_Code'].max()) + 1 +if not df[code_col_name].isna().all(): + next_code = int(df[code_col_name].max()) + 1 else: next_code = 0 # Iterate through unique values for value in unique_values: # Check if there is at least one existing code for this value - matching_rows = df.loc[df['Equip_Item'] == value, 'Equip_Code'].dropna() + matching_rows = df.loc[df[item_col_name] == value, code_col_name].dropna() if not matching_rows.empty: # Use the existing code for this value existing_code = int(matching_rows.iloc[0]) @@ -60,11 +65,11 @@ existing_code = next_code next_code += 1 value_to_code[value] = existing_code - # Update the 'Equip_Code' column for matching rows - df.loc[df['Equip_Item'] == value, 'Equip_Code'] = existing_code + # Update the code_col_name column for matching rows + df.loc[df[item_col_name] == value, code_col_name] = existing_code -# Convert 'Equip_Code' column to integers -df['Equip_Code'] = df['Equip_Code'].astype('Int64') # Convert to nullable integer type +# Convert code_col_name column to integers +df[code_col_name] = df[code_col_name].astype('Int64') # Convert to nullable integer type # Save CSV with equipment codes df.to_csv(Path(file_path) / str(csv_file_to_update_name + '_new.csv'), index=False) From a214a888158b263d24d9fcd1ca7752c99e10a67d Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Thu, 16 Nov 2023 17:55:35 +0000 Subject: [PATCH 026/118] hs: equipment added to HSIEventDetails --- src/tlo/methods/healthsystem.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index bdddc1d76a..fe254fbadc 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -126,6 +126,7 @@ class HSIEventDetails(NamedTuple): facility_level: Optional[str] appt_footprint: Tuple[Tuple[str, int]] beddays_footprint: Tuple[Tuple[str, int]] + equipment: set class HSIEventQueueItem(NamedTuple): @@ -399,7 +400,8 @@ def as_namedtuple( appt_footprint=tuple(sorted(appt_footprint.items())), beddays_footprint=tuple( sorted((k, v) for k, v in self.BEDDAYS_FOOTPRINT.items() if v > 0) - ) + ), + equipment=(tuple(self.EQUIPMENT)) ) @@ -1837,10 +1839,13 @@ def write_to_hsi_log( 'did_run': did_run, 'Facility_Level': event_details.facility_level if event_details.facility_level is not None else -99, 'Facility_ID': facility_id if facility_id is not None else -99, + 'equipment': equipment, }, description="record of each HSI event" ) if did_run: + print("\nevent_details") + print(event_details) if self._hsi_event_count_log_period is not None: event_details_key = self._hsi_event_details.setdefault( event_details, len(self._hsi_event_details) @@ -1885,7 +1890,7 @@ def write_to_never_ran_hsi_log( event_details: HSIEventDetails, person_id: int, facility_id: Optional[int], - priority: int, + priority: int ): """Write the log `HSI_Event` and add to the summary counter.""" logger.debug( @@ -1910,7 +1915,7 @@ def write_to_never_ran_hsi_log( treatment_id=event_details.treatment_id, hsi_event_name=event_details.event_name, appt_footprint=event_details.appt_footprint, - level=event_details.facility_level, + level=event_details.facility_level ) def log_current_capabilities_and_usage(self): @@ -2170,7 +2175,7 @@ def run_individual_level_events_in_mode_0_or_1(self, actual_appt_footprint=actual_appt_footprint, squeeze_factor=squeeze_factor, did_run=True, - priority=_priority + priority=_priority, ) # if not ok_to_run @@ -2668,7 +2673,8 @@ def apply(self, population): treatment_id='Inpatient_Care', facility_level=self.module._facility_by_facility_id[_fac_id].level, appt_footprint=tuple(sorted(_inpatient_appts.items())), - beddays_footprint=() + beddays_footprint=(), + equipment=() # TODO: what should be in here? ), person_id=-1, facility_id=_fac_id, From 32145b6f373dba22a70ba60c0b1cc8eca4d23455 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Fri, 17 Nov 2023 18:51:25 +0000 Subject: [PATCH 027/118] equip_catalogue: make catalogues from new logging (equip included in hsi_event_details, counts logged in hsi_event_counts) --- .../equipment/equipment_catalogue.py | 117 +++++++++++------- 1 file changed, 69 insertions(+), 48 deletions(-) diff --git a/src/scripts/healthsystem/equipment/equipment_catalogue.py b/src/scripts/healthsystem/equipment/equipment_catalogue.py index ecbc206aba..9672c92474 100644 --- a/src/scripts/healthsystem/equipment/equipment_catalogue.py +++ b/src/scripts/healthsystem/equipment/equipment_catalogue.py @@ -3,36 +3,35 @@ import pandas as pd -from tlo.analysis.utils import extract_results +from tlo.analysis.utils import extract_results, load_pickled_dataframes -def get_annual_equipment_declarations_by_levels(results_folder: Path) -> pd.DataFrame: - """Return pd.DataFrame gives the simulated annual equipment declaration by facility levels for each simulated - year. - NB. healthsystem.summary logger required to have been set at the level INFO or higher.""" +def get_annual_hsi_event_counts(results_folder: Path) -> pd.DataFrame: + """Return pd.DataFrame gives the simulated annual counts of all the hsi event details logged (details as keys) + for each simulated year. + NB. 'healthsystem.summary' logger required to have been set at the level INFO or higher.""" - def get_equipment_declaration_by_levels(_df): - """Get the equipment declaration by facility levels for the year.""" + def get_hsi_event_counts(_df): + """Get the counts of all the hsi event details logged.""" def unpack_dict_in_series(_raw: pd.Series): # Create an empty DataFrame to store the data df = pd.DataFrame() # Iterate through the dictionary items - for col_name, mydict in _raw.items(): + for _, mydict in _raw.items(): for date, inner_dict in mydict.items(): # Convert the inner_dict to a list of dictionaries with 'date' - data = [{'date': date, 'fac_level': inner_dict_key, 'value': inner_dict_set} for + data = [{'date': date, 'event_details_key': inner_dict_key, 'count': inner_dict_set} for inner_dict_key, inner_dict_set in inner_dict.items()] # Create a DataFrame from the list with date & fac_level as indexes temp_df = pd.DataFrame(data) - temp_df.set_index(['date', 'fac_level'], inplace=True) + temp_df.set_index(['date', 'event_details_key'], inplace=True) temp_df.columns = [None] # Concatenate the temporary DataFrame to the result DataFrame df = pd.concat([df, temp_df]) - # print(f"\ndf\n {df}") df.columns = [None] return df @@ -46,47 +45,69 @@ def unpack_dict_in_series(_raw: pd.Series): return extract_results( results_folder, module='tlo.methods.healthsystem.summary', - key='Equipment', - custom_generate_series=get_equipment_declaration_by_levels + key='hsi_event_counts', + custom_generate_series=get_hsi_event_counts ) -def create_equipment_catalogue(results_folder: Path, output_folder: Path): - # Declare path for output file from this script - output_file_name = 'equipment_catalogue_by_level.csv' - output_detailed_file_name = 'equipment_catalogue_by_date_level_sim.csv' +def get_hsi_event_keys(results_folder: Path): + return load_pickled_dataframes(results_folder, 0, 0, "tlo.methods.healthsystem.summary")[ + "tlo.methods.healthsystem.summary" + ]["hsi_event_details"]["hsi_event_key_to_event_details"][0] - sim_equipment = get_annual_equipment_declarations_by_levels(results_folder) + +def create_equipment_catalogues(results_folder: Path, output_folder: Path): + + # Declare output file names + output_detailed_file_name = 'equipment_annual_counts__all_event_details.csv' + output_file_name = 'equipment_annual_counts__by_Date_EventName_FacLevel.csv' + + # %% Catalogue equipment by all HSI event details + sim_equipment = get_annual_hsi_event_counts(results_folder) sim_equipment_df = pd.DataFrame(sim_equipment) - sim_equipment_df.index.names = ['date', 'fac_level'] - - # Save the detailed CSV - sim_equipment_df.to_csv(output_folder / output_detailed_file_name) - print('equipment_catalogue_by_date_level_sim.csv saved.') - - # Prepare a catalogue only by facility levels - # Define a custom aggregation function to combine sets in columns for each row - def combine_sets(row): - combined_set = set() - for col in row: - combined_set.update(col) - return combined_set - - # Apply the custom aggregation function to each row - sim_equipment_by_level_df = sim_equipment_df.copy() - sim_equipment_by_level_df['equipment'] = sim_equipment_by_level_df.apply(combine_sets, axis=1) - # Group by 'fac_level' and join rows with the same 'fac_level' into one set - sim_equipment_by_level_df.reset_index(inplace=True) - sim_equipment_by_level_df = sim_equipment_by_level_df.groupby('fac_level')['equipment'].apply( - lambda x: list(set.union(*x)) - ).reset_index() - - # Explode the 'equipment' column to separate elements into rows - sim_equipment_by_level_df = sim_equipment_by_level_df.explode('equipment', ignore_index=True).set_index('fac_level') - - # Save the CSV equipment catalogue - sim_equipment_by_level_df.to_csv(output_folder / output_file_name) - print('equipment_catalogue_by_level.csv saved.') + sim_equipment_df.fillna(0, inplace=True) + + hsi_event_keys = get_hsi_event_keys(results_folder) + + decoded_keys = sim_equipment_df.index.get_level_values(1).astype(str).map(hsi_event_keys) + sim_equipment_df = pd.concat([sim_equipment_df, pd.DataFrame(decoded_keys.tolist(), index=sim_equipment_df.index)], axis=1) + # Make values in 'appt_footprint', and 'beddays_footprint' columns to be string + sim_equipment_df['appt_footprint'] = sim_equipment_df['appt_footprint'].apply(lambda x: ', '.join(map(str, x))) + sim_equipment_df['beddays_footprint'] = sim_equipment_df['beddays_footprint'].apply(lambda x: ', '.join(map(str, x))) + # Explode the 'equipment' column + exploded_df = sim_equipment_df.explode('equipment') + # Remove the 'event_details_key' and replace the index with hsi event details as indexes + exploded_df = exploded_df.droplevel(level=1) + exploded_df = exploded_df.set_index( + ['event_name', 'module_name', 'treatment_id', 'facility_level', 'appt_footprint', 'beddays_footprint', + 'equipment'], append=True + ) + # Sum values with the same multi-index + exploded_df = exploded_df.groupby(exploded_df.index).sum() + # Convert the index back to a MultiIndex + exploded_df.index = pd.MultiIndex.from_tuples(exploded_df.index) + exploded_df.index.names = \ + ['date', 'event_name', 'module_name', 'treatment_id', 'facility_level', 'appt_footprint', 'beddays_footprint', + 'equipment'] + + # Save the detailed equipment catalogue by levels + exploded_df.to_csv(output_folder / output_detailed_file_name) + print(f'{output_detailed_file_name} saved.') + # --- + + # %% Catalogue equipment by Facility Levels and HSI Event Names + + equipment_counts_by_date_hsi_name_level_df = exploded_df.copy() + + # Sum counts for each equipment with the same date, hsi event name, and level (remaining indexes removed) + # equipment_counts_by_date_hsi_name_level_df + equipment_counts_by_date_hsi_name_level_df = \ + equipment_counts_by_date_hsi_name_level_df.groupby(['date', 'event_name', 'facility_level', 'equipment']).sum() + + # Save the CSV equipment counts + equipment_counts_by_date_hsi_name_level_df.to_csv(output_folder / output_file_name) + print(f'{output_file_name} saved.') + # --- return 0 @@ -96,7 +117,7 @@ def combine_sets(row): parser.add_argument("results_folder", type=Path) args = parser.parse_args() - create_equipment_catalogue( + create_equipment_catalogues( results_folder=args.results_folder, output_folder=args.results_folder, ) From d8e299f6445e394e54da6b1d744d1f8b6371c643 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Sun, 19 Nov 2023 16:44:27 +0000 Subject: [PATCH 028/118] hs: sort equipment for logging --- src/tlo/methods/healthsystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index fe254fbadc..230a7c1b31 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -401,7 +401,7 @@ def as_namedtuple( beddays_footprint=tuple( sorted((k, v) for k, v in self.BEDDAYS_FOOTPRINT.items() if v > 0) ), - equipment=(tuple(self.EQUIPMENT)) + equipment=(tuple(sorted(self.EQUIPMENT))) ) From 598368d84a8942073b49f56e655dba4b892a0488 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Sun, 19 Nov 2023 23:58:02 +0000 Subject: [PATCH 029/118] test_hs: assert equipment logging within detailed_hsi_event --- tests/test_healthsystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_healthsystem.py b/tests/test_healthsystem.py index e07a2d5889..22b9d7ff8c 100644 --- a/tests/test_healthsystem.py +++ b/tests/test_healthsystem.py @@ -862,7 +862,7 @@ def apply(self, person_id, squeeze_factor): detailed_consumables = log["tlo.methods.healthsystem"]['Consumables'] assert {'date', 'TREATMENT_ID', 'did_run', 'Squeeze_Factor', 'priority', 'Number_By_Appt_Type_Code', 'Person_ID', - 'Facility_Level', 'Facility_ID', 'Event_Name', + 'Facility_Level', 'Facility_ID', 'Event_Name', 'equipment' } == set(detailed_hsi_event.columns) assert {'date', 'Frac_Time_Used_Overall', 'Frac_Time_Used_By_Facility_ID', 'Frac_Time_Used_By_OfficerType', } == set(detailed_capacity.columns) From e2fdafb8a5c88509fe85396e92720ec1dc4f6985 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Thu, 23 Nov 2023 12:00:46 +0000 Subject: [PATCH 030/118] [no ci] hs: typo; Equipment logging removed diff --git src/tlo/methods/healthsystem.py src/tlo/methods/healthsystem.py index d14f8f10a..5ecb43368 100644 --- src/tlo/methods/healthsystem.py +++ src/tlo/methods/healthsystem.py @@ -2611,7 +2611,7 @@ class HealthSystemScheduler(RegularEvent, PopulationScopeEventMixin): squeeze_factor=0.0, priority=-1, did_run=True, - equipment=set() # TODO: explore more, should it be non-emtpy in some cases? + equipment=set(), # TODO: explore more, should it be non-empty in some cases? ) # Restart the total footprint of all calls today, beginning with those due to existing in-patients. @@ -2666,7 +2666,6 @@ class HealthSystemSummaryCounter: self._appts = defaultdict(int) # Running record of the Appointments of `HSI_Event`s that have run self._appts_by_level = {_level: defaultdict(int) for _level in ('0', '1a', '1b', '2', '3', '4')} # <--Same as `self._appts` but also split by facility_level - self._equip_by_level = {_level: set() for _level in ('0', '1a', '1b', '2', '3', '4')} # Log HSI_Events that never ran to monitor shortcoming of Health System self._never_ran_treatment_ids = defaultdict(int) # As above, but for `HSI_Event`s that never ran @@ -2683,8 +2682,7 @@ class HealthSystemSummaryCounter: hsi_event_name: str, squeeze_factor: float, appt_footprint: Counter, - level: str, - equipment: set + level: str ) -> None: """Add information about an `HSI_Event` to the running summaries.""" @@ -2701,9 +2699,6 @@ class HealthSystemSummaryCounter: self._appts[appt_type] += number self._appts_by_level[level][appt_type] += number - # Update used equipment by level - self._equip_by_level[level].update(equipment) - def record_never_ran_hsi_event(self, treatment_id: str, hsi_event_name: str, @@ -2766,17 +2761,6 @@ class HealthSystemSummaryCounter: }, ) - # Sort equipment within levels, and log them - for key in self._equip_by_level: - self._equip_by_level[key] = sorted(self._equip_by_level[key]) - logger_summary.info( - key="Equipment", - description="Sets of used equipment for each facility level in this calendar year.", - data={ - "Equipment_By_Level": self._equip_by_level, - }, - ) - self._reset_internal_stores() --- src/tlo/methods/healthsystem.py | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 230a7c1b31..af0610fe6c 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -2681,7 +2681,7 @@ def apply(self, population): squeeze_factor=0.0, priority=-1, did_run=True, - equipment=set() # TODO: explore more, should it be non-emtpy in some cases? + equipment=set(), # TODO: explore more, should it be non-empty in some cases? ) # Restart the total footprint of all calls today, beginning with those due to existing in-patients. @@ -2736,7 +2736,6 @@ def _reset_internal_stores(self) -> None: self._appts = defaultdict(int) # Running record of the Appointments of `HSI_Event`s that have run self._appts_by_level = {_level: defaultdict(int) for _level in ('0', '1a', '1b', '2', '3', '4')} # <--Same as `self._appts` but also split by facility_level - self._equip_by_level = {_level: set() for _level in ('0', '1a', '1b', '2', '3', '4')} # Log HSI_Events that never ran to monitor shortcoming of Health System self._never_ran_treatment_ids = defaultdict(int) # As above, but for `HSI_Event`s that never ran @@ -2753,8 +2752,7 @@ def record_hsi_event(self, hsi_event_name: str, squeeze_factor: float, appt_footprint: Counter, - level: str, - equipment: set + level: str ) -> None: """Add information about an `HSI_Event` to the running summaries.""" @@ -2771,9 +2769,6 @@ def record_hsi_event(self, self._appts[appt_type] += number self._appts_by_level[level][appt_type] += number - # Update used equipment by level - self._equip_by_level[level].update(equipment) - def record_never_ran_hsi_event(self, treatment_id: str, hsi_event_name: str, @@ -2836,17 +2831,6 @@ def write_to_log_and_reset_counters(self): }, ) - # Sort equipment within levels, and log them - for key in self._equip_by_level: - self._equip_by_level[key] = sorted(self._equip_by_level[key]) - logger_summary.info( - key="Equipment", - description="Sets of used equipment for each facility level in this calendar year.", - data={ - "Equipment_By_Level": self._equip_by_level, - }, - ) - self._reset_internal_stores() From acf3a6361fbecfdcf1ca4bcbf55051f4bd64d562 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Fri, 1 Dec 2023 23:16:05 +0000 Subject: [PATCH 031/118] equip_catalogue: fix the keys mapping to be done for each run --- .../equipment/equipment_catalogue.py | 157 ++++++++++++------ 1 file changed, 104 insertions(+), 53 deletions(-) diff --git a/src/scripts/healthsystem/equipment/equipment_catalogue.py b/src/scripts/healthsystem/equipment/equipment_catalogue.py index 9672c92474..1ba0f83cb4 100644 --- a/src/scripts/healthsystem/equipment/equipment_catalogue.py +++ b/src/scripts/healthsystem/equipment/equipment_catalogue.py @@ -1,14 +1,23 @@ import argparse +import warnings from pathlib import Path import pandas as pd -from tlo.analysis.utils import extract_results, load_pickled_dataframes +from tlo.analysis.utils import extract_results +# %%% TO SET %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +# Declare whether to scale the counts to Malawi population size +do_scaling = True +# Declare output file names +output_detailed_file_name = 'equipment_monthly_counts__all_event_details.csv' +output_file_name = 'equipment_annual_counts__by_Year_TreatmentID_FacLevel.csv' +# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -def get_annual_hsi_event_counts(results_folder: Path) -> pd.DataFrame: - """Return pd.DataFrame gives the simulated annual counts of all the hsi event details logged (details as keys) - for each simulated year. + +def get_monthly_hsi_event_counts(results_folder: Path) -> pd.DataFrame: + """Returned pd.DataFrame gives the monthly counts of all the hsi event details logged (details as keys) + for each simulated month. NB. 'healthsystem.summary' logger required to have been set at the level INFO or higher.""" def get_hsi_event_counts(_df): @@ -46,68 +55,110 @@ def unpack_dict_in_series(_raw: pd.Series): results_folder, module='tlo.methods.healthsystem.summary', key='hsi_event_counts', - custom_generate_series=get_hsi_event_counts + custom_generate_series=get_hsi_event_counts, + do_scaling=do_scaling ) -def get_hsi_event_keys(results_folder: Path): - return load_pickled_dataframes(results_folder, 0, 0, "tlo.methods.healthsystem.summary")[ - "tlo.methods.healthsystem.summary" - ]["hsi_event_details"]["hsi_event_key_to_event_details"][0] +def get_hsi_event_keys_all_runs(results_folder: Path) -> pd.DataFrame: + """Returned pd.DataFrame gives the dictionaries of hsi_event_details for each draw and run. + NB. 'healthsystem.summary' logger required to have been set at the level INFO or higher.""" + def get_hsi_event_keys(_df): + """Get the hsi_event_keys for one particular run.""" + return _df['hsi_event_key_to_event_details'] -def create_equipment_catalogues(results_folder: Path, output_folder: Path): + return extract_results( + results_folder, + module='tlo.methods.healthsystem.summary', + key='hsi_event_details', + custom_generate_series=get_hsi_event_keys + ) - # Declare output file names - output_detailed_file_name = 'equipment_annual_counts__all_event_details.csv' - output_file_name = 'equipment_annual_counts__by_Date_EventName_FacLevel.csv' + +def create_equipment_catalogues(results_folder: Path, output_folder: Path): # %% Catalogue equipment by all HSI event details - sim_equipment = get_annual_hsi_event_counts(results_folder) + sim_equipment = get_monthly_hsi_event_counts(results_folder) sim_equipment_df = pd.DataFrame(sim_equipment) - sim_equipment_df.fillna(0, inplace=True) - - hsi_event_keys = get_hsi_event_keys(results_folder) - - decoded_keys = sim_equipment_df.index.get_level_values(1).astype(str).map(hsi_event_keys) - sim_equipment_df = pd.concat([sim_equipment_df, pd.DataFrame(decoded_keys.tolist(), index=sim_equipment_df.index)], axis=1) - # Make values in 'appt_footprint', and 'beddays_footprint' columns to be string - sim_equipment_df['appt_footprint'] = sim_equipment_df['appt_footprint'].apply(lambda x: ', '.join(map(str, x))) - sim_equipment_df['beddays_footprint'] = sim_equipment_df['beddays_footprint'].apply(lambda x: ', '.join(map(str, x))) - # Explode the 'equipment' column - exploded_df = sim_equipment_df.explode('equipment') - # Remove the 'event_details_key' and replace the index with hsi event details as indexes - exploded_df = exploded_df.droplevel(level=1) - exploded_df = exploded_df.set_index( - ['event_name', 'module_name', 'treatment_id', 'facility_level', 'appt_footprint', 'beddays_footprint', - 'equipment'], append=True - ) - # Sum values with the same multi-index - exploded_df = exploded_df.groupby(exploded_df.index).sum() - # Convert the index back to a MultiIndex - exploded_df.index = pd.MultiIndex.from_tuples(exploded_df.index) - exploded_df.index.names = \ - ['date', 'event_name', 'module_name', 'treatment_id', 'facility_level', 'appt_footprint', 'beddays_footprint', - 'equipment'] - - # Save the detailed equipment catalogue by levels - exploded_df.to_csv(output_folder / output_detailed_file_name) + hsi_event_keys = get_hsi_event_keys_all_runs(results_folder) + + final_df = pd.DataFrame() + + def details_col_to_str(details_col): + return details_col.apply(lambda x: ', '.join(map(str, x))) + + for col in hsi_event_keys.columns: + df_col = sim_equipment_df[col].dropna() + decoded_keys = df_col.index.get_level_values(1).astype(str).map(hsi_event_keys.at[0, col]) + + # %%% Verify the keys in dictionary and dataframe for the run 'col' are same + # Check if all keys in hsi_event_keys_set are in the 'event_details_key' of df_col + hsi_event_keys_set = set(hsi_event_keys.at[0, col].keys()) + missing_keys_df =\ + [key for key in hsi_event_keys_set if key not in df_col.index.get_level_values('event_details_key')] + + # Check if all keys in the 'event_details_key' of df_col are in hsi_event_keys_set + missing_keys_dict =\ + [key for key in df_col.index.get_level_values('event_details_key') if key not in hsi_event_keys_set] + + # Warn if some keys are missing + if missing_keys_df: + warnings.warn(UserWarning(f"Keys missing in sim_equipment_df for the run {col}: {missing_keys_df}")) + + if missing_keys_dict: + warnings.warn(UserWarning(f"Keys missing in hsi_event_keys for the run {col}: {missing_keys_dict}")) + # %%% + + df_col = pd.concat([df_col, pd.DataFrame(decoded_keys.tolist(), index=df_col.index)], axis=1) + # Make values in 'appt_footprint', 'beddays_footprint' columns to be string + df_col['appt_footprint'] = details_col_to_str(df_col['appt_footprint']) + df_col['beddays_footprint'] = details_col_to_str(df_col['beddays_footprint']) + # Explode the 'equipment' column + exploded_df = df_col.explode('equipment') + # Remove the 'event_details_key' and replace the index with hsi event details as indexes + exploded_df = exploded_df.droplevel(level=1) + exploded_df = exploded_df.set_index( + ['event_name', 'module_name', 'treatment_id', 'facility_level', 'appt_footprint', 'beddays_footprint', + 'equipment'], append=True + ) + # Sum values with the same multi-index (keep also empty indexes) + exploded_df = exploded_df.groupby(level=exploded_df.index.names, dropna=False).sum() + # Add the results for the run 'col' to final_df + final_df = pd.concat([final_df, exploded_df], axis=1) + + # Replace NaN with 0 + final_df.fillna(0, inplace=True) + # Save the detailed equipment catalogue + final_df.to_csv(output_folder / output_detailed_file_name) print(f'{output_detailed_file_name} saved.') # --- - # %% Catalogue equipment by Facility Levels and HSI Event Names - - equipment_counts_by_date_hsi_name_level_df = exploded_df.copy() - - # Sum counts for each equipment with the same date, hsi event name, and level (remaining indexes removed) - # equipment_counts_by_date_hsi_name_level_df - equipment_counts_by_date_hsi_name_level_df = \ - equipment_counts_by_date_hsi_name_level_df.groupby(['date', 'event_name', 'facility_level', 'equipment']).sum() - - # Save the CSV equipment counts - equipment_counts_by_date_hsi_name_level_df.to_csv(output_folder / output_file_name) + # %% Catalogue equipment by Treatment ID and Facility Levels + equipment_counts_by_date_treatment_id_level_df = final_df.copy() + + # Sum counts for each equipment with the same date, treatment id, and facility level (remaining indexes removed), + # keeping only non-empty 'equipment' indexes + equipment_counts_by_date_treatment_id_level_df = equipment_counts_by_date_treatment_id_level_df.groupby( + ['date', 'treatment_id', 'facility_level', 'equipment'], + dropna=True + # TODO: make 'treatment_id', 'facility_level' to be an input + ).sum() + + # Sum counts annually + equipment_counts_by_date_treatment_id_level_df['year'] = \ + equipment_counts_by_date_treatment_id_level_df.index.get_level_values('date').year + # TODO: make annual/monthly results according to input + equipment_counts_by_date_treatment_id_level_df.set_index('year', append=True, inplace=True) + equipment_counts_by_date_treatment_id_level_df.index.droplevel('date') + equipment_counts_by_date_treatment_id_level_df = equipment_counts_by_date_treatment_id_level_df.groupby( + ['year', 'treatment_id', 'facility_level', 'equipment'] + ).sum() + + # Save the equipment counts CSV + equipment_counts_by_date_treatment_id_level_df.to_csv(output_folder / output_file_name) print(f'{output_file_name} saved.') - # --- + # # --- return 0 From 97f0f3f8a6cc44f1cbb5007ebec94c1b533e4019 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Sat, 2 Dec 2023 00:11:13 +0000 Subject: [PATCH 032/118] equip_catalogue: input hsi event details by which to catalog equipment --- .../equipment/equipment_catalogue.py | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/scripts/healthsystem/equipment/equipment_catalogue.py b/src/scripts/healthsystem/equipment/equipment_catalogue.py index 1ba0f83cb4..89d0c804c7 100644 --- a/src/scripts/healthsystem/equipment/equipment_catalogue.py +++ b/src/scripts/healthsystem/equipment/equipment_catalogue.py @@ -6,13 +6,23 @@ from tlo.analysis.utils import extract_results -# %%% TO SET %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +# %%% TO SET %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% # Declare whether to scale the counts to Malawi population size +# (True/False) do_scaling = True -# Declare output file names +# Declare as list by which hsi event details you want the equipment be grouped in the catalogue (choose one or more) +# (event details: 'event_name', 'module_name', 'treatment_id', 'facility_level', 'appt_footprint', 'beddays_footprint') +catalog_by = ['treatment_id', 'facility_level'] +# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +# %%% Output file names +# detailed CSV name output_detailed_file_name = 'equipment_monthly_counts__all_event_details.csv' -output_file_name = 'equipment_annual_counts__by_Year_TreatmentID_FacLevel.csv' -# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +# requested details only CSV name +output_file_name_prefix = 'equipment_annual_counts__by_year_' +output_file_name_suffix = '.csv' +output_file_name_details_specified = '_'.join(catalog_by) +output_file_name = output_file_name_prefix + output_file_name_details_specified + output_file_name_suffix def get_monthly_hsi_event_counts(results_folder: Path) -> pd.DataFrame: @@ -78,7 +88,7 @@ def get_hsi_event_keys(_df): def create_equipment_catalogues(results_folder: Path, output_folder: Path): - # %% Catalogue equipment by all HSI event details + # %% Catalog equipment by all HSI event details sim_equipment = get_monthly_hsi_event_counts(results_folder) sim_equipment_df = pd.DataFrame(sim_equipment) hsi_event_keys = get_hsi_event_keys_all_runs(results_folder) @@ -134,15 +144,15 @@ def details_col_to_str(details_col): print(f'{output_detailed_file_name} saved.') # --- - # %% Catalogue equipment by Treatment ID and Facility Levels + # %% Catalog equipment by requested details equipment_counts_by_date_treatment_id_level_df = final_df.copy() # Sum counts for each equipment with the same date, treatment id, and facility level (remaining indexes removed), # keeping only non-empty 'equipment' indexes + to_be_grouped_by = ['date'] + catalog_by + ['equipment'] equipment_counts_by_date_treatment_id_level_df = equipment_counts_by_date_treatment_id_level_df.groupby( - ['date', 'treatment_id', 'facility_level', 'equipment'], + to_be_grouped_by, dropna=True - # TODO: make 'treatment_id', 'facility_level' to be an input ).sum() # Sum counts annually From 1435d221719db19f92474c03809f0e3fdec377ea Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Sat, 2 Dec 2023 00:41:54 +0000 Subject: [PATCH 033/118] equip_catalogue: input time period by which to catalog equipment --- .../equipment/equipment_catalogue.py | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/src/scripts/healthsystem/equipment/equipment_catalogue.py b/src/scripts/healthsystem/equipment/equipment_catalogue.py index 89d0c804c7..da62e30afd 100644 --- a/src/scripts/healthsystem/equipment/equipment_catalogue.py +++ b/src/scripts/healthsystem/equipment/equipment_catalogue.py @@ -10,19 +10,22 @@ # Declare whether to scale the counts to Malawi population size # (True/False) do_scaling = True -# Declare as list by which hsi event details you want the equipment be grouped in the catalogue (choose one or more) +# Declare as a list by which hsi event details you want the equipment be grouped in the catalogue (choose one or more) # (event details: 'event_name', 'module_name', 'treatment_id', 'facility_level', 'appt_footprint', 'beddays_footprint') -catalog_by = ['treatment_id', 'facility_level'] +catalog_by_details = ['treatment_id', 'facility_level'] +# Declare which time period you want the equipment be grouped in the catalogue (choose only one) +# (periods: 'monthly', 'annual') +catalog_by_time = 'annual' # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +# TODO: verify inputs are as expected # %%% Output file names # detailed CSV name output_detailed_file_name = 'equipment_monthly_counts__all_event_details.csv' # requested details only CSV name -output_file_name_prefix = 'equipment_annual_counts__by_year_' -output_file_name_suffix = '.csv' -output_file_name_details_specified = '_'.join(catalog_by) -output_file_name = output_file_name_prefix + output_file_name_details_specified + output_file_name_suffix +time_index = 'year' if catalog_by_time == 'annual' else 'date' +output_file_name = \ + 'equipment_' + catalog_by_time + '_counts__by_' + time_index + '_' + '_'.join(catalog_by_details) + '.csv' def get_monthly_hsi_event_counts(results_folder: Path) -> pd.DataFrame: @@ -145,28 +148,29 @@ def details_col_to_str(details_col): # --- # %% Catalog equipment by requested details - equipment_counts_by_date_treatment_id_level_df = final_df.copy() + equipment_counts_by_time_and_requested_details = final_df.copy() # Sum counts for each equipment with the same date, treatment id, and facility level (remaining indexes removed), # keeping only non-empty 'equipment' indexes - to_be_grouped_by = ['date'] + catalog_by + ['equipment'] - equipment_counts_by_date_treatment_id_level_df = equipment_counts_by_date_treatment_id_level_df.groupby( + to_be_grouped_by = ['date'] + catalog_by_details + ['equipment'] + equipment_counts_by_time_and_requested_details = equipment_counts_by_time_and_requested_details.groupby( to_be_grouped_by, dropna=True ).sum() - # Sum counts annually - equipment_counts_by_date_treatment_id_level_df['year'] = \ - equipment_counts_by_date_treatment_id_level_df.index.get_level_values('date').year - # TODO: make annual/monthly results according to input - equipment_counts_by_date_treatment_id_level_df.set_index('year', append=True, inplace=True) - equipment_counts_by_date_treatment_id_level_df.index.droplevel('date') - equipment_counts_by_date_treatment_id_level_df = equipment_counts_by_date_treatment_id_level_df.groupby( - ['year', 'treatment_id', 'facility_level', 'equipment'] - ).sum() + if catalog_by_time == 'annual': + # Sum counts annually + equipment_counts_by_time_and_requested_details['year'] = \ + equipment_counts_by_time_and_requested_details.index.get_level_values('date').year + equipment_counts_by_time_and_requested_details.set_index('year', append=True, inplace=True) + equipment_counts_by_time_and_requested_details.index.droplevel('date') + to_be_grouped_by = ['year'] + catalog_by_details + ['equipment'] + equipment_counts_by_time_and_requested_details = equipment_counts_by_time_and_requested_details.groupby( + to_be_grouped_by + ).sum() # Save the equipment counts CSV - equipment_counts_by_date_treatment_id_level_df.to_csv(output_folder / output_file_name) + equipment_counts_by_time_and_requested_details.to_csv(output_folder / output_file_name) print(f'{output_file_name} saved.') # # --- From 14a11a7c76858013c3361350809c762f83cbcbc3 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Sat, 2 Dec 2023 00:48:08 +0000 Subject: [PATCH 034/118] equip_catalogue: list of requested details can be empty --- src/scripts/healthsystem/equipment/equipment_catalogue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scripts/healthsystem/equipment/equipment_catalogue.py b/src/scripts/healthsystem/equipment/equipment_catalogue.py index da62e30afd..d69bc9e638 100644 --- a/src/scripts/healthsystem/equipment/equipment_catalogue.py +++ b/src/scripts/healthsystem/equipment/equipment_catalogue.py @@ -10,7 +10,7 @@ # Declare whether to scale the counts to Malawi population size # (True/False) do_scaling = True -# Declare as a list by which hsi event details you want the equipment be grouped in the catalogue (choose one or more) +# Declare as a list by which hsi event details you want the equipment be grouped in the catalogue (choose any number) # (event details: 'event_name', 'module_name', 'treatment_id', 'facility_level', 'appt_footprint', 'beddays_footprint') catalog_by_details = ['treatment_id', 'facility_level'] # Declare which time period you want the equipment be grouped in the catalogue (choose only one) From 6d85fe13a65b3a5956c7b1d8237479a84520798c Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Sat, 2 Dec 2023 01:44:39 +0000 Subject: [PATCH 035/118] equip_catalogue: verify inputs as expected & set output file names in 'create_equipment_catalogues' fnc --- .../equipment/equipment_catalogue.py | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/src/scripts/healthsystem/equipment/equipment_catalogue.py b/src/scripts/healthsystem/equipment/equipment_catalogue.py index d69bc9e638..66fcff01ae 100644 --- a/src/scripts/healthsystem/equipment/equipment_catalogue.py +++ b/src/scripts/healthsystem/equipment/equipment_catalogue.py @@ -17,15 +17,6 @@ # (periods: 'monthly', 'annual') catalog_by_time = 'annual' # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -# TODO: verify inputs are as expected - -# %%% Output file names -# detailed CSV name -output_detailed_file_name = 'equipment_monthly_counts__all_event_details.csv' -# requested details only CSV name -time_index = 'year' if catalog_by_time == 'annual' else 'date' -output_file_name = \ - 'equipment_' + catalog_by_time + '_counts__by_' + time_index + '_' + '_'.join(catalog_by_details) + '.csv' def get_monthly_hsi_event_counts(results_folder: Path) -> pd.DataFrame: @@ -90,6 +81,27 @@ def get_hsi_event_keys(_df): def create_equipment_catalogues(results_folder: Path, output_folder: Path): + # %%% Verify inputs are as expected + assert isinstance(do_scaling, bool), "The input parameter 'do_scaling' must be a boolean (True or False)" + assert isinstance(catalog_by_details, list), "The input parameter 'catalog_by_details' must be a list" + event_details = \ + {'event_name', 'module_name', 'treatment_id', 'facility_level', 'appt_footprint', 'beddays_footprint'} + for item in catalog_by_details: + assert isinstance(item, str) and item in event_details, \ + f"Each element in the input list 'catalog_by_details' must be a string and be one of the details:\n" \ + f"{event_details}" + assert catalog_by_time in {'monthly', 'annual'}, \ + "The input parameter 'catalog_by_time' must be one of the strings ('monthly' or 'annual')" + # --- + + # %%% Set output file names + # detailed CSV name + output_detailed_file_name = 'equipment_monthly_counts__all_event_details.csv' + # requested details only CSV name + time_index = 'year' if catalog_by_time == 'annual' else 'date' + output_file_name = \ + 'equipment_' + catalog_by_time + '_counts__by_' + time_index + '_' + '_'.join(catalog_by_details) + '.csv' + # --- # %% Catalog equipment by all HSI event details sim_equipment = get_monthly_hsi_event_counts(results_folder) @@ -172,7 +184,7 @@ def details_col_to_str(details_col): # Save the equipment counts CSV equipment_counts_by_time_and_requested_details.to_csv(output_folder / output_file_name) print(f'{output_file_name} saved.') - # # --- + # --- return 0 From 934e52d9281f92d9e56f4e34be11b4670ccdec9c Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Wed, 6 Dec 2023 16:54:53 +0000 Subject: [PATCH 036/118] equip_catalogue: (1) detailed - equip set as string in one row, module_name before event_name, sorted by indexes; (2) added summary catal. (3) focused - only this cat. without empty equip rows and split by item per row --- .../equipment/equipment_catalogue.py | 62 +++++++++++++------ 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/src/scripts/healthsystem/equipment/equipment_catalogue.py b/src/scripts/healthsystem/equipment/equipment_catalogue.py index 66fcff01ae..6ba03916c8 100644 --- a/src/scripts/healthsystem/equipment/equipment_catalogue.py +++ b/src/scripts/healthsystem/equipment/equipment_catalogue.py @@ -99,8 +99,9 @@ def create_equipment_catalogues(results_folder: Path, output_folder: Path): output_detailed_file_name = 'equipment_monthly_counts__all_event_details.csv' # requested details only CSV name time_index = 'year' if catalog_by_time == 'annual' else 'date' - output_file_name = \ + output_focused_file_name = \ 'equipment_' + catalog_by_time + '_counts__by_' + time_index + '_' + '_'.join(catalog_by_details) + '.csv' + output_summary_file_name = 'equipment_summary__module_name_event_name_treatment_id.csv' # --- # %% Catalog equipment by all HSI event details @@ -113,6 +114,12 @@ def create_equipment_catalogues(results_folder: Path, output_folder: Path): def details_col_to_str(details_col): return details_col.apply(lambda x: ', '.join(map(str, x))) + def lists_of_strings_to_strings_of_list(list_of_strings_col): + return list_of_strings_col.apply(lambda x: "['" + "', '".join(map(str, x)) + "']") + + def strings_of_list_to_lists_of_strings(strings_of_list_col): + return strings_of_list_col.apply(lambda x: x.strip('][').split(', ')) + for col in hsi_event_keys.columns: df_col = sim_equipment_df[col].dropna() decoded_keys = df_col.index.get_level_values(1).astype(str).map(hsi_event_keys.at[0, col]) @@ -136,34 +143,38 @@ def details_col_to_str(details_col): # %%% df_col = pd.concat([df_col, pd.DataFrame(decoded_keys.tolist(), index=df_col.index)], axis=1) - # Make values in 'appt_footprint', 'beddays_footprint' columns to be string + # Make values in 'appt_footprint', 'beddays_footprint', and 'equipment' columns to be string df_col['appt_footprint'] = details_col_to_str(df_col['appt_footprint']) df_col['beddays_footprint'] = details_col_to_str(df_col['beddays_footprint']) - # Explode the 'equipment' column - exploded_df = df_col.explode('equipment') - # Remove the 'event_details_key' and replace the index with hsi event details as indexes - exploded_df = exploded_df.droplevel(level=1) - exploded_df = exploded_df.set_index( - ['event_name', 'module_name', 'treatment_id', 'facility_level', 'appt_footprint', 'beddays_footprint', - 'equipment'], append=True - ) - # Sum values with the same multi-index (keep also empty indexes) - exploded_df = exploded_df.groupby(level=exploded_df.index.names, dropna=False).sum() - # Add the results for the run 'col' to final_df - final_df = pd.concat([final_df, exploded_df], axis=1) + df_col['equipment'] = lists_of_strings_to_strings_of_list(df_col['equipment']) + df_col = (df_col.droplevel(level=1) + .set_index(['module_name', 'event_name', 'treatment_id', 'facility_level', 'appt_footprint', + 'beddays_footprint', 'equipment'], append=True)) + final_df = pd.concat([final_df, df_col], axis=1) # Replace NaN with 0 final_df.fillna(0, inplace=True) + final_df.sort_index(inplace=True) # Save the detailed equipment catalogue final_df.to_csv(output_folder / output_detailed_file_name) print(f'{output_detailed_file_name} saved.') # --- + # %% Catalog equipment summary + equipment_summary = final_df.copy() + equipment_summary = equipment_summary.groupby(['module_name', 'event_name', 'treatment_id', 'equipment']).sum() + equipment_summary = \ + equipment_summary.reset_index().set_index(['module_name', 'event_name', 'treatment_id', 'equipment']) + # Save the summary equipment catalogue + equipment_summary.index.to_frame().to_csv(output_folder / output_summary_file_name, index=False) + print(f'{output_summary_file_name} saved.') + # --- + # %% Catalog equipment by requested details equipment_counts_by_time_and_requested_details = final_df.copy() - # Sum counts for each equipment with the same date, treatment id, and facility level (remaining indexes removed), - # keeping only non-empty 'equipment' indexes + # Sum counts for each equipment set with the same date, treatment id, and facility level + # (remaining indexes removed), keeping only non-empty 'equipment' indexes to_be_grouped_by = ['date'] + catalog_by_details + ['equipment'] equipment_counts_by_time_and_requested_details = equipment_counts_by_time_and_requested_details.groupby( to_be_grouped_by, @@ -181,9 +192,24 @@ def details_col_to_str(details_col): to_be_grouped_by ).sum() + # Remove rows with no equipment used + equipment_counts_by_time_and_requested_details.drop("['']", level='equipment', axis=0, inplace=True) + # Split the equipment by an item per row + equipment_counts_by_time_and_requested_details['equipment'] = \ + equipment_counts_by_time_and_requested_details.index.get_level_values('equipment') + equipment_counts_by_time_and_requested_details.index = \ + equipment_counts_by_time_and_requested_details.index.droplevel('equipment') + equipment_counts_by_time_and_requested_details['equipment'] = strings_of_list_to_lists_of_strings( + equipment_counts_by_time_and_requested_details['equipment'] + ) + exploded_df = equipment_counts_by_time_and_requested_details.explode('equipment') + exploded_df = exploded_df.set_index(['equipment'], append=True) + # Sum values with the same multi-index + exploded_df = exploded_df.groupby(level=exploded_df.index.names).sum() + # Save the equipment counts CSV - equipment_counts_by_time_and_requested_details.to_csv(output_folder / output_file_name) - print(f'{output_file_name} saved.') + exploded_df.to_csv(output_folder / output_focused_file_name) + print(f'{output_focused_file_name} saved.') # --- return 0 From bdbd8f73b859d2bac1e0f95c259ea0fb5d39a5e2 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Wed, 6 Dec 2023 21:30:58 +0000 Subject: [PATCH 037/118] equip_catalogue: suffix (as input) added for output file names --- .../healthsystem/equipment/equipment_catalogue.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/scripts/healthsystem/equipment/equipment_catalogue.py b/src/scripts/healthsystem/equipment/equipment_catalogue.py index 6ba03916c8..4506892e8b 100644 --- a/src/scripts/healthsystem/equipment/equipment_catalogue.py +++ b/src/scripts/healthsystem/equipment/equipment_catalogue.py @@ -16,6 +16,8 @@ # Declare which time period you want the equipment be grouped in the catalogue (choose only one) # (periods: 'monthly', 'annual') catalog_by_time = 'annual' +# Suffix for output file names +suffix_file_names = '__5y_20Kpop_10runs' # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -96,12 +98,13 @@ def create_equipment_catalogues(results_folder: Path, output_folder: Path): # %%% Set output file names # detailed CSV name - output_detailed_file_name = 'equipment_monthly_counts__all_event_details.csv' + output_detailed_file_name = 'equipment_monthly_counts__all_event_details' + suffix_file_names + '.csv' # requested details only CSV name time_index = 'year' if catalog_by_time == 'annual' else 'date' output_focused_file_name = \ - 'equipment_' + catalog_by_time + '_counts__by_' + time_index + '_' + '_'.join(catalog_by_details) + '.csv' - output_summary_file_name = 'equipment_summary__module_name_event_name_treatment_id.csv' + 'equipment_' + catalog_by_time + '_counts__by_' + time_index + '_' + '_'.join(catalog_by_details) + \ + suffix_file_names + '.csv' + output_summary_file_name = 'equipment_summary__module_name_event_name_treatment_id)' + suffix_file_names + '.csv' # --- # %% Catalog equipment by all HSI event details From 070454c13881d50c2e41812e10dbafdfe487ebb4 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Sat, 13 Jan 2024 10:26:11 +0000 Subject: [PATCH 038/118] hs: structure v2; alri+co: examples to test new structure --- src/tlo/methods/alri.py | 4 ++++ src/tlo/methods/contraception.py | 19 ++++++++++++++++++- src/tlo/methods/healthsystem.py | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/src/tlo/methods/alri.py b/src/tlo/methods/alri.py index e4b0b46d17..c211e33179 100644 --- a/src/tlo/methods/alri.py +++ b/src/tlo/methods/alri.py @@ -2290,6 +2290,8 @@ def __init__(self, module: Module, person_id: int, facility_level: str = "0", in self._treatment_id_stub = 'Alri_Pneumonia_Treatment' self._facility_levels = ("0", "1a", "1b", "2") # Health facility levels at which care may be provided assert facility_level in self._facility_levels + self.set_essential_equipment({'Pulse oximeter'}) + # TODO: CORRECT --- an example with ess. equipm. set (which may or may not be used at the end) self.is_followup_following_treatment_failure = is_followup_following_treatment_failure if not inpatient: @@ -2596,6 +2598,8 @@ def _get_disease_classification_for_treatment_decision(self, 'cough_or_cold' (symptoms-based assessment) }.""" + self.update_equipment({'Pulse oximeter'}) + child_is_younger_than_2_months = age_exact_years < (2.0 / 12.0) imci_classification_based_on_symptoms = self._get_imci_classification_based_on_symptoms( diff --git a/src/tlo/methods/contraception.py b/src/tlo/methods/contraception.py index 0a4884d0f3..2a615420ef 100644 --- a/src/tlo/methods/contraception.py +++ b/src/tlo/methods/contraception.py @@ -1109,6 +1109,7 @@ def __init__(self, module, person_id, new_contraceptive): self.TREATMENT_ID = "Contraception_Routine" self.ACCEPTED_FACILITY_LEVEL = _facility_level + self.set_essential_equipment({''}) @property def EXPECTED_APPT_FOOTPRINT(self): @@ -1144,6 +1145,11 @@ def apply(self, person_id, squeeze_factor): # Record the date that Family Planning Appointment happened for this person self.sim.population.props.at[person_id, "co_date_of_last_fp_appt"] = self.sim.date + # Measure weight, height and BP even if contraception not administrated + self.update_equipment({ + 'Weighing scale', 'Height Pole (Stadiometer)', 'Blood pressure machine' + }) + # Determine essential and optional items # TODO: we don't distinguish essential X optional for contraception methods yet, will need to update once we do items_essential = self.module.cons_codes[self.new_contraceptive] @@ -1170,7 +1176,8 @@ def apply(self, person_id, squeeze_factor): items_all = {**items_essential, **items_optional} # Determine whether the contraception is administrated (ie all essential items are available), - # if so do log the availability of all items, if not set the contraception to "not_using": + # if so do log the availability of all items and update used equipment if any, if not set the contraception to + # "not_using": co_administrated = all(v for k, v in cons_available.items() if k in items_essential) if co_administrated: @@ -1195,6 +1202,16 @@ def apply(self, person_id, squeeze_factor): _new_contraceptive = self.new_contraceptive + # Update equipment if any needed for the method + if _new_contraceptive == 'female_sterilization': + self.update_equipment({ + 'Cusco’s/ bivalved Speculum (small, medium, large)', 'Lamp, Anglepoise' + }) + elif _new_contraceptive == 'IUD': + self.update_equipment({ + 'Cusco’s/ bivalved Speculum (small, medium, large)', 'Sponge Holding Forceps' + }) + else: _new_contraceptive = "not_using" diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index af0610fe6c..6021f0eb46 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -183,6 +183,7 @@ def __init__(self, module, *args, **kwargs): self._received_info_about_bed_days = None self.expected_time_requests = {} self.facility_info = None + # self.set_essential_equipment({''}) # HSI needs this attribute, but it is not defined in the Base class self.EQUIPMENT = set() @property @@ -339,6 +340,37 @@ def make_appt_footprint(self, dict_of_appts): "values" ) + def set_essential_equipment(self, set_of_equip): + """Helper function to set essential equipment. + + Should be passed a set of equipment items names (strings) or an empty set. + """ + # Set EQUIPMENT if the given set_of_equip in correct format, ie a set of strings or an empty set + if isinstance(set_of_equip, set) and all(isinstance(item, str) for item in set_of_equip): + self.ESSENTIAL_EQUIPMENT = set_of_equip + return 0 + + raise ValueError( + "Argument to set_essential_equipment should be an empty set or a set of strings of equipment item names " + "from ResourceFile_Equipment.csv." + ) + + def update_equipment(self, set_of_equip): + """Helper function to update equipment. + + Should be passed a set of equipment item names (strings). + """ + # Update EQUIPMENT if the given set_of_equip in correct format, ie a non-empty set of strings + if isinstance(set_of_equip, set) and (all(isinstance(item, str) for item in set_of_equip)) and \ + (set_of_equip not in [set(), None, {''}]): + self.EQUIPMENT.update(set_of_equip) + return self.EQUIPMENT.discard('') + + raise ValueError( + "Argument to update_equipment should be a non-empty set of strings of equipment item names " + "from ResourceFile_Equipment.csv." + ) + def initialise(self): """Initialise the HSI: * Set the facility_info From 99be34a2d17b217baaceab81df6e637c612604fa Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Sat, 13 Jan 2024 10:27:05 +0000 Subject: [PATCH 039/118] equip_catalogue: TODO added --- src/scripts/healthsystem/equipment/equipment_catalogue.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/scripts/healthsystem/equipment/equipment_catalogue.py b/src/scripts/healthsystem/equipment/equipment_catalogue.py index 4506892e8b..877aa7460b 100644 --- a/src/scripts/healthsystem/equipment/equipment_catalogue.py +++ b/src/scripts/healthsystem/equipment/equipment_catalogue.py @@ -21,6 +21,7 @@ # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +# TODO: Could I have use the bin_hsi_event_details from src/tlo/analysis/utils.py instead? If so, how? def get_monthly_hsi_event_counts(results_folder: Path) -> pd.DataFrame: """Returned pd.DataFrame gives the monthly counts of all the hsi event details logged (details as keys) for each simulated month. From 5076e3ba38f62ba3259245b7bf6fc96c79781145 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Wed, 6 Dec 2023 22:39:54 +0000 Subject: [PATCH 040/118] equip_catalogue: typo --- src/scripts/healthsystem/equipment/equipment_catalogue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scripts/healthsystem/equipment/equipment_catalogue.py b/src/scripts/healthsystem/equipment/equipment_catalogue.py index 877aa7460b..dd2dde8015 100644 --- a/src/scripts/healthsystem/equipment/equipment_catalogue.py +++ b/src/scripts/healthsystem/equipment/equipment_catalogue.py @@ -105,7 +105,7 @@ def create_equipment_catalogues(results_folder: Path, output_folder: Path): output_focused_file_name = \ 'equipment_' + catalog_by_time + '_counts__by_' + time_index + '_' + '_'.join(catalog_by_details) + \ suffix_file_names + '.csv' - output_summary_file_name = 'equipment_summary__module_name_event_name_treatment_id)' + suffix_file_names + '.csv' + output_summary_file_name = 'equipment_summary__module_name_event_name_treatment_id' + suffix_file_names + '.csv' # --- # %% Catalog equipment by all HSI event details From d47b53ef2d8b36198788786e281d9680f45f483f Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Sat, 13 Jan 2024 11:08:04 +0000 Subject: [PATCH 041/118] equip_catalogue: TODO added --- src/scripts/healthsystem/equipment/equipment_catalogue.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/scripts/healthsystem/equipment/equipment_catalogue.py b/src/scripts/healthsystem/equipment/equipment_catalogue.py index dd2dde8015..4b91510c61 100644 --- a/src/scripts/healthsystem/equipment/equipment_catalogue.py +++ b/src/scripts/healthsystem/equipment/equipment_catalogue.py @@ -6,6 +6,7 @@ from tlo.analysis.utils import extract_results +# TODO: make these to be arguments of called fnc # %%% TO SET %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% # Declare whether to scale the counts to Malawi population size # (True/False) From 057fb440bbbfcc942d4d9be6946e8c6fbc93a0be Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Sat, 13 Jan 2024 11:37:07 +0000 Subject: [PATCH 042/118] equip_catalogue: bug fixed --- src/scripts/healthsystem/equipment/equipment_catalogue.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/scripts/healthsystem/equipment/equipment_catalogue.py b/src/scripts/healthsystem/equipment/equipment_catalogue.py index 4b91510c61..4f47af6d0d 100644 --- a/src/scripts/healthsystem/equipment/equipment_catalogue.py +++ b/src/scripts/healthsystem/equipment/equipment_catalogue.py @@ -18,7 +18,7 @@ # (periods: 'monthly', 'annual') catalog_by_time = 'annual' # Suffix for output file names -suffix_file_names = '__5y_20Kpop_10runs' +suffix_file_names = '__2y_2Kpop_4runs_1draw' # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -123,7 +123,8 @@ def lists_of_strings_to_strings_of_list(list_of_strings_col): return list_of_strings_col.apply(lambda x: "['" + "', '".join(map(str, x)) + "']") def strings_of_list_to_lists_of_strings(strings_of_list_col): - return strings_of_list_col.apply(lambda x: x.strip('][').split(', ')) + lists_of_strings_col = strings_of_list_col.apply(lambda x: x.strip('][').split("'")) + return lists_of_strings_col.apply(lambda x: [s for s in x if (s != '' and s != ', ')]) for col in hsi_event_keys.columns: df_col = sim_equipment_df[col].dropna() From 2bf3e8ae13cbfb9733c7d72caff3ca9ff68283f1 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Sat, 13 Jan 2024 18:57:30 +0000 Subject: [PATCH 043/118] brc: TODO added --- src/tlo/methods/breast_cancer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tlo/methods/breast_cancer.py b/src/tlo/methods/breast_cancer.py index 1ce9ad2bf6..2529911d84 100644 --- a/src/tlo/methods/breast_cancer.py +++ b/src/tlo/methods/breast_cancer.py @@ -646,6 +646,7 @@ def __init__(self, module, person_id): self.TREATMENT_ID = "BreastCancer_Investigation" self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint({"Over5OPD": 1, "Mammography": 1}) self.ACCEPTED_FACILITY_LEVEL = '3' # Biopsy only available at level 3 and above. + # TODO: but the appt footprints suggests mammography to be provided def apply(self, person_id, squeeze_factor): df = self.sim.population.props From fa3380f52c17bc39b55b655c69203bb605db2345 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Sat, 13 Jan 2024 19:00:02 +0000 Subject: [PATCH 044/118] hs: rm prints --- src/tlo/methods/healthsystem.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 6021f0eb46..9fd7428f76 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -1876,8 +1876,6 @@ def write_to_hsi_log( description="record of each HSI event" ) if did_run: - print("\nevent_details") - print(event_details) if self._hsi_event_count_log_period is not None: event_details_key = self._hsi_event_details.setdefault( event_details, len(self._hsi_event_details) From 4b7904a443bd6fb6b065ec44c1d8d38dd1328d13 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Sat, 13 Jan 2024 19:01:28 +0000 Subject: [PATCH 045/118] hs: rm old code --- src/tlo/methods/healthsystem.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 9fd7428f76..8bd9fac9ea 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -1887,7 +1887,6 @@ def write_to_hsi_log( squeeze_factor=squeeze_factor, appt_footprint=event_details.appt_footprint, level=event_details.facility_level, - equipment=equipment, ) def call_and_record_never_ran_hsi_event(self, hsi_event, priority=None): From 8c8bcc544eeac5a03f877c539d1d2bdd1b20e727 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Sat, 13 Jan 2024 19:04:14 +0000 Subject: [PATCH 046/118] hs: rm/add accidentally added/rmd commas --- src/tlo/methods/healthsystem.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 8bd9fac9ea..1315b93c0d 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -1919,7 +1919,7 @@ def write_to_never_ran_hsi_log( event_details: HSIEventDetails, person_id: int, facility_id: Optional[int], - priority: int + priority: int, ): """Write the log `HSI_Event` and add to the summary counter.""" logger.debug( @@ -1944,7 +1944,7 @@ def write_to_never_ran_hsi_log( treatment_id=event_details.treatment_id, hsi_event_name=event_details.event_name, appt_footprint=event_details.appt_footprint, - level=event_details.facility_level + level=event_details.facility_level, ) def log_current_capabilities_and_usage(self): @@ -2204,7 +2204,7 @@ def run_individual_level_events_in_mode_0_or_1(self, actual_appt_footprint=actual_appt_footprint, squeeze_factor=squeeze_factor, did_run=True, - priority=_priority, + priority=_priority ) # if not ok_to_run From 6b03eddf9d2a037d96d61919125bf5d306747cfe Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Mon, 15 Jan 2024 16:36:57 +0000 Subject: [PATCH 047/118] hs & alri+co: rename and correct return of fncs related to equipment --- src/tlo/methods/alri.py | 4 ++-- src/tlo/methods/contraception.py | 8 +++---- src/tlo/methods/healthsystem.py | 38 +++++++++++++++++--------------- 3 files changed, 26 insertions(+), 24 deletions(-) diff --git a/src/tlo/methods/alri.py b/src/tlo/methods/alri.py index c211e33179..640d4ec054 100644 --- a/src/tlo/methods/alri.py +++ b/src/tlo/methods/alri.py @@ -2290,7 +2290,7 @@ def __init__(self, module: Module, person_id: int, facility_level: str = "0", in self._treatment_id_stub = 'Alri_Pneumonia_Treatment' self._facility_levels = ("0", "1a", "1b", "2") # Health facility levels at which care may be provided assert facility_level in self._facility_levels - self.set_essential_equipment({'Pulse oximeter'}) + self.set_equipment_essential_to_run_event({'Pulse oximeter'}) # TODO: CORRECT --- an example with ess. equipm. set (which may or may not be used at the end) self.is_followup_following_treatment_failure = is_followup_following_treatment_failure @@ -2598,7 +2598,7 @@ def _get_disease_classification_for_treatment_decision(self, 'cough_or_cold' (symptoms-based assessment) }.""" - self.update_equipment({'Pulse oximeter'}) + self.add_equipment({'Pulse oximeter'}) child_is_younger_than_2_months = age_exact_years < (2.0 / 12.0) diff --git a/src/tlo/methods/contraception.py b/src/tlo/methods/contraception.py index 2a615420ef..dab1151ed4 100644 --- a/src/tlo/methods/contraception.py +++ b/src/tlo/methods/contraception.py @@ -1109,7 +1109,7 @@ def __init__(self, module, person_id, new_contraceptive): self.TREATMENT_ID = "Contraception_Routine" self.ACCEPTED_FACILITY_LEVEL = _facility_level - self.set_essential_equipment({''}) + self.set_equipment_essential_to_run_event({''}) @property def EXPECTED_APPT_FOOTPRINT(self): @@ -1146,7 +1146,7 @@ def apply(self, person_id, squeeze_factor): self.sim.population.props.at[person_id, "co_date_of_last_fp_appt"] = self.sim.date # Measure weight, height and BP even if contraception not administrated - self.update_equipment({ + self.add_equipment({ 'Weighing scale', 'Height Pole (Stadiometer)', 'Blood pressure machine' }) @@ -1204,11 +1204,11 @@ def apply(self, person_id, squeeze_factor): # Update equipment if any needed for the method if _new_contraceptive == 'female_sterilization': - self.update_equipment({ + self.add_equipment({ 'Cusco’s/ bivalved Speculum (small, medium, large)', 'Lamp, Anglepoise' }) elif _new_contraceptive == 'IUD': - self.update_equipment({ + self.add_equipment({ 'Cusco’s/ bivalved Speculum (small, medium, large)', 'Sponge Holding Forceps' }) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 1315b93c0d..6a735baddf 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -183,7 +183,10 @@ def __init__(self, module, *args, **kwargs): self._received_info_about_bed_days = None self.expected_time_requests = {} self.facility_info = None - # self.set_essential_equipment({''}) # HSI needs this attribute, but it is not defined in the Base class + # self.set_equipment_essential_to_run_event({''}) # HSI needs this attribute, but it is not defined in the Base + # class to allow verification of its existence as a test for + # each HSI event, showing that equipment setup was thought + # through for the event. self.EQUIPMENT = set() @property @@ -340,36 +343,35 @@ def make_appt_footprint(self, dict_of_appts): "values" ) - def set_essential_equipment(self, set_of_equip): + def set_equipment_essential_to_run_event(self, set_of_equip): """Helper function to set essential equipment. Should be passed a set of equipment items names (strings) or an empty set. """ # Set EQUIPMENT if the given set_of_equip in correct format, ie a set of strings or an empty set - if isinstance(set_of_equip, set) and all(isinstance(item, str) for item in set_of_equip): - self.ESSENTIAL_EQUIPMENT = set_of_equip - return 0 + if not isinstance(set_of_equip, set) or any(not isinstance(item, str) for item in set_of_equip): + raise ValueError( + "Argument to set_equipment_essential_to_run_event should be an empty set or a set of strings of " + "equipment item names from ResourceFile_Equipment.csv." + ) - raise ValueError( - "Argument to set_essential_equipment should be an empty set or a set of strings of equipment item names " - "from ResourceFile_Equipment.csv." - ) + self.ESSENTIAL_EQUIPMENT = set_of_equip - def update_equipment(self, set_of_equip): + def add_equipment(self, set_of_equip): """Helper function to update equipment. Should be passed a set of equipment item names (strings). """ # Update EQUIPMENT if the given set_of_equip in correct format, ie a non-empty set of strings - if isinstance(set_of_equip, set) and (all(isinstance(item, str) for item in set_of_equip)) and \ - (set_of_equip not in [set(), None, {''}]): - self.EQUIPMENT.update(set_of_equip) - return self.EQUIPMENT.discard('') + if not isinstance(set_of_equip, set) or any(not isinstance(item, str) for item in set_of_equip) or \ + (set_of_equip in [set(), None, {''}]): + raise ValueError( + "Argument to add_equipment should be a non-empty set of strings of " + "equipment item names from ResourceFile_Equipment.csv." + ) - raise ValueError( - "Argument to update_equipment should be a non-empty set of strings of equipment item names " - "from ResourceFile_Equipment.csv." - ) + self.EQUIPMENT.update(set_of_equip) + self.EQUIPMENT.discard('') def initialise(self): """Initialise the HSI: From c37e1de9ac120eb7f2f37c0fdcc05722862a4c19 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Tue, 16 Jan 2024 17:44:06 +0000 Subject: [PATCH 048/118] hs: log equip item codes instead of names --- src/tlo/methods/healthsystem.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 6a735baddf..0411bdbe9a 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -343,6 +343,10 @@ def make_appt_footprint(self, dict_of_appts): "values" ) + def get_equip_item_code_from_item_name(self, lookup_df: pd.DataFrame, equip_item_name: str) -> int: + """Helper function to provide the equip_item_code (an int) when provided with the equip_item_name of the item""" + return int(pd.unique(lookup_df.loc[lookup_df["Equip_Item"] == equip_item_name, "Equip_Code"])[0]) + def set_equipment_essential_to_run_event(self, set_of_equip): """Helper function to set essential equipment. @@ -369,8 +373,13 @@ def add_equipment(self, set_of_equip): "Argument to add_equipment should be a non-empty set of strings of " "equipment item names from ResourceFile_Equipment.csv." ) - - self.EQUIPMENT.update(set_of_equip) + # from the set of equip item names create a set of item codes + # this function is calling parameters from this + equip_codes = set(self.get_equip_item_code_from_item_name( + self.sim.modules['HealthSystem'].parameters['equip_item_and_package_code_lookups'], item_name + ) for item_name in set_of_equip + ) + self.EQUIPMENT.update(equip_codes) self.EQUIPMENT.discard('') def initialise(self): @@ -561,6 +570,9 @@ class HealthSystem(Module): "Availability of beds. If 'default' then use the availability specified in the ResourceFile; if " "'none', then let no beds be ever be available; if 'all', then all beds are always available. NB. This " "parameter is over-ridden if an argument is provided to the module initialiser."), + 'equip_item_and_package_code_lookups': Parameter( + Types.DATA_FRAME, "Items based on the the HSSP III 1K Equipment Costing (SEL Costing Sheet): " + "https://www.health.gov.mw/download/hssp-iii/, packages created in consultation with clinicians."), # Service Availability 'Service_Availability': Parameter( @@ -848,6 +860,10 @@ def read_parameters(self, data_folder): self.parameters['BedCapacity'] = pd.read_csv( path_to_resourcefiles_for_healthsystem / 'infrastructure_and_equipment' / 'ResourceFile_Bed_Capacity.csv') + # Read in ResourceFile_Equipment + self.parameters['equip_item_and_package_code_lookups'] = pd.read_csv( + path_to_resourcefiles_for_healthsystem / 'infrastructure_and_equipment' / 'ResourceFile_Equipment.csv') + # Data on the priority of each Treatment_ID that should be adopted in the queueing system according to different # priority policies. Load all policies at this stage, and decide later which one to adopt. self.parameters['priority_rank'] = pd.read_excel(path_to_resourcefiles_for_healthsystem / 'priority_policies' / From 99ad1389eabbf4c9c3d92346059997479cf870e6 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Wed, 17 Jan 2024 17:41:47 +0000 Subject: [PATCH 049/118] equip_catalogue: updated for logged equip item codes --- .../equipment/equipment_catalogue.py | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/scripts/healthsystem/equipment/equipment_catalogue.py b/src/scripts/healthsystem/equipment/equipment_catalogue.py index 4f47af6d0d..aa4552e536 100644 --- a/src/scripts/healthsystem/equipment/equipment_catalogue.py +++ b/src/scripts/healthsystem/equipment/equipment_catalogue.py @@ -109,6 +109,12 @@ def create_equipment_catalogues(results_folder: Path, output_folder: Path): output_summary_file_name = 'equipment_summary__module_name_event_name_treatment_id' + suffix_file_names + '.csv' # --- + # %%% Load RF + # Equipment + equip_resource_items_pkgs_df = pd.read_csv( + 'resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv' + ) + # %% Catalog equipment by all HSI event details sim_equipment = get_monthly_hsi_event_counts(results_folder) sim_equipment_df = pd.DataFrame(sim_equipment) @@ -119,8 +125,15 @@ def create_equipment_catalogues(results_folder: Path, output_folder: Path): def details_col_to_str(details_col): return details_col.apply(lambda x: ', '.join(map(str, x))) - def lists_of_strings_to_strings_of_list(list_of_strings_col): - return list_of_strings_col.apply(lambda x: "['" + "', '".join(map(str, x)) + "']") + def get_equip_item_name_from_item_code(lookup_df: pd.DataFrame, equip_item_code: str) -> int: + """Helper function to provide the equip_item_code (an int) when provided with the equip_item_name of the item""" + return str(pd.unique(lookup_df.loc[lookup_df["Equip_Code"] == equip_item_code, "Equip_Item"])[0]) + + def lists_of_equip_item_codes_to_strings_of_list_of_equip_item_names(list_of_equip_item_codes_col): + return list_of_equip_item_codes_col.apply( + lambda x: + str(sorted([get_equip_item_name_from_item_code(equip_resource_items_pkgs_df, item_code) for item_code in x])) + ) def strings_of_list_to_lists_of_strings(strings_of_list_col): lists_of_strings_col = strings_of_list_col.apply(lambda x: x.strip('][').split("'")) @@ -152,7 +165,7 @@ def strings_of_list_to_lists_of_strings(strings_of_list_col): # Make values in 'appt_footprint', 'beddays_footprint', and 'equipment' columns to be string df_col['appt_footprint'] = details_col_to_str(df_col['appt_footprint']) df_col['beddays_footprint'] = details_col_to_str(df_col['beddays_footprint']) - df_col['equipment'] = lists_of_strings_to_strings_of_list(df_col['equipment']) + df_col['equipment'] = lists_of_equip_item_codes_to_strings_of_list_of_equip_item_names(df_col['equipment']) df_col = (df_col.droplevel(level=1) .set_index(['module_name', 'event_name', 'treatment_id', 'facility_level', 'appt_footprint', 'beddays_footprint', 'equipment'], append=True)) @@ -199,8 +212,7 @@ def strings_of_list_to_lists_of_strings(strings_of_list_col): ).sum() # Remove rows with no equipment used - equipment_counts_by_time_and_requested_details.drop("['']", level='equipment', axis=0, inplace=True) - # Split the equipment by an item per row + equipment_counts_by_time_and_requested_details.drop("[]", level='equipment', axis=0, inplace=True) equipment_counts_by_time_and_requested_details['equipment'] = \ equipment_counts_by_time_and_requested_details.index.get_level_values('equipment') equipment_counts_by_time_and_requested_details.index = \ @@ -230,4 +242,5 @@ def strings_of_list_to_lists_of_strings(strings_of_list_col): results_folder=args.results_folder, output_folder=args.results_folder, ) -# NB. Edit run configuration, the Parameters: "./outputs/sejjej5@ucl.ac.uk/long_run_all_diseases-2023-09-04T233551Z" +# NB. Edit run configuration, the Parameters: +# "./outputs/sejjej5@ucl.ac.uk/equip_jobs/long_run_all_diseases-2023-09-04T233551Z" From a281e56c204bd95588c2a7c8e60bcb30f15e3be2 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Fri, 19 Jan 2024 13:58:10 +0000 Subject: [PATCH 050/118] brc: change Andrew suggested --- src/tlo/methods/breast_cancer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tlo/methods/breast_cancer.py b/src/tlo/methods/breast_cancer.py index 2529911d84..47f583ff64 100644 --- a/src/tlo/methods/breast_cancer.py +++ b/src/tlo/methods/breast_cancer.py @@ -663,7 +663,7 @@ def apply(self, person_id, squeeze_factor): if not pd.isnull(df.at[person_id, "brc_date_diagnosis"]): return hs.get_blank_appt_footprint() - df.brc_breast_lump_discernible_investigated = True + df.at[person_id, 'brc_breast_lump_discernible_investigated'] = True # Use a biopsy to diagnose whether the person has breast Cancer: # todo: request consumables needed for this From 1787ffe5bf8549a5e3f7dc82469124873bb219e2 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Fri, 19 Jan 2024 14:53:41 +0000 Subject: [PATCH 051/118] hs: updates for better readability; rm unused code --- src/tlo/methods/healthsystem.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 0411bdbe9a..562d268e6f 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -6,7 +6,7 @@ from collections.abc import Iterable from itertools import repeat from pathlib import Path -from typing import Dict, List, NamedTuple, Optional, Tuple, Union +from typing import Dict, List, NamedTuple, Optional, Tuple, Union, Set import numpy as np import pandas as pd @@ -347,7 +347,7 @@ def get_equip_item_code_from_item_name(self, lookup_df: pd.DataFrame, equip_item """Helper function to provide the equip_item_code (an int) when provided with the equip_item_name of the item""" return int(pd.unique(lookup_df.loc[lookup_df["Equip_Item"] == equip_item_name, "Equip_Code"])[0]) - def set_equipment_essential_to_run_event(self, set_of_equip): + def set_equipment_essential_to_run_event(self, set_of_equip: Set[str]) -> None: """Helper function to set essential equipment. Should be passed a set of equipment items names (strings) or an empty set. @@ -361,7 +361,7 @@ def set_equipment_essential_to_run_event(self, set_of_equip): self.ESSENTIAL_EQUIPMENT = set_of_equip - def add_equipment(self, set_of_equip): + def add_equipment(self, set_of_equip: Set[str]) -> None: """Helper function to update equipment. Should be passed a set of equipment item names (strings). @@ -375,12 +375,13 @@ def add_equipment(self, set_of_equip): ) # from the set of equip item names create a set of item codes # this function is calling parameters from this - equip_codes = set(self.get_equip_item_code_from_item_name( - self.sim.modules['HealthSystem'].parameters['equip_item_and_package_code_lookups'], item_name - ) for item_name in set_of_equip + equip_codes = set( + self.get_equip_item_code_from_item_name( + self.sim.modules['HealthSystem'].parameters['equip_item_and_package_code_lookups'], + item_name + ) for item_name in set_of_equip ) self.EQUIPMENT.update(equip_codes) - self.EQUIPMENT.discard('') def initialise(self): """Initialise the HSI: From f9616405b542775aa27b5d0dd8b54f368a71bac1 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Fri, 19 Jan 2024 15:59:53 +0000 Subject: [PATCH 052/118] hs: PEP8 --- src/tlo/methods/healthsystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 562d268e6f..0d4807eab0 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -6,7 +6,7 @@ from collections.abc import Iterable from itertools import repeat from pathlib import Path -from typing import Dict, List, NamedTuple, Optional, Tuple, Union, Set +from typing import Dict, List, NamedTuple, Optional, Set, Tuple, Union import numpy as np import pandas as pd From 12142b0d887ee53b7c78bb09fa59caf46776fbd8 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Fri, 19 Jan 2024 22:45:00 +0000 Subject: [PATCH 053/118] hs: get_equip_item_code_from_item_name fnc updated; ESS.EQUIP as codes --- src/tlo/methods/healthsystem.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 0d4807eab0..946bb32643 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -343,8 +343,9 @@ def make_appt_footprint(self, dict_of_appts): "values" ) - def get_equip_item_code_from_item_name(self, lookup_df: pd.DataFrame, equip_item_name: str) -> int: + def get_equip_item_code_from_item_name(self, equip_item_name: str) -> int: """Helper function to provide the equip_item_code (an int) when provided with the equip_item_name of the item""" + lookup_df = self.sim.modules['HealthSystem'].parameters['equip_item_and_package_code_lookups'] return int(pd.unique(lookup_df.loc[lookup_df["Equip_Item"] == equip_item_name, "Equip_Code"])[0]) def set_equipment_essential_to_run_event(self, set_of_equip: Set[str]) -> None: @@ -359,7 +360,11 @@ def set_equipment_essential_to_run_event(self, set_of_equip: Set[str]) -> None: "equipment item names from ResourceFile_Equipment.csv." ) - self.ESSENTIAL_EQUIPMENT = set_of_equip + if set_of_equip not in [set(), None, {''}]: + equip_codes = set(self.get_equip_item_code_from_item_name(item_name) for item_name in set_of_equip) + self.ESSENTIAL_EQUIPMENT = equip_codes + else: + self.ESSENTIAL_EQUIPMENT = set() def add_equipment(self, set_of_equip: Set[str]) -> None: """Helper function to update equipment. @@ -375,12 +380,7 @@ def add_equipment(self, set_of_equip: Set[str]) -> None: ) # from the set of equip item names create a set of item codes # this function is calling parameters from this - equip_codes = set( - self.get_equip_item_code_from_item_name( - self.sim.modules['HealthSystem'].parameters['equip_item_and_package_code_lookups'], - item_name - ) for item_name in set_of_equip - ) + equip_codes = set(self.get_equip_item_code_from_item_name(item_name) for item_name in set_of_equip) self.EQUIPMENT.update(equip_codes) def initialise(self): From 36f0523f0e07226a7d46f69b496560ae675c7fd0 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Sun, 21 Jan 2024 21:58:17 +0000 Subject: [PATCH 054/118] equip_catalogue: add item codes to catalogue by requested details (1 item per row) --- .../equipment/equipment_catalogue.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/scripts/healthsystem/equipment/equipment_catalogue.py b/src/scripts/healthsystem/equipment/equipment_catalogue.py index aa4552e536..4dcd533903 100644 --- a/src/scripts/healthsystem/equipment/equipment_catalogue.py +++ b/src/scripts/healthsystem/equipment/equipment_catalogue.py @@ -125,14 +125,20 @@ def create_equipment_catalogues(results_folder: Path, output_folder: Path): def details_col_to_str(details_col): return details_col.apply(lambda x: ', '.join(map(str, x))) - def get_equip_item_name_from_item_code(lookup_df: pd.DataFrame, equip_item_code: str) -> int: - """Helper function to provide the equip_item_code (an int) when provided with the equip_item_name of the item""" + def get_equip_item_name_from_item_code(equip_item_code: int) -> str: + """Helper function to provide the equip item name (a string) when provided with the equip_item_code (an int).""" + lookup_df = equip_resource_items_pkgs_df return str(pd.unique(lookup_df.loc[lookup_df["Equip_Code"] == equip_item_code, "Equip_Item"])[0]) + def get_equip_item_code_from_item_name(equip_item_name: str) -> int: + """Helper function to provide the equip item code (an int) when provided with the equip_item_name (a string)""" + lookup_df = equip_resource_items_pkgs_df + return int(pd.unique(lookup_df.loc[lookup_df["Equip_Item"] == equip_item_name, "Equip_Code"])[0]) + def lists_of_equip_item_codes_to_strings_of_list_of_equip_item_names(list_of_equip_item_codes_col): return list_of_equip_item_codes_col.apply( lambda x: - str(sorted([get_equip_item_name_from_item_code(equip_resource_items_pkgs_df, item_code) for item_code in x])) + str(sorted([get_equip_item_name_from_item_code(item_code) for item_code in x])) ) def strings_of_list_to_lists_of_strings(strings_of_list_col): @@ -221,7 +227,10 @@ def strings_of_list_to_lists_of_strings(strings_of_list_col): equipment_counts_by_time_and_requested_details['equipment'] ) exploded_df = equipment_counts_by_time_and_requested_details.explode('equipment') - exploded_df = exploded_df.set_index(['equipment'], append=True) + # Add column with equip item code + exploded_df['equip_code'] = exploded_df['equipment'].apply(lambda x: get_equip_item_code_from_item_name(x)) + exploded_df = exploded_df.set_index(['equipment', 'equip_code'], append=True) + # Sum values with the same multi-index exploded_df = exploded_df.groupby(level=exploded_df.index.names).sum() From b40f9558567029fbf40ff278922168cd1deef35c Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Sun, 21 Jan 2024 18:47:29 +0000 Subject: [PATCH 055/118] hs: allow adding equip by pkg name(s) --- src/tlo/methods/healthsystem.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 946bb32643..1c27fcee75 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -348,6 +348,12 @@ def get_equip_item_code_from_item_name(self, equip_item_name: str) -> int: lookup_df = self.sim.modules['HealthSystem'].parameters['equip_item_and_package_code_lookups'] return int(pd.unique(lookup_df.loc[lookup_df["Equip_Item"] == equip_item_name, "Equip_Code"])[0]) + def get_equip_item_codes_from_pkg_name(self, equip_pkg_name: str) -> Set[int]: + """Helper function to provide the equip_item_codes (a set of ints) when provided with the equip_pkg_name of the + equipment package""" + lookup_df = self.sim.modules['HealthSystem'].parameters['equip_item_and_package_code_lookups'] + return set(lookup_df.loc[lookup_df["Equip_Pkg"] == equip_pkg_name, "Equip_Code"]) + def set_equipment_essential_to_run_event(self, set_of_equip: Set[str]) -> None: """Helper function to set essential equipment. @@ -378,11 +384,26 @@ def add_equipment(self, set_of_equip: Set[str]) -> None: "Argument to add_equipment should be a non-empty set of strings of " "equipment item names from ResourceFile_Equipment.csv." ) - # from the set of equip item names create a set of item codes - # this function is calling parameters from this + # from the set of equip item names create a set of equip item codes equip_codes = set(self.get_equip_item_code_from_item_name(item_name) for item_name in set_of_equip) self.EQUIPMENT.update(equip_codes) + def add_equipment_from_pkg(self, set_of_pkgs: Set[str]) -> None: + """Helper function to update equipment with equipment from pkg(s). + + Should be passed a set of equipment pkgs names (strings). + """ + # Update EQUIPMENT if the given set_of_pkgs in correct format, ie a non-empty set of strings + if not isinstance(set_of_pkgs, set) or any(not isinstance(item, str) for item in set_of_pkgs) or \ + (set_of_pkgs in [set(), None, {''}]): + raise ValueError( + "Argument to add_equipment_from_pkg should be a non-empty set of strings of " + "equipment pkg names from ResourceFile_Equipment.csv." + ) + # update EQUIPMENT with eqip item codes from equip pkgs with provided names + for pkg_name in set_of_pkgs: + self.EQUIPMENT.update(self.get_equip_item_codes_from_pkg_name(pkg_name)) + def initialise(self): """Initialise the HSI: * Set the facility_info From 9c8b6ad3655e326095ca469cf3dd178f632fcfff Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Sun, 21 Jan 2024 19:04:31 +0000 Subject: [PATCH 056/118] co & RF_Equip: an example of usage of equipment pkg --- .../infrastructure_and_equipment/ResourceFile_Equipment.csv | 4 ++-- src/tlo/methods/contraception.py | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv index f0a2b53c9f..90012472cb 100644 --- a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv +++ b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cd2c3558a4d30dfb6d913054513e1d6df1b91f4955b60371e762eff05ac99afd -size 32759 +oid sha256:8b5c09ef0800c69ed916edab8bf469ce83c47a09e1e75c7263a44c24f692ed3f +size 33197 diff --git a/src/tlo/methods/contraception.py b/src/tlo/methods/contraception.py index dab1151ed4..b672407c3b 100644 --- a/src/tlo/methods/contraception.py +++ b/src/tlo/methods/contraception.py @@ -1207,6 +1207,11 @@ def apply(self, person_id, squeeze_factor): self.add_equipment({ 'Cusco’s/ bivalved Speculum (small, medium, large)', 'Lamp, Anglepoise' }) + self.add_equipment_from_pkg({ + 'Minor Surgery' + }) + # TODO: this is just an example - update once figured out what we want in the pkgs + # (! Update also the RF_Equipment accordingly !) elif _new_contraceptive == 'IUD': self.add_equipment({ 'Cusco’s/ bivalved Speculum (small, medium, large)', 'Sponge Holding Forceps' From 0552e48cb9310692fd95c9e0d884f3e70a7b9944 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Wed, 24 Jan 2024 15:32:31 +0000 Subject: [PATCH 057/118] utils: use pandas fnc (instead of make one) --- src/tlo/analysis/utils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/tlo/analysis/utils.py b/src/tlo/analysis/utils.py index a741eefee3..53b755a7cf 100644 --- a/src/tlo/analysis/utils.py +++ b/src/tlo/analysis/utils.py @@ -18,6 +18,7 @@ import numpy as np import pandas as pd import squarify +from pandas.api.types import is_numeric_dtype from tlo import Date, Simulation, logging, util from tlo.logging.reader import LogData @@ -282,9 +283,6 @@ def generate_series(dataframe: pd.DataFrame) -> pd.Series: # get number of draws and numbers of runs info = get_scenario_info(results_folder) - def is_number(element): - return isinstance(element, (int, float)) - # Collect results from each draw/run res = dict() for draw in range(info['number_of_draws']): @@ -296,7 +294,7 @@ def is_number(element): df: pd.DataFrame = load_pickled_dataframes(results_folder, draw, run, module)[module][key] output_from_eval: pd.Series = generate_series(df) assert pd.Series == type(output_from_eval), 'Custom command does not generate a pd.Series' - if output_from_eval.apply(is_number).all(): + if is_numeric_dtype(output_from_eval): res[draw_run] = output_from_eval * get_multiplier(draw, run) else: res[draw_run] = output_from_eval From 0106e20b6cb6661ee4d41120d40b123171067000 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Wed, 24 Jan 2024 22:47:41 +0000 Subject: [PATCH 058/118] hs: ESS_EQUIP as HSI_Event's attribute; if settings of ESS_EQUIP forgotten -> set to empty & warn --- src/tlo/methods/healthsystem.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 1c27fcee75..68a901017b 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -183,12 +183,14 @@ def __init__(self, module, *args, **kwargs): self._received_info_about_bed_days = None self.expected_time_requests = {} self.facility_info = None + self.ESSENTIAL_EQUIPMENT = None # self.set_equipment_essential_to_run_event({''}) # HSI needs this attribute, but it is not defined in the Base # class to allow verification of its existence as a test for # each HSI event, showing that equipment setup was thought # through for the event. self.EQUIPMENT = set() - + self._hsi_event_names_missing_ess_equip = set() # The names of hsi events for which the settings of + # essential equipment is missing. @property def bed_days_allocated_to_this_event(self): if self._received_info_about_bed_days is None: @@ -435,6 +437,11 @@ def initialise(self): # Do checks _ = self._check_if_appt_footprint_can_run() + # Set essential equip to empty set if not exists and warn about missing settings + if self.ESSENTIAL_EQUIPMENT is None: + self.set_equipment_essential_to_run_event({''}) + self._hsi_event_names_missing_ess_equip.update(self.__class__.__name__) + def _check_if_appt_footprint_can_run(self): """Check that event (if individual level) is able to run with this configuration of officers (i.e. check that this does not demand officers that are _never_ available), and issue warning if not.""" @@ -469,6 +476,18 @@ def as_namedtuple( equipment=(tuple(sorted(self.EQUIPMENT))) ) + def on_simulation_end(self): + """Do tasks at the end of the simulation: Raise warning and enter to log the set of hsi event names which were + initialised but the settings of essential equipment is missing.""" + if self._hsi_event_names_missing_ess_equip: + warnings.warn(UserWarning(f"The HSI event names which were initialised but the settings of essential" + f"equipment is missing:/n" + f"{self._hsi_event_names_missing_ess_equip}")) + logger.info( + key="hsi_event_names_missing_ess_equip", + data={"event_names": self._hsi_event_names_missing_ess_equip} + ) + class HSIEventWrapper(Event): """This is wrapper that contains an HSI event. From 455f4e0943acc50078614fc7dccaa699de2263ae Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Wed, 24 Jan 2024 23:58:52 +0000 Subject: [PATCH 059/118] hs: fixed saving _hsi_event_names_missing_ess_equip --- src/tlo/methods/healthsystem.py | 34 ++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 68a901017b..da17f0b89e 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -189,8 +189,7 @@ def __init__(self, module, *args, **kwargs): # each HSI event, showing that equipment setup was thought # through for the event. self.EQUIPMENT = set() - self._hsi_event_names_missing_ess_equip = set() # The names of hsi events for which the settings of - # essential equipment is missing. + @property def bed_days_allocated_to_this_event(self): if self._received_info_about_bed_days is None: @@ -440,7 +439,7 @@ def initialise(self): # Set essential equip to empty set if not exists and warn about missing settings if self.ESSENTIAL_EQUIPMENT is None: self.set_equipment_essential_to_run_event({''}) - self._hsi_event_names_missing_ess_equip.update(self.__class__.__name__) + self.sim.modules['HealthSystem']._hsi_event_names_missing_ess_equip.update(self.__class__.__name__) def _check_if_appt_footprint_can_run(self): """Check that event (if individual level) is able to run with this configuration of officers (i.e. check that @@ -476,19 +475,6 @@ def as_namedtuple( equipment=(tuple(sorted(self.EQUIPMENT))) ) - def on_simulation_end(self): - """Do tasks at the end of the simulation: Raise warning and enter to log the set of hsi event names which were - initialised but the settings of essential equipment is missing.""" - if self._hsi_event_names_missing_ess_equip: - warnings.warn(UserWarning(f"The HSI event names which were initialised but the settings of essential" - f"equipment is missing:/n" - f"{self._hsi_event_names_missing_ess_equip}")) - logger.info( - key="hsi_event_names_missing_ess_equip", - data={"event_names": self._hsi_event_names_missing_ess_equip} - ) - - class HSIEventWrapper(Event): """This is wrapper that contains an HSI event. @@ -858,6 +844,9 @@ def __init__( "'year', 'simulation' or None." ) + self._hsi_event_names_missing_ess_equip = set() # The names of HSI events for which the settings of essential + # equipment is missing. + def read_parameters(self, data_folder): path_to_resourcefiles_for_healthsystem = Path(self.resourcefilepath) / 'healthsystem' @@ -1017,7 +1006,9 @@ def on_birth(self, mother_id, child_id): self.bed_days.on_birth(self.sim.population.props, mother_id, child_id) def on_simulation_end(self): - """Put out to the log the information from the tracker of the last day of the simulation""" + """Put out to the log the information from the tracker of the last day of the simulation. + Raise warning and enter to log the set of hsi event names which were initialised but the settings of essential + equipment is missing.""" self.bed_days.on_simulation_end() self.consumables.on_simulation_end() if self._hsi_event_count_log_period == "simulation": @@ -1043,6 +1034,15 @@ def on_simulation_end(self): } ) + if self._hsi_event_names_missing_ess_equip: + warnings.warn(UserWarning(f"The HSI event names which were initialised but the settings of essential" + f"equipment is missing:/n" + f"{self._hsi_event_names_missing_ess_equip}")) + logger_summary.info( + key="hsi_event_names_missing_ess_equip", + data={"event_names": self._hsi_event_names_missing_ess_equip} + ) + def setup_priority_policy(self): # Determine name of policy to be considered **at the start of the simulation**. From 503036dc65a5d3f9d2c8fcebe51f0c07acb4a7f7 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Mon, 29 Jan 2024 17:36:24 +0000 Subject: [PATCH 060/118] hs: fixed updating _hsi_event_names_missing_ess_equip --- src/tlo/methods/healthsystem.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index da17f0b89e..63570701bd 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -439,7 +439,7 @@ def initialise(self): # Set essential equip to empty set if not exists and warn about missing settings if self.ESSENTIAL_EQUIPMENT is None: self.set_equipment_essential_to_run_event({''}) - self.sim.modules['HealthSystem']._hsi_event_names_missing_ess_equip.update(self.__class__.__name__) + self.sim.modules['HealthSystem']._hsi_event_names_missing_ess_equip.update({self.__class__.__name__}) def _check_if_appt_footprint_can_run(self): """Check that event (if individual level) is able to run with this configuration of officers (i.e. check that @@ -1035,7 +1035,7 @@ def on_simulation_end(self): ) if self._hsi_event_names_missing_ess_equip: - warnings.warn(UserWarning(f"The HSI event names which were initialised but the settings of essential" + warnings.warn(UserWarning(f"The HSI event names which were initialised but the settings of essential " f"equipment is missing:/n" f"{self._hsi_event_names_missing_ess_equip}")) logger_summary.info( From 8ea0b27e2f65e4c7bcc4aefa7d84b379900eda81 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Mon, 29 Jan 2024 18:36:20 +0000 Subject: [PATCH 061/118] hs: TODO smt odd going on with hsi_event_names_missing_ess_equip warning --- src/tlo/methods/healthsystem.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 63570701bd..1ca98dfe70 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -1042,6 +1042,10 @@ def on_simulation_end(self): key="hsi_event_names_missing_ess_equip", data={"event_names": self._hsi_event_names_missing_ess_equip} ) + # TODO: smt odd is going on, some hsi events were logged, according to my equipment_catalogue script, + # for which the essential equipment is not define, but they are not included in this warning. + # E.g. HSI_BladderCancer_Investigation_Following_Blood_Urine, HSI_BladderCancer_StartTreatment, + # HSI_BreastCancer_Investigation_Following_breast_lump_discernible, ... def setup_priority_policy(self): From 7ad13cc5391f3bc703b078c48e374471a6ba04b5 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Mon, 29 Jan 2024 18:37:10 +0000 Subject: [PATCH 062/118] hs: sort hsi_event_names_missing_ess_equip warning --- src/tlo/methods/healthsystem.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 1ca98dfe70..4bb0fc3a26 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -1035,12 +1035,13 @@ def on_simulation_end(self): ) if self._hsi_event_names_missing_ess_equip: + hsi_event_names_missing_ess_equip = sorted(self._hsi_event_names_missing_ess_equip) warnings.warn(UserWarning(f"The HSI event names which were initialised but the settings of essential " f"equipment is missing:/n" - f"{self._hsi_event_names_missing_ess_equip}")) + f"{hsi_event_names_missing_ess_equip}")) logger_summary.info( key="hsi_event_names_missing_ess_equip", - data={"event_names": self._hsi_event_names_missing_ess_equip} + data={"event_names": hsi_event_names_missing_ess_equip} ) # TODO: smt odd is going on, some hsi events were logged, according to my equipment_catalogue script, # for which the essential equipment is not define, but they are not included in this warning. From 39a5d21faeda87837d6c5e8211af365d8b7e2356 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Mon, 29 Jan 2024 18:38:06 +0000 Subject: [PATCH 063/118] hs: equip_item_and_package_code_lookups renamed to equip_item_and_package_lookups --- src/tlo/methods/healthsystem.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 4bb0fc3a26..09521ce712 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -346,13 +346,13 @@ def make_appt_footprint(self, dict_of_appts): def get_equip_item_code_from_item_name(self, equip_item_name: str) -> int: """Helper function to provide the equip_item_code (an int) when provided with the equip_item_name of the item""" - lookup_df = self.sim.modules['HealthSystem'].parameters['equip_item_and_package_code_lookups'] + lookup_df = self.sim.modules['HealthSystem'].parameters['equip_item_and_package_lookups'] return int(pd.unique(lookup_df.loc[lookup_df["Equip_Item"] == equip_item_name, "Equip_Code"])[0]) def get_equip_item_codes_from_pkg_name(self, equip_pkg_name: str) -> Set[int]: """Helper function to provide the equip_item_codes (a set of ints) when provided with the equip_pkg_name of the equipment package""" - lookup_df = self.sim.modules['HealthSystem'].parameters['equip_item_and_package_code_lookups'] + lookup_df = self.sim.modules['HealthSystem'].parameters['equip_item_and_package_lookups'] return set(lookup_df.loc[lookup_df["Equip_Pkg"] == equip_pkg_name, "Equip_Code"]) def set_equipment_essential_to_run_event(self, set_of_equip: Set[str]) -> None: @@ -597,7 +597,7 @@ class HealthSystem(Module): "Availability of beds. If 'default' then use the availability specified in the ResourceFile; if " "'none', then let no beds be ever be available; if 'all', then all beds are always available. NB. This " "parameter is over-ridden if an argument is provided to the module initialiser."), - 'equip_item_and_package_code_lookups': Parameter( + 'equip_item_and_package_lookups': Parameter( Types.DATA_FRAME, "Items based on the the HSSP III 1K Equipment Costing (SEL Costing Sheet): " "https://www.health.gov.mw/download/hssp-iii/, packages created in consultation with clinicians."), @@ -891,7 +891,7 @@ def read_parameters(self, data_folder): path_to_resourcefiles_for_healthsystem / 'infrastructure_and_equipment' / 'ResourceFile_Bed_Capacity.csv') # Read in ResourceFile_Equipment - self.parameters['equip_item_and_package_code_lookups'] = pd.read_csv( + self.parameters['equip_item_and_package_lookups'] = pd.read_csv( path_to_resourcefiles_for_healthsystem / 'infrastructure_and_equipment' / 'ResourceFile_Equipment.csv') # Data on the priority of each Treatment_ID that should be adopted in the queueing system according to different From 9dcf46bf8d8748f6d150b1f4853de8645c9f9974 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Mon, 29 Jan 2024 22:07:24 +0000 Subject: [PATCH 064/118] hs: ignore_unknown_equip_names --- src/tlo/methods/healthsystem.py | 116 +++++++++++++++++++++++++++----- 1 file changed, 98 insertions(+), 18 deletions(-) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 09521ce712..2b845a4b82 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -355,6 +355,45 @@ def get_equip_item_codes_from_pkg_name(self, equip_pkg_name: str) -> Set[int]: lookup_df = self.sim.modules['HealthSystem'].parameters['equip_item_and_package_lookups'] return set(lookup_df.loc[lookup_df["Equip_Pkg"] == equip_pkg_name, "Equip_Code"]) + def ignore_unknown_equip_names(self, set_of_names: Set[str], type_in_set: str) -> Set[str]: + """Helper function to check if the equipment item or pkg names (depending on type_in_set: 'item' or 'pkg') from + the provided set are in the RF_Equipment. If they are not, they are added to a set to be warned about at the end + of the simulation. + + Only known (item or pkg) names are returned.""" + if set_of_names in [set(), None, {''}]: + return set() + + def add_unknown_names_to_dict(unknown_names_to_add: Set[str], dict_to_be_added_to: Dict) -> Dict: + if self.__class__.__name__ not in dict_to_be_added_to.keys(): + dict_to_be_added_to.update( + {self.__class__.__name__: unknown_names_to_add} + ) + else: + dict_to_be_added_to[self.__class__.__name__].update( + unknown_names_to_add + ) + return dict_to_be_added_to + + lookup_df = self.sim.modules['HealthSystem'].parameters['equip_item_and_package_lookups'] + if type_in_set == "item": + unknown_names = set_of_names.difference(set(lookup_df["Equip_Item"])) + if unknown_names: + self.sim.modules['HealthSystem']._equip_items_missing_in_RF = \ + add_unknown_names_to_dict( + unknown_names, self.sim.modules['HealthSystem']._equip_items_missing_in_RF + ) + + elif type_in_set == "pkg": + unknown_names = set_of_names.difference(set(lookup_df["Equip_Pkg"])) + if unknown_names: + self.sim.modules['HealthSystem']._equip_pkgs_missing_in_RF = \ + add_unknown_names_to_dict( + unknown_names, self.sim.modules['HealthSystem']._equip_pkgs_missing_in_RF + ) + + return set_of_names.difference(unknown_names) + def set_equipment_essential_to_run_event(self, set_of_equip: Set[str]) -> None: """Helper function to set essential equipment. @@ -367,7 +406,8 @@ def set_equipment_essential_to_run_event(self, set_of_equip: Set[str]) -> None: "equipment item names from ResourceFile_Equipment.csv." ) - if set_of_equip not in [set(), None, {''}]: + set_of_equip = self.ignore_unknown_equip_names(set_of_equip, "item") + if set_of_equip: equip_codes = set(self.get_equip_item_code_from_item_name(item_name) for item_name in set_of_equip) self.ESSENTIAL_EQUIPMENT = equip_codes else: @@ -385,9 +425,12 @@ def add_equipment(self, set_of_equip: Set[str]) -> None: "Argument to add_equipment should be a non-empty set of strings of " "equipment item names from ResourceFile_Equipment.csv." ) - # from the set of equip item names create a set of equip item codes - equip_codes = set(self.get_equip_item_code_from_item_name(item_name) for item_name in set_of_equip) - self.EQUIPMENT.update(equip_codes) + # from the set of equip item names create a set of equip item codes, ignore unknown equip names + # (ie not included in RF_Equipment) + set_of_equip = self.ignore_unknown_equip_names(set_of_equip, "item") + if set_of_equip: + equip_codes = set(self.get_equip_item_code_from_item_name(item_name) for item_name in set_of_equip) + self.EQUIPMENT.update(equip_codes) def add_equipment_from_pkg(self, set_of_pkgs: Set[str]) -> None: """Helper function to update equipment with equipment from pkg(s). @@ -401,9 +444,12 @@ def add_equipment_from_pkg(self, set_of_pkgs: Set[str]) -> None: "Argument to add_equipment_from_pkg should be a non-empty set of strings of " "equipment pkg names from ResourceFile_Equipment.csv." ) - # update EQUIPMENT with eqip item codes from equip pkgs with provided names - for pkg_name in set_of_pkgs: - self.EQUIPMENT.update(self.get_equip_item_codes_from_pkg_name(pkg_name)) + # update EQUIPMENT with eqip item codes from equip pkgs with provided names, ignore unknown equip names + # (ie not included in RF_Equipment) + set_of_pkgs = self.ignore_unknown_equip_names(set_of_pkgs, "pkg") + if set_of_pkgs: + for pkg_name in set_of_pkgs: + self.EQUIPMENT.update(self.get_equip_item_codes_from_pkg_name(pkg_name)) def initialise(self): """Initialise the HSI: @@ -846,6 +892,10 @@ def __init__( self._hsi_event_names_missing_ess_equip = set() # The names of HSI events for which the settings of essential # equipment is missing. + self._equip_items_missing_in_RF = dict() # The equipment item names called for an HSI event, but are missing in + # the RF_Equipment. + self._equip_pkgs_missing_in_RF = dict() # The equipment pkg names called for an HSI event, but are missing in + # the RF_Equipment. def read_parameters(self, data_folder): @@ -1034,19 +1084,49 @@ def on_simulation_end(self): } ) - if self._hsi_event_names_missing_ess_equip: - hsi_event_names_missing_ess_equip = sorted(self._hsi_event_names_missing_ess_equip) - warnings.warn(UserWarning(f"The HSI event names which were initialised but the settings of essential " - f"equipment is missing:/n" - f"{hsi_event_names_missing_ess_equip}")) + if self._hsi_event_names_missing_ess_equip: + hsi_event_names_missing_ess_equip = sorted(self._hsi_event_names_missing_ess_equip) + warnings.warn(UserWarning(f"The HSI event names which were initialised but the settings of essential " + f"equipment is missing:/n" + f"{hsi_event_names_missing_ess_equip}")) + logger_summary.info( + key="hsi_event_names_missing_ess_equip", + data={"event_names": hsi_event_names_missing_ess_equip} + ) + # TODO: smt odd is going on, some hsi events were logged, according to my equipment_catalogue script, + # for which the essential equipment is not define, but they are not included in this warning. + # E.g. HSI_BladderCancer_Investigation_Following_Blood_Urine, HSI_BladderCancer_StartTreatment, + # HSI_BreastCancer_Investigation_Following_breast_lump_discernible, ... + + def sort_dict_for_print(dict_to_sort: Dict) -> Dict: + sorted_list = sorted(dict_to_sort.items()) + sorted_dict = {} + for key, value in sorted_list: + sorted_dict[key] = sorted(value) + return sorted_dict + + if self._equip_items_missing_in_RF: + sorted_equip_items_missing_in_RF = sort_dict_for_print(self._equip_items_missing_in_RF) + warnings.warn(UserWarning(f"The equipment item names called for an HSI event, but missing in the " + f"RF_Equipment:/n" + f"{sorted_equip_items_missing_in_RF}")) + + for _hsi_event_name, _item_names in sorted_equip_items_missing_in_RF.items(): + logger_summary.info( + key="equip_items_missing_in_RF", + data={_hsi_event_name: _item_names} + ) + + if self._equip_pkgs_missing_in_RF: + sorted_equip_pkgs_missing_in_RF = sort_dict_for_print(self._equip_pkgs_missing_in_RF) + warnings.warn(UserWarning(f"The equipment pkg names called for an HSI event, but missing in the " + f"RF_Equipment:/n" + f"{sorted_equip_pkgs_missing_in_RF}")) + for _hsi_event_name, _pkg_names in sorted_equip_pkgs_missing_in_RF.items(): logger_summary.info( - key="hsi_event_names_missing_ess_equip", - data={"event_names": hsi_event_names_missing_ess_equip} + key="equip_pkgs_missing_in_RF", + data={_hsi_event_name: _pkg_names} ) - # TODO: smt odd is going on, some hsi events were logged, according to my equipment_catalogue script, - # for which the essential equipment is not define, but they are not included in this warning. - # E.g. HSI_BladderCancer_Investigation_Following_Blood_Urine, HSI_BladderCancer_StartTreatment, - # HSI_BreastCancer_Investigation_Following_breast_lump_discernible, ... def setup_priority_policy(self): From 0bda4d0b8b9944c1785c1c97d666328feef1e328 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Tue, 30 Jan 2024 16:35:15 +0000 Subject: [PATCH 065/118] hs: warning messages shortened --- src/tlo/methods/healthsystem.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 2b845a4b82..178d97c3fc 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -1086,8 +1086,7 @@ def on_simulation_end(self): if self._hsi_event_names_missing_ess_equip: hsi_event_names_missing_ess_equip = sorted(self._hsi_event_names_missing_ess_equip) - warnings.warn(UserWarning(f"The HSI event names which were initialised but the settings of essential " - f"equipment is missing:/n" + warnings.warn(UserWarning(f"Missing settings of essential equipment for HSI events:/n" f"{hsi_event_names_missing_ess_equip}")) logger_summary.info( key="hsi_event_names_missing_ess_equip", @@ -1107,8 +1106,7 @@ def sort_dict_for_print(dict_to_sort: Dict) -> Dict: if self._equip_items_missing_in_RF: sorted_equip_items_missing_in_RF = sort_dict_for_print(self._equip_items_missing_in_RF) - warnings.warn(UserWarning(f"The equipment item names called for an HSI event, but missing in the " - f"RF_Equipment:/n" + warnings.warn(UserWarning(f"Equipment item names were not recognised:/n" f"{sorted_equip_items_missing_in_RF}")) for _hsi_event_name, _item_names in sorted_equip_items_missing_in_RF.items(): @@ -1119,8 +1117,7 @@ def sort_dict_for_print(dict_to_sort: Dict) -> Dict: if self._equip_pkgs_missing_in_RF: sorted_equip_pkgs_missing_in_RF = sort_dict_for_print(self._equip_pkgs_missing_in_RF) - warnings.warn(UserWarning(f"The equipment pkg names called for an HSI event, but missing in the " - f"RF_Equipment:/n" + warnings.warn(UserWarning(f"Equipment pkg names were not recognised:/n" f"{sorted_equip_pkgs_missing_in_RF}")) for _hsi_event_name, _pkg_names in sorted_equip_pkgs_missing_in_RF.items(): logger_summary.info( From 0d387a4f872830225d6417a40d7317c339b60553 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Thu, 21 Mar 2024 14:10:34 +0000 Subject: [PATCH 066/118] co: example of setting ess. equip based on condition --- src/tlo/methods/contraception.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/tlo/methods/contraception.py b/src/tlo/methods/contraception.py index b672407c3b..a8e0ac680d 100644 --- a/src/tlo/methods/contraception.py +++ b/src/tlo/methods/contraception.py @@ -1109,7 +1109,20 @@ def __init__(self, module, person_id, new_contraceptive): self.TREATMENT_ID = "Contraception_Routine" self.ACCEPTED_FACILITY_LEVEL = _facility_level - self.set_equipment_essential_to_run_event({''}) + + # Set essential equipment based on the contraception method + if new_contraceptive == 'female_sterilization': + self.set_equipment_essential_to_run_event({ + 'Cusco’s/ bivalved Speculum (small, medium, large)', 'Lamp, Anglepoise' + }) + # + 'Minor Surgery' pkg + # TODO: How to set pkg as essential? + elif new_contraceptive == 'IUD': + self.set_equipment_essential_to_run_event({ + 'Cusco’s/ bivalved Speculum (small, medium, large)', 'Sponge Holding Forceps' + }) + else: + self.set_equipment_essential_to_run_event({''}) @property def EXPECTED_APPT_FOOTPRINT(self): From b87781e45bfc6e9ed2a43e7057730ceb57f0b808 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Thu, 21 Mar 2024 14:11:09 +0000 Subject: [PATCH 067/118] co: comment updated; TODO added --- src/tlo/methods/contraception.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/tlo/methods/contraception.py b/src/tlo/methods/contraception.py index a8e0ac680d..581969c0b7 100644 --- a/src/tlo/methods/contraception.py +++ b/src/tlo/methods/contraception.py @@ -1159,6 +1159,7 @@ def apply(self, person_id, squeeze_factor): self.sim.population.props.at[person_id, "co_date_of_last_fp_appt"] = self.sim.date # Measure weight, height and BP even if contraception not administrated + # TODO: Always or only if it's not a rescheduled appt? self.add_equipment({ 'Weighing scale', 'Height Pole (Stadiometer)', 'Blood pressure machine' }) @@ -1215,7 +1216,7 @@ def apply(self, person_id, squeeze_factor): _new_contraceptive = self.new_contraceptive - # Update equipment if any needed for the method + # Add equipment if any used with the method if _new_contraceptive == 'female_sterilization': self.add_equipment({ 'Cusco’s/ bivalved Speculum (small, medium, large)', 'Lamp, Anglepoise' From 88632f6da3dc3efc20ac0ea8b588696102341ed0 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Thu, 21 Mar 2024 14:17:55 +0000 Subject: [PATCH 068/118] co, hs, RF_Equip, RF_HS_params, test_alri, test_co, test_hs: checking equip availability (before/within HSI); avail switcher; dummy probs by fac_level = 0.5; tests updated; TODOs added --- .../ResourceFile_HealthSystem_parameters.csv | 4 +- .../ResourceFile_Equipment.csv | 4 +- src/tlo/methods/contraception.py | 5 + src/tlo/methods/healthsystem.py | 107 ++++++-- tests/test_alri.py | 69 +++--- tests/test_contraception.py | 230 +++++++++++++++--- tests/test_healthsystem.py | 3 + 7 files changed, 327 insertions(+), 95 deletions(-) diff --git a/resources/healthsystem/ResourceFile_HealthSystem_parameters.csv b/resources/healthsystem/ResourceFile_HealthSystem_parameters.csv index 2828a9376c..ed560fb93e 100644 --- a/resources/healthsystem/ResourceFile_HealthSystem_parameters.csv +++ b/resources/healthsystem/ResourceFile_HealthSystem_parameters.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e36cbd225191f893f2de5f0f34adc251046af5ec58daaf7c86b09a6c83c1e31d -size 379 +oid sha256:b3666c302a8e0eb15fee119a6edd777ab21b602e8847b2bbd9357691c69e622a +size 406 diff --git a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv index 90012472cb..17c3f82ba7 100644 --- a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv +++ b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8b5c09ef0800c69ed916edab8bf469ce83c47a09e1e75c7263a44c24f692ed3f -size 33197 +oid sha256:0be4639d20a28222ecba14f04025c0270fd799e13195ccf76c937448c4c5e39a +size 42931 diff --git a/src/tlo/methods/contraception.py b/src/tlo/methods/contraception.py index 581969c0b7..33baabfa03 100644 --- a/src/tlo/methods/contraception.py +++ b/src/tlo/methods/contraception.py @@ -91,6 +91,7 @@ class Contraception(Module): 'max_number_of_runs_of_hsi_if_consumable_not_available': Parameter( Types.INT, "The maximum number of time an HSI can run (repeats occur if the consumables are not " "available)."), + # TODO: We don't have anything like this for equipment, should we? 'max_days_delay_between_decision_to_change_method_and_hsi_scheduled': Parameter( Types.INT, "The maximum delay (in days) between the decision for a contraceptive to change and the `topen` " @@ -1245,6 +1246,10 @@ def apply(self, person_id, squeeze_factor): # If the intended change was not possible due to non-available consumable, reschedule the appointment if (not co_administrated) and ( + # TODO: the max nmb of runs is set to 1000, so they will be coming back every day for 1000 consecutive days? + # -- but only 8 footprints for did_not_run event when switch to f. steril when consumables available but + # equipment not available in test_contraception (test_record_of_appt_footprint_for_switching_to_methods). + # What am I missing? ???? self._number_of_times_run < self.module.parameters['max_number_of_runs_of_hsi_if_consumable_not_available'] ): self.reschedule() diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 178d97c3fc..73aff18988 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -346,13 +346,13 @@ def make_appt_footprint(self, dict_of_appts): def get_equip_item_code_from_item_name(self, equip_item_name: str) -> int: """Helper function to provide the equip_item_code (an int) when provided with the equip_item_name of the item""" - lookup_df = self.sim.modules['HealthSystem'].parameters['equip_item_and_package_lookups'] + lookup_df = self.sim.modules['HealthSystem'].parameters['Equipment'] return int(pd.unique(lookup_df.loc[lookup_df["Equip_Item"] == equip_item_name, "Equip_Code"])[0]) def get_equip_item_codes_from_pkg_name(self, equip_pkg_name: str) -> Set[int]: """Helper function to provide the equip_item_codes (a set of ints) when provided with the equip_pkg_name of the equipment package""" - lookup_df = self.sim.modules['HealthSystem'].parameters['equip_item_and_package_lookups'] + lookup_df = self.sim.modules['HealthSystem'].parameters['Equipment'] return set(lookup_df.loc[lookup_df["Equip_Pkg"] == equip_pkg_name, "Equip_Code"]) def ignore_unknown_equip_names(self, set_of_names: Set[str], type_in_set: str) -> Set[str]: @@ -375,7 +375,7 @@ def add_unknown_names_to_dict(unknown_names_to_add: Set[str], dict_to_be_added_t ) return dict_to_be_added_to - lookup_df = self.sim.modules['HealthSystem'].parameters['equip_item_and_package_lookups'] + lookup_df = self.sim.modules['HealthSystem'].parameters['Equipment'] if type_in_set == "item": unknown_names = set_of_names.difference(set(lookup_df["Equip_Item"])) if unknown_names: @@ -391,7 +391,7 @@ def add_unknown_names_to_dict(unknown_names_to_add: Set[str], dict_to_be_added_t add_unknown_names_to_dict( unknown_names, self.sim.modules['HealthSystem']._equip_pkgs_missing_in_RF ) - + # TODO: What happens if all equip in set_of_names has unknown name? return set_of_names.difference(unknown_names) def set_equipment_essential_to_run_event(self, set_of_equip: Set[str]) -> None: @@ -451,6 +451,12 @@ def add_equipment_from_pkg(self, set_of_pkgs: Set[str]) -> None: for pkg_name in set_of_pkgs: self.EQUIPMENT.update(self.get_equip_item_codes_from_pkg_name(pkg_name)) + def get_essential_equip_availability(self, set_of_pkgs: Set[str]) -> bool: + # TODO: Or, should it be called set_essential_equip_and_get_availability to be more transparent about what the + # fnc does? + self.set_equipment_essential_to_run_event(set_of_pkgs) + return self.sim.modules['HealthSystem'].get_essential_equip_availability(self.ESSENTIAL_EQUIPMENT) + def initialise(self): """Initialise the HSI: * Set the facility_info @@ -643,9 +649,13 @@ class HealthSystem(Module): "Availability of beds. If 'default' then use the availability specified in the ResourceFile; if " "'none', then let no beds be ever be available; if 'all', then all beds are always available. NB. This " "parameter is over-ridden if an argument is provided to the module initialiser."), - 'equip_item_and_package_lookups': Parameter( - Types.DATA_FRAME, "Items based on the the HSSP III 1K Equipment Costing (SEL Costing Sheet): " - "https://www.health.gov.mw/download/hssp-iii/, packages created in consultation with clinicians."), + 'Equipment': Parameter( + Types.DATA_FRAME, "Data on equipment items, packages, and availability probabilities by facility level."), + 'equip_availability': Parameter( + Types.STRING, + "Availability of equipment. If 'default' then use the availability specified in the ResourceFile;" + " if 'none', then let no equipment ever be available; if 'all', then all equipment is always available. NB." + " This parameter is over-ridden if an argument is provided to the module initialiser."), # Service Availability 'Service_Availability': Parameter( @@ -719,6 +729,7 @@ def __init__( mode_appt_constraints: Optional[int] = None, cons_availability: Optional[str] = None, beds_availability: Optional[str] = None, + equip_availability: Optional[str] = None, randomise_queue: bool = True, ignore_priority: bool = False, policy_name: Optional[str] = None, @@ -743,6 +754,8 @@ def __init__( or 'none', requests for consumables are not logged. :param beds_availability: If 'default' then use the availability specified in the ResourceFile; if 'none', then let no beds be ever be available; if 'all', then all beds are always available. + :param equip_availability: If 'default' then use the availability specified in the ResourceFile; if 'none', then + let no equipment ever be available; if 'all', then all equipment is always available. :param randomise_queue ensure that the queue is not model-dependent, i.e. properly randomised for equal topen and priority :param ignore_priority: If ``True`` do not use the priority information in HSI @@ -835,13 +848,17 @@ def __init__( self.HSI_EVENT_QUEUE = [] self.hsi_event_queue_counter = 0 # Counter to help with the sorting in the heapq - # Store the argument provided for cons_availability + # Store the arguments provided for cons/beds/equip_availability assert cons_availability in (None, 'default', 'all', 'none') self.arg_cons_availability = cons_availability assert beds_availability in (None, 'default', 'all', 'none') self.arg_beds_availability = beds_availability + assert equip_availability in (None, 'default', 'all', 'none') + self.arg_equip_availability = equip_availability + self.equip_availability = 'all' # provided so that there is a default even before simulation is run + # `compute_squeeze_factor_to_district_level` is a Boolean indicating whether the computation of squeeze_factors # should be specific to each district (when `True`), or if the computation of squeeze_factors should be on the # basis that resources from all districts can be effectively "pooled" (when `False). @@ -941,7 +958,7 @@ def read_parameters(self, data_folder): path_to_resourcefiles_for_healthsystem / 'infrastructure_and_equipment' / 'ResourceFile_Bed_Capacity.csv') # Read in ResourceFile_Equipment - self.parameters['equip_item_and_package_lookups'] = pd.read_csv( + self.parameters['Equipment'] = pd.read_csv( path_to_resourcefiles_for_healthsystem / 'infrastructure_and_equipment' / 'ResourceFile_Equipment.csv') # Data on the priority of each Treatment_ID that should be adopted in the queueing system according to different @@ -958,7 +975,6 @@ def read_parameters(self, data_folder): sheet_name=None # all sheets read in ) - def pre_initialise_population(self): """Generate the accessory classes used by the HealthSystem and pass to them the data that has been read.""" @@ -994,6 +1010,9 @@ def pre_initialise_population(self): availability=self.get_cons_availability() ) + # Determine equip_availability + self.equip_availability = self.get_equip_availability() + self.tclose_overwrite = self.parameters['tclose_overwrite'] self.tclose_days_offset_overwrite = self.parameters['tclose_days_offset_overwrite'] @@ -1151,8 +1170,6 @@ def setup_priority_policy(self): def process_human_resources_files(self, use_funded_or_actual_staffing: str): """Create the data-structures needed from the information read into the parameters.""" - - # * Define Facility Levels self._facility_levels = set(self.parameters['Master_Facilities_List']['Facility_Level']) - {'5'} assert self._facility_levels == {'0', '1a', '1b', '2', '3', '4'} # todo soft code this? @@ -1457,6 +1474,42 @@ def get_beds_availability(self) -> str: return _beds_availability + def get_equip_availability(self) -> str: + """Returns equipment availability. (Should be equal to what is specified by the parameter, but overwrite with + what was provided in argument if an argument was specified -- provided for backward compatibility/debugging.)""" + + if self.arg_equip_availability is None: + _equip_availability = self.parameters['equip_availability'] + else: + _equip_availability = self.arg_equip_availability + + # Log the equip_availability + logger.info(key="message", + data=f"Running Health System With the Following Equipment Availability: " + f"{_equip_availability}" + ) + + return _equip_availability + + def get_equip_item_availability(self, equip_item_code: str) -> bool: + # TODO: update with implementation of essential equipment availability for the HSI event to run + # for now, always available + if equip_item_code in [243, 41]: # 243 = Pulse oximeter, 41 = 'Lamp, Anglepoise' + return False + return True # True of False + + def get_essential_equip_availability(self, essential_equip_set: Set[int]) -> bool: + if self.equip_availability == 'all': + # Always all equipment available + return True + elif self.equip_availability == 'default': + # True if all items of essential equipment available; False if any unavailable + return all(self.get_equip_item_availability(item_code) for item_code in essential_equip_set) + else: # self.equip_availability == 'none': + # True if no essential equipment requested, otherwise False as assumed no equipment available + # TODO: Should no equipment be logged then? + return not bool(essential_equip_set) + def schedule_to_call_never_ran_on_date(self, hsi_event: 'HSI_Event', tdate: datetime.datetime): """Function to schedule never_ran being called on a given date""" self.sim.schedule_event(HSIEventWrapper(hsi_event=hsi_event, run_hsi=False), tdate) @@ -2285,8 +2338,7 @@ def run_individual_level_events_in_mode_0_or_1(self, # Mode 0: All HSI Event run, with no squeeze # Mode 1: All HSI Events run with squeeze provided latter is not inf - ok_to_run = True - + ok_to_run = self.get_essential_equip_availability(event.ESSENTIAL_EQUIPMENT) if self.mode_appt_constraints == 1 and squeeze_factor == float('inf'): ok_to_run = False @@ -2624,18 +2676,25 @@ def process_events_mode_2(self, hold_over: List[HSIEventQueueItem]) -> None: # based on queue information, and we assume no squeeze ever takes place. squeeze_factor = 0. - # Check if any of the officers required have run out. - out_of_resources = False - for officer, call in original_call.items(): - # If any of the officers are not available, then out of resources - if officer not in set_capabilities_still_available: - out_of_resources = True + # Check if all essential equipment available and the officers required available. + ok_to_run = \ + self.module.sim.modules['HealthSystem'].get_essential_equip_availability( + next_event_tuple.hsi_event.ESSENTIAL_EQUIPMENT + ) # True if all essential equipment available + + if ok_to_run: + for officer, call in original_call.items(): + # If any of the officers are not available, we are out of resources, hence not ok_to_run + if officer not in set_capabilities_still_available: + ok_to_run = False + if not ok_to_run: + break # If officers still available, run event. Note: in current logic, a little # overtime is allowed to run last event of the day. This seems more realistic # than medical staff leaving earlier than # planned if seeing another patient would take them into overtime. - if out_of_resources: + if not ok_to_run: # Do not run, # Call did_not_run for the hsi_event @@ -2660,7 +2719,7 @@ def process_events_mode_2(self, hold_over: List[HSIEventQueueItem]) -> None: priority=_priority ) - # Have enough capabilities left to run event + # Have enough capabilities left to run event, ie ok_to_run else: # Notes-to-self: Shouldn't this be done after checking the footprint? # Compute the bed days that are allocated to this HSI and provide this @@ -3011,6 +3070,7 @@ class HealthSystemChangeParameters(Event, PopulationScopeEventMixin): * `capabilities_coefficient` * `cons_availability` * `beds_availability` + * `equip_availability` Note that no checking is done here on the suitability of values of each parameter.""" def __init__(self, module: HealthSystem, parameters: Dict): @@ -3037,6 +3097,9 @@ def apply(self, population): if 'beds_availability' in self._parameters: self.module.bed_days.availability = self._parameters['beds_availability'] + if 'equip_availability' in self._parameters: + self.module.equip_availability = self._parameters['equip_availability'] + class DynamicRescalingHRCapabilities(RegularEvent, PopulationScopeEventMixin): """ This event exists to scale the daily capabilities assumed at fixed time intervals""" diff --git a/tests/test_alri.py b/tests/test_alri.py index a98d2f277c..0fba5fea8d 100644 --- a/tests/test_alri.py +++ b/tests/test_alri.py @@ -54,7 +54,7 @@ def _get_person_id(df, age_bounds: tuple = (0.0, np.inf)) -> int: ].index[0] -def get_sim(tmpdir, seed, cons_available): +def get_sim(tmpdir, seed, cons_available, equip_available='all'): """Return simulation objection with Alri and other necessary modules registered.""" sim = Simulation( start_date=start_date, @@ -77,7 +77,8 @@ def get_sim(tmpdir, seed, cons_available): healthseekingbehaviour.HealthSeekingBehaviour(resourcefilepath=resourcefilepath), healthburden.HealthBurden(resourcefilepath=resourcefilepath), healthsystem.HealthSystem(resourcefilepath=resourcefilepath, - cons_availability=cons_available), + cons_availability=cons_available, + equip_availability=equip_available), alri.Alri(resourcefilepath=resourcefilepath, log_indivdual=0, do_checks=True), AlriPropertiesOfOtherModules(), ) @@ -85,10 +86,10 @@ def get_sim(tmpdir, seed, cons_available): @pytest.fixture -def sim_hs_all_consumables(tmpdir, seed): +def sim_hs_all_consumables_and_equipment(tmpdir, seed): """Return simulation objection with Alri and other necessary modules registered. All consumables available""" - return get_sim(tmpdir=tmpdir, seed=seed, cons_available='all') + return get_sim(tmpdir=tmpdir, seed=seed, cons_available='all', equip_available='all') @pytest.fixture @@ -127,17 +128,17 @@ def sim_hs_default_consumables(tmpdir, seed): return sim -def check_dtypes(sim_hs_all_consumables): - sim = sim_hs_all_consumables +def check_dtypes(sim_hs_all_consumables_and_equipment): + sim = sim_hs_all_consumables_and_equipment # Check types of columns df = sim.population.props orig = sim.population.new_row assert (df.dtypes == orig.dtypes).all() -def test_integrity_of_linear_models(sim_hs_all_consumables): +def test_integrity_of_linear_models(sim_hs_all_consumables_and_equipment): """Run the models to make sure that is specified correctly and can run.""" - sim = sim_hs_all_consumables + sim = sim_hs_all_consumables_and_equipment sim.make_initial_population(n=5000) alri_module = sim.modules['Alri'] df = sim.population.props @@ -322,21 +323,21 @@ def test_integrity_of_linear_models(sim_hs_all_consumables): assert isinstance(res, float) and (res is not None) and (0.0 <= res <= 1.0), f"Problem with: {kwargs=}" -def test_basic_run(sim_hs_all_consumables): +def test_basic_run(sim_hs_all_consumables_and_equipment): """Short run of the module using default parameters with check on dtypes""" - sim = sim_hs_all_consumables + sim = sim_hs_all_consumables_and_equipment dur = pd.DateOffset(months=1) popsize = 100 sim.make_initial_population(n=popsize) sim.simulate(end_date=start_date + dur) - check_dtypes(sim_hs_all_consumables) + check_dtypes(sim_hs_all_consumables_and_equipment) @pytest.mark.slow -def test_basic_run_lasting_two_years(sim_hs_all_consumables): +def test_basic_run_lasting_two_years(sim_hs_all_consumables_and_equipment): """Check logging results in a run of the model for two years, including HSI, with daily property config checking""" - sim = sim_hs_all_consumables + sim = sim_hs_all_consumables_and_equipment dur = pd.DateOffset(years=2) popsize = 5000 @@ -362,9 +363,9 @@ def test_basic_run_lasting_two_years(sim_hs_all_consumables): assert set(log_one_person.columns) == set(sim.modules['Alri'].PROPERTIES.keys()) -def test_alri_polling(sim_hs_all_consumables): +def test_alri_polling(sim_hs_all_consumables_and_equipment): """Check polling events leads to incident cases""" - sim = sim_hs_all_consumables + sim = sim_hs_all_consumables_and_equipment # get simulation object: popsize = 100 @@ -386,10 +387,10 @@ def test_alri_polling(sim_hs_all_consumables): assert len([q for q in sim.event_queue.queue if isinstance(q[3], AlriIncidentCase)]) > 0 -def test_nat_hist_recovery(sim_hs_all_consumables): +def test_nat_hist_recovery(sim_hs_all_consumables_and_equipment): """Check: Infection onset --> recovery""" - sim = sim_hs_all_consumables + sim = sim_hs_all_consumables_and_equipment popsize = 100 sim.make_initial_population(n=popsize) @@ -466,9 +467,9 @@ def __will_die_of_alri(**kwargs): assert 0 == sim.modules['Alri'].logging_event.trackers['cured_cases'].report_current_total() -def test_nat_hist_death(sim_hs_all_consumables): +def test_nat_hist_death(sim_hs_all_consumables_and_equipment): """Check: Infection onset --> death""" - sim = sim_hs_all_consumables + sim = sim_hs_all_consumables_and_equipment popsize = 100 sim.make_initial_population(n=popsize) @@ -523,10 +524,10 @@ def __will_die_of_alri(**kwargs): assert 0 == sim.modules['Alri'].logging_event.trackers['cured_cases'].report_current_total() -def test_nat_hist_cure_if_recovery_scheduled(sim_hs_all_consumables): +def test_nat_hist_cure_if_recovery_scheduled(sim_hs_all_consumables_and_equipment): """Show that if a cure event is run before when a person was going to recover naturally, it cause the episode to end earlier.""" - sim = sim_hs_all_consumables + sim = sim_hs_all_consumables_and_equipment popsize = 100 @@ -598,10 +599,10 @@ def death(**kwargs): assert 1 == sim.modules['Alri'].logging_event.trackers['cured_cases'].report_current_total() -def test_nat_hist_cure_if_death_scheduled(sim_hs_all_consumables): +def test_nat_hist_cure_if_death_scheduled(sim_hs_all_consumables_and_equipment): """Show that if a cure event is run before when a person was going to die, it cause the episode to end without the person dying.""" - sim = sim_hs_all_consumables + sim = sim_hs_all_consumables_and_equipment popsize = 100 sim.make_initial_population(n=popsize) @@ -667,10 +668,10 @@ def death(**kwargs): assert 1 == sim.modules['Alri'].logging_event.trackers['cured_cases'].report_current_total() -def test_immediate_onset_complications(sim_hs_all_consumables): +def test_immediate_onset_complications(sim_hs_all_consumables_and_equipment): """Check that if probability of immediately onsetting complications is 100%, then a person has all those complications immediately onset""" - sim = sim_hs_all_consumables + sim = sim_hs_all_consumables_and_equipment popsize = 100 sim.make_initial_population(n=popsize) @@ -712,11 +713,11 @@ def test_immediate_onset_complications(sim_hs_all_consumables): assert df.at[person_id, 'ri_SpO2_level'] != '>=93%' -def test_no_immediate_onset_complications(sim_hs_all_consumables): +def test_no_immediate_onset_complications(sim_hs_all_consumables_and_equipment): """Check that if probability of immediately onsetting complications is 0%, then a person has none of those complications immediately onset """ - sim = sim_hs_all_consumables + sim = sim_hs_all_consumables_and_equipment popsize = 100 @@ -748,7 +749,7 @@ def test_no_immediate_onset_complications(sim_hs_all_consumables): assert not df.loc[person_id, complications_cols].any() -def test_classification_based_on_symptoms_and_imci(sim_hs_all_consumables): +def test_classification_based_on_symptoms_and_imci(sim_hs_all_consumables_and_equipment): """Check that `_get_disease_classification` gives the expected classification.""" def make_hw_assesement_perfect(sim): @@ -760,7 +761,7 @@ def make_hw_assesement_perfect(sim): p['sensitivity_of_classification_of_non_severe_pneumonia_facility_level2'] = 1.0 p['sensitivity_of_classification_of_severe_pneumonia_facility_level2'] = 1.0 - sim = sim_hs_all_consumables + sim = sim_hs_all_consumables_and_equipment make_hw_assesement_perfect(sim) sim.make_initial_population(n=1000) hsi_alri_treatment = HSI_Alri_Treatment(sim.modules['Alri'], 0) @@ -846,9 +847,9 @@ def make_hw_assesement_perfect(sim): ), f"{_correct_imci_classification_on_symptoms=}" -def test_do_effects_of_alri_treatment(sim_hs_all_consumables): +def test_do_effects_of_alri_treatment(sim_hs_all_consumables_and_equipment): """Check that running `do_alri_treatment` can prevent a death from occurring.""" - sim = sim_hs_all_consumables + sim = sim_hs_all_consumables_and_equipment popsize = 100 sim.make_initial_population(n=popsize) @@ -921,10 +922,10 @@ def test_do_effects_of_alri_treatment(sim_hs_all_consumables): assert 1 == sim.modules['Alri'].logging_event.trackers['cured_cases'].report_current_total() -def test_severe_pneumonia_referral_from_hsi_first_appts(sim_hs_all_consumables): +def test_severe_pneumonia_referral_from_hsi_first_appts(sim_hs_all_consumables_and_equipment): """Check that a person is scheduled a treatment HSI following a presentation at HSI_GenericFirstApptAtFacilityLevel0 with severe pneumonia.""" - sim = sim_hs_all_consumables + sim = sim_hs_all_consumables_and_equipment popsize = 100 sim.make_initial_population(n=popsize) @@ -1234,6 +1235,7 @@ def initialise_simulation(self, sim): healthsystem.HealthSystem(resourcefilepath=resourcefilepath, disable_and_reject_all=disable_and_reject_all, cons_availability='all', + equip_availability='all', ), alri.Alri(resourcefilepath=resourcefilepath), AlriPropertiesOfOtherModules(), @@ -1338,6 +1340,7 @@ def initialise_simulation(self, sim): healthburden.HealthBurden(resourcefilepath=resourcefilepath), healthsystem.HealthSystem(resourcefilepath=resourcefilepath, cons_availability='all', + equip_availability='all', ), alri.Alri(resourcefilepath=resourcefilepath), AlriPropertiesOfOtherModules(), diff --git a/tests/test_contraception.py b/tests/test_contraception.py index 388b834393..b11402bfc2 100644 --- a/tests/test_contraception.py +++ b/tests/test_contraception.py @@ -20,6 +20,7 @@ def run_sim(tmpdir, disable=False, healthsystem_disable_and_reject_all=False, consumables_available=True, + equipment_available=True, run=True, no_discontinuation=False, incr_prob_of_failure=False, @@ -54,6 +55,14 @@ def __check_dtypes(simulation): else: _cons_available = consumables_available + # Determine availability of equipment (True --> all available; False --> none available; other --> custom arg.) + if equipment_available is True: + _equip_available = 'all' + elif equipment_available is False: + _equip_available = 'none' + else: + _equip_available = equipment_available + resourcefilepath = Path(os.path.dirname(__file__)) / '../resources' start_date = Date(2010, 1, 1) @@ -80,6 +89,7 @@ def __check_dtypes(simulation): disable=disable, disable_and_reject_all=healthsystem_disable_and_reject_all, cons_availability=_cons_available, + equip_availability=_equip_available, ), # - modules for mechanistic representation of contraception -> pregnancy -> labour -> delivery etc. @@ -417,8 +427,11 @@ def test_record_of_appt_footprint_for_switching_to_methods(tmpdir, seed): """Check that the APPT_FOOTPRINTS recorded by the HealthSystem match the expectation: specifically, that the appointment depends on the nature of the switch and whether it is a reoccurrence.""" - def get_appt_footprints(switch_from, switch_to, consumables_available) -> List[str]: - """Return a list of the APPT_FOOTPRINTS that are logged for one person for a particular switch.""" + def get_appt_footprints_when_did_not_or_did_run( + switch_from, switch_to, consumables_available, equipment_available + ) -> List[str]: + """Return a list of the APPT_FOOTPRINTS that are logged for one person for a particular switch, for HSI events + that 1) did not run, 2) did run.""" person_id = 0 sim = run_sim(tmpdir, @@ -426,6 +439,7 @@ def get_appt_footprints(switch_from, switch_to, consumables_available) -> List[s use_healthsystem=True, disable=False, consumables_available=consumables_available, + equipment_available=equipment_available, no_changes_in_contraception=True, no_discontinuation=True, equalised_risk_of_preg=0.0, @@ -436,7 +450,7 @@ def get_appt_footprints(switch_from, switch_to, consumables_available) -> List[s # Set the person's initial sex, age and contraceptive method sim.population.props.at[person_id, 'sex'] = 'F' - sim.population.props.at[person_id, 'age_years'] = 25 + sim.population.props.at[person_id, 'age_years'] = 31 sim.population.props.at[person_id, 'co_contraception'] = switch_from # Schedule the initial HSI for the change @@ -450,49 +464,193 @@ def get_appt_footprints(switch_from, switch_to, consumables_available) -> List[s sim.simulate(end_date=sim.start_date + pd.DateOffset(months=1)) hsi_run = parse_log_file(sim.log_filepath, level=logging.DEBUG)["tlo.methods.healthsystem"]["HSI_Event"] - return hsi_run.loc[ - hsi_run.did_run + return (hsi_run.loc[ + ~hsi_run.did_run & (hsi_run['Person_ID'] == person_id) & (hsi_run['TREATMENT_ID'] == 'Contraception_Routine'), 'Number_By_Appt_Type_Code' - ].to_list() + ].to_list(), + hsi_run.loc[ + hsi_run.did_run + & (hsi_run['Person_ID'] == person_id) + & (hsi_run['TREATMENT_ID'] == 'Contraception_Routine'), 'Number_By_Appt_Type_Code' + ].to_list()) + + def assert_expected_conds_for_appt_footprints( + cond_did_not_run: list, cond_did_run: list, _switch_from: str, _switch_to: str + ) -> None: + """Returns true if footprints for did_not_run and did_run HSIs satisfy given conditions. + Possible Conditions: + ['appt_footprint_equal', 'appt_footprint']; + ['len_equal_0']; + ['len_greater_0_and_all_footprints_equal', 'appt_footprint']; + ['len_greater_1_and_first_footprint_equal_subsequent_blank', 'appt_footprint']; + ['none'] + """ + # print(f"\n{set_cons_avail=}") + # print(f"{set_equip_avail=}") + # print(f"{_switch_from=}") + # print(f"{_switch_to=}") + did_not_run_footprints, did_run_footprints = get_appt_footprints_when_did_not_or_did_run( + switch_from=_switch_from, switch_to=_switch_to, + consumables_available=set_cons_avail, equipment_available=set_equip_avail + ) + # Assert condition for did_not_run footprints + if cond_did_not_run[0] == 'len_equal_0': + assert 0 == len(did_not_run_footprints) + elif cond_did_not_run[0] == 'len_greater_0_and_all_footprints_equal': + assert 0 < len(did_not_run_footprints) and all([_x == cond_did_not_run[1] for _x in did_not_run_footprints]) + elif cond_did_not_run[0] == 'none': + assert [] == did_not_run_footprints + else: + warnings.warn(f'\nWarning: {cond_did_not_run=} does not exist.') + assert 0 + + # Assert condition for did_run footprints + if cond_did_run[0] == 'appt_footprint_equal': + assert [cond_did_run[1]] == did_run_footprints + elif cond_did_run[0] == 'len_greater_1_and_first_footprint_equal_subsequent_blank': + assert len(did_run_footprints) > 1 and did_run_footprints[0] == cond_did_run[1] and \ + (0 == len([_x for _i, _x in enumerate(did_run_footprints) if (_i != 0) and (_x != {})])) + elif cond_did_run[0] == 'none': + assert [] == did_run_footprints + else: + warnings.warn(f'\nWarning: {cond_did_run=} does not exist.') + assert 0 + + # 1) If both consumables and equipment available + set_cons_avail = True + set_equip_avail = True + # ... the HSI will only be run once, hence no footprint for did_not_run and exactly one footprint for did_run: + # - If switch to female_sterilization => 'MinorSurg' + assert_expected_conds_for_appt_footprints( + cond_did_not_run=['len_equal_0'], cond_did_run=['appt_footprint_equal', {'MinorSurg': 1}], + _switch_from='not_using', _switch_to='female_sterilization' + ) + # - If maintaining IUD => 'FamilyPlanning' + assert_expected_conds_for_appt_footprints( + cond_did_not_run=['len_equal_0'], cond_did_run=['appt_footprint_equal', {'FamPlan': 1}], + _switch_from='IUD', _switch_to='IUD' + ) + # - If switching to anything new => 'FamilyPlanning' + assert_expected_conds_for_appt_footprints( + cond_did_not_run=['len_equal_0'], cond_did_run=['appt_footprint_equal', {'FamPlan': 1}], + _switch_from='not_using', _switch_to='pill' + ) + # - If maintaining on implant => 'FamilyPlanning' + assert_expected_conds_for_appt_footprints( + cond_did_not_run=['len_equal_0'], cond_did_run=['appt_footprint_equal', {'FamPlan': 1}], + _switch_from='implant', _switch_to='implant' + ) + # - If maintaining on pill => 'PharmDispensing' + assert_expected_conds_for_appt_footprints( + cond_did_not_run=['len_equal_0'], cond_did_run=['appt_footprint_equal', {'PharmDispensing': 1}], + _switch_from='pill', _switch_to='pill' + ) - # 1) If consumables available, the HSI will only be run once: - # - If switch to female_sterilization => 'MinorSurg'" - assert [{'MinorSurg': 1}] == get_appt_footprints(switch_from='not_using', - switch_to='female_sterilization', - consumables_available=True) + # 2) If consumables available, but equipment not available + set_cons_avail = True + set_equip_avail = False + # ... when a method with essential equipment (f. sterilization or IUD) requested, the HSI will never run hence no + # appt_footprint will be returned for did_run HSI, but it will be returned for did_not_run HSI multiple times: + # - If switch to female_sterilization => 'MinorSurg' + assert_expected_conds_for_appt_footprints( + cond_did_not_run=['len_greater_0_and_all_footprints_equal', {'MinorSurg': 1}], cond_did_run=['none'], + _switch_from='not_using', _switch_to='female_sterilization' + ) + # - If maintaining IUD => 'FamilyPlanning' + assert_expected_conds_for_appt_footprints( + cond_did_not_run=['len_greater_0_and_all_footprints_equal', {'FamPlan': 1}], cond_did_run=['none'], + _switch_from='IUD', _switch_to='IUD' + ) + # ... otherwise the HSI will run and only once: # - If switching to anything new => 'FamilyPlanning' - assert [{'FamPlan': 1}] == get_appt_footprints(switch_from='not_using', - switch_to='pill', - consumables_available=True) + assert_expected_conds_for_appt_footprints( + cond_did_not_run=['len_equal_0'], cond_did_run=['appt_footprint_equal', {'FamPlan': 1}], + _switch_from='not_using', _switch_to='pill' + ) # - If maintaining on implant => 'FamilyPlanning' - assert [{'FamPlan': 1}] == get_appt_footprints(switch_from='implant', - switch_to='implant', - consumables_available=True) + assert_expected_conds_for_appt_footprints( + cond_did_not_run=['len_equal_0'], cond_did_run=['appt_footprint_equal', {'FamPlan': 1}], + _switch_from='implant', _switch_to='implant' + ) # - If maintaining on pill => 'PharmDispensing' - assert [{'PharmDispensing': 1}] == get_appt_footprints(switch_from='pill', - switch_to='pill', - consumables_available=True) - - # 2) If consumables not available... there should be multiple footprints, but only the first is non-blank. - def is_list_longer_than_length_of_one_and_with_first_element_nonblank_and_subsequent_blank(x): - return ( - (len(x) > 1) - & (x[0] != {}) - & (0 == len([_x for _i, _x in enumerate(x) if (_i != 0) and (_x != {})])) - ) + assert_expected_conds_for_appt_footprints( + cond_did_not_run=['len_equal_0'], cond_did_run=['appt_footprint_equal', {'PharmDispensing': 1}], + _switch_from='pill', _switch_to='pill' + ) - assert is_list_longer_than_length_of_one_and_with_first_element_nonblank_and_subsequent_blank( - get_appt_footprints(switch_from='not_using', switch_to='female_sterilization', consumables_available=False) + # 3) If consumables not available, but equipment available + set_cons_avail = False + set_equip_avail = True + # ... it does run and there should be multiple footprints, but only the first is non-blank: + # - If switch to female_sterilization => 'MinorSurg' + assert_expected_conds_for_appt_footprints( + cond_did_not_run=['none'], + cond_did_run=['len_greater_1_and_first_footprint_equal_subsequent_blank', {'MinorSurg': 1}], + _switch_from='not_using', _switch_to='female_sterilization' + ) + # - If maintaining IUD => 'FamilyPlanning' + assert_expected_conds_for_appt_footprints( + cond_did_not_run=['none'], + cond_did_run=['len_greater_1_and_first_footprint_equal_subsequent_blank', {'FamPlan': 1}], + _switch_from='IUD', _switch_to='IUD' + ) + # - If switching to anything new => 'FamilyPlanning' + assert_expected_conds_for_appt_footprints( + cond_did_not_run=['none'], + cond_did_run=['len_greater_1_and_first_footprint_equal_subsequent_blank', {'FamPlan': 1}], + _switch_from='not_using', _switch_to='pill' ) - assert is_list_longer_than_length_of_one_and_with_first_element_nonblank_and_subsequent_blank( - get_appt_footprints(switch_from='not_using', switch_to='pill', consumables_available=False) + # - If maintaining on implant => 'FamilyPlanning' + assert_expected_conds_for_appt_footprints( + cond_did_not_run=['none'], + cond_did_run=['len_greater_1_and_first_footprint_equal_subsequent_blank', {'FamPlan': 1}], + _switch_from='implant', _switch_to='implant' + ) + # - If maintaining on pill => 'PharmDispensing' + assert_expected_conds_for_appt_footprints( + cond_did_not_run=['none'], + cond_did_run=['len_greater_1_and_first_footprint_equal_subsequent_blank', {'PharmDispensing': 1}], + _switch_from='pill', _switch_to='pill' ) - assert is_list_longer_than_length_of_one_and_with_first_element_nonblank_and_subsequent_blank( - get_appt_footprints(switch_from='implant', switch_to='implant', consumables_available=False) + + # 4) If both consumables and equipment not available, ... + set_cons_avail = False + set_equip_avail = False + # ... when a method with essential equipment (f. sterilization or IUD) requested, the HSI will never run hence no + # appt_footprint will be returned for did_run HSI, but it will be returned for did_not_run HSI multiple times: + # - If switch to female_sterilization => 'MinorSurg' + assert_expected_conds_for_appt_footprints( + cond_did_not_run=['len_greater_0_and_all_footprints_equal', {'MinorSurg': 1}], cond_did_run=['none'], + _switch_from='not_using', _switch_to='female_sterilization' ) - assert is_list_longer_than_length_of_one_and_with_first_element_nonblank_and_subsequent_blank( - get_appt_footprints(switch_from='pill', switch_to='pill', consumables_available=False) + # - If maintaining IUD => 'FamilyPlanning' + assert_expected_conds_for_appt_footprints( + cond_did_not_run=['len_greater_0_and_all_footprints_equal', {'FamPlan': 1}], cond_did_run=['none'], + _switch_from='IUD', _switch_to='IUD' + ) + # ... otherwise the HSI will run and there should be multiple footprints, but only the first is non-blank: + # TODO: this is the current logic, but it doesn't sound right, if it is never performed due to missing consumables + # it should never have the footprint equal, or do we expect that they will be coming again and again and again + # and again ... until they will get it in very far future? + + # - If switching to anything new => 'FamilyPlanning' + assert_expected_conds_for_appt_footprints( + cond_did_not_run=['none'], + cond_did_run=['len_greater_1_and_first_footprint_equal_subsequent_blank', {'FamPlan': 1}], + _switch_from='not_using', _switch_to='pill' + ) + # - If maintaining on implant => 'FamilyPlanning' + assert_expected_conds_for_appt_footprints( + cond_did_not_run=['none'], + cond_did_run=['len_greater_1_and_first_footprint_equal_subsequent_blank', {'FamPlan': 1}], + _switch_from='implant', _switch_to='implant' + ) + # - If maintaining on pill => 'PharmDispensing' + assert_expected_conds_for_appt_footprints( + cond_did_not_run=['none'], + cond_did_run=['len_greater_1_and_first_footprint_equal_subsequent_blank', {'PharmDispensing': 1}], + _switch_from='pill', _switch_to='pill' ) diff --git a/tests/test_healthsystem.py b/tests/test_healthsystem.py index 22b9d7ff8c..f85fd3e735 100644 --- a/tests/test_healthsystem.py +++ b/tests/test_healthsystem.py @@ -1260,6 +1260,7 @@ def test_HealthSystemChangeParameters(seed, tmpdir): 'capabilities_coefficient': 0.5, 'cons_availability': 'all', 'beds_availability': 'default', + 'equip_availability': 'default', } new_parameters = { 'mode_appt_constraints': 2, @@ -1267,6 +1268,7 @@ def test_HealthSystemChangeParameters(seed, tmpdir): 'capabilities_coefficient': 1.0, 'cons_availability': 'none', 'beds_availability': 'none', + 'equip_availability': 'all', } class CheckHealthSystemParameters(RegularEvent, PopulationScopeEventMixin): @@ -1282,6 +1284,7 @@ def apply(self, population): _params['capabilities_coefficient'] = hs.capabilities_coefficient _params['cons_availability'] = hs.consumables.cons_availability _params['beds_availability'] = hs.bed_days.availability + _params['equip_availability'] = hs.equip_availability logger = logging.getLogger('tlo.methods.healthsystem') logger.info(key='CheckHealthSystemParameters', data=_params) From a6fffc9fd2a1579371fbeb7f7f902c548613eecd Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Mon, 25 Mar 2024 15:25:21 +0000 Subject: [PATCH 069/118] example test suite --- tests/test_equipment.py | 187 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 tests/test_equipment.py diff --git a/tests/test_equipment.py b/tests/test_equipment.py new file mode 100644 index 0000000000..01aff8aab8 --- /dev/null +++ b/tests/test_equipment.py @@ -0,0 +1,187 @@ +"""This file contains all the tests to do with Equipment use logging and availability checks.""" +import os +from pathlib import Path +from typing import Union, Dict, Iterable + +import pandas as pd + +from tlo import Simulation, Module, Date +from tlo.analysis.utils import parse_log_file +from tlo.events import IndividualScopeEventMixin +from tlo.methods import Metadata, demography, healthsystem +from tlo.methods.hsi_event import HSI_Event + +resourcefilepath = Path(os.path.dirname(__file__)) / '../resources' + +equipment_item_code_that_is_available = [0, 1, ] +equipment_item_code_that_is_not_available = [2, 3, ] + + +def run_simulation_return_log(seed, tmpdir, essential_equipment: Iterable[str], other_equipment: Iterable[str]) -> Dict: + """Returns a parsed logs from `tlo.methods.healthsystem.summary` from a simulation object, in which a single + event has been scheduled with the specified equipment usage, and the availability of equipment has been manipulated. + """ + + class DummyHSIEvent(HSI_Event, IndividualScopeEventMixin): + def __init__(self, + module, + person_id, + level, + essential_equipment: Union[int, None], + other_equipment: Union[int, None] + ): + super().__init__(module, person_id=person_id) + self.TREATMENT_ID = "DummyHSIEvent" + self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint({}) + self.ACCEPTED_FACILITY_LEVEL = level + self.ESSENTIAL_EQUIPMENT = str(essential_equipment) if essential_equipment is not None else set() + self._other_equipment = other_equipment + + def apply(self, person_id, squeeze_factor): + if self._other_equipment is not None: + self.add_equipment(self._other_equipment) + + + class DummyModule(Module): + METADATA = {Metadata.DISEASE_MODULE, Metadata.USES_HEALTHSYSTEM} + + def __init__(self, essential_equipment, other_equipment, name=None): + super().__init__(name) + self.essential_equipment = essential_equipment + self.other_equipment = other_equipment + + def read_parameters(self, data_folder): + pass + + def initialise_population(self, population): + pass + + def initialise_simulation(self, sim): + # Schedule the HSI_Event to occur on the first day of the simulation + sim.modules['HealthSystem'].schedule_hsi_event( + hsi_event=DummyHSIEvent( + person_id=0, + level='2', + module=sim.modules['DummyModule'], + essential_equipment=self.essential_equipment, + other_equipment=self.other_equipment, + ), + do_hsi_event_checks=False, + topen=sim.date, + tclose=None, + priority=0, + ) + + log_config = {"filename": "log", "directory": tmpdir} + sim = Simulation(start_date=Date(2010, 1, 1), seed=seed, log_config=log_config) + sim.register( + demography.Demography(resourcefilepath=resourcefilepath), + healthsystem.HealthSystem(resourcefilepath=resourcefilepath), + DummyModule(essential_equipment=essential_equipment, other_equipment=other_equipment), + ) + + # Manipulate availability of equipment + df = sim.modules['HealthSystem'].parameters['Equipment'] + col_for_availability = df.columns[df.columns.str.startswith('Avail_')] + df.loc[df['Equip_Code'].isin(equipment_item_code_that_is_available), col_for_availability] = True + df.loc[df['Equip_Code'].isin(equipment_item_code_that_is_not_available), col_for_availability] = False + df['Equip_Item'] = df['Equip_Code'].astype(str) + sim.modules['HealthSystem'].parameters['Equipment'] = df.loc[df['Equip_Code'].isin(set(equipment_item_code_that_is_available) | set(equipment_item_code_that_is_not_available))] + + sim.make_initial_population(n=100) + sim.simulate(end_date=pd.DateOffset(months=1)) + + return parse_log_file(sim.log_filepath)['tlo.methods.healthsystem.summary'] + + + + +def test_equipment_use_is_logged(seed, tmpdir): + """Check that an HSI that after an HSI is run, the logs reflect the use of the equipment (and correctly record the + name of the HSI and the facility_level at which ran). + This is repeated for: + * An HSI that declares use of equipment during its `apply` method (but no essential equipment); + * An HSI that declare use of essential equipment but nothing in its `apply` method`; + * An HSI that declare use of essential equipment and equipment during its `apply` method; + * An HSI that declares not use of any equipment (logs should be empty). + """ + + def logged_equipment_used(sim: Dict) -> pd.DataFrame: + """Read the log to work out what equipment usage has been logged.""" + # @Eva - I think this will somehow use the function that is currently in `src/scripts/healthsystem/equipment/equipment_catalogue.py` + pass + + def get_sim(essential_equipment, other_equipment): + """Pass-through to `run_simulation_return_log` to make call simpler.""" + return run_simulation_return_log( + seed=seed, + tmpdir=tmpdir, + essential_equipment=essential_equipment, + other_equipment=other_equipment, + ) + + # Check that the log matches expectation under each permutation + item_available_as_set_of_str = {str(equipment_item_code_that_is_available[0])} + + # * An HSI that declares use of equipment during its `apply` method (but no essential equipment) + expected_df = pd.DataFrame() # <-- fill in what we expect it to look like + assert expected_df.equals(logged_equipment_used(get_sim( + essential_equipment={}, + other_equipment=item_available_as_set_of_str, + ))) + + # * An HSI that declare use of essential equipment but nothing in its `apply` method`; + expected_df = pd.DataFrame() # <-- fill in what we expect it to look like + assert expected_df.equals(logged_equipment_used(get_sim( + essential_equipment=item_available_as_set_of_str, + other_equipment={}, + ))) + + # * An HSI that declare use of essential equipment and equipment during its `apply` method; + expected_df = pd.DataFrame() # <-- fill in what we expect it to look like + assert expected_df.equals(logged_equipment_used(get_sim( + essential_equipment=item_available_as_set_of_str, + other_equipment=item_available_as_set_of_str, + ))) + + # * An HSI that declares not use of any equipment (logs should be empty). + expected_df = pd.DataFrame() # <-- fill in what we expect it to look like + assert expected_df.equals(logged_equipment_used(get_sim( + essential_equipment={}, + other_equipment={}, + ))) + + +def test_hsi_does_not_run_if_essential_equipment_is_not_available(seed, tmpdir): + """Check that an HSI which declares an item of equipment that is essential does run if that item is available + and does not run if that item is not available.""" + + def did_hsi_run(sim: Dict) -> bool: + """Read the log to work out if the Dummy HSI Event ran or not.""" + pass + + def get_sim(essential_equipment): + """Pass-through to `run_simulation_return_log` to make call simpler.""" + return run_simulation_return_log( + seed=seed, + tmpdir=tmpdir, + essential_equipment=essential_equipment, + other_equipment=None + ) + + # HSI_Event that requires equipment that is available --> will run + assert did_hsi_run( + get_sim( + essential_equipment=set(str(equipment_item_code_that_is_available[0])) + ) + ) + + # HSI_Event that requires equipment that is not available --> will not run + assert not did_hsi_run( + get_sim( + essential_equipment=set(str(equipment_item_code_that_is_not_available[0])) + ) + ) + + + From 8428d8ae194d5ccbf7e215a43bf50ff1218c3898 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Mon, 25 Mar 2024 15:26:43 +0000 Subject: [PATCH 070/118] typo and add todo --- src/tlo/methods/hsi_event.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index ed3703338d..c7981f2dbc 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -334,14 +334,21 @@ def set_equipment_essential_to_run_event(self, set_of_equip: Set[str]) -> None: else: self.ESSENTIAL_EQUIPMENT = set() + # todo add function to set essential equipment + def add_equipment(self, set_of_equip: Set[str]) -> None: """Helper function to update equipment. Should be passed a set of equipment item names (strings). """ # Update EQUIPMENT if the given set_of_equip in correct format, ie a non-empty set of strings - if not isinstance(set_of_equip, set) or any(not isinstance(item, str) for item in set_of_equip) or \ - (set_of_equip in [set(), None, {''}]): + if ( + (not isinstance(set_of_equip, set)) + or + any(not isinstance(item, str) for item in set_of_equip) + or + (set_of_equip in [set(), None, {''}]) + ): raise ValueError( "Argument to add_equipment should be a non-empty set of strings of " "equipment item names from ResourceFile_Equipment.csv." From 2a3c8f8b6c45ac2fef51a8b48ba1f98652813471 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Mon, 25 Mar 2024 15:37:05 +0000 Subject: [PATCH 071/118] further tests --- tests/test_equipment.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_equipment.py b/tests/test_equipment.py index 01aff8aab8..3d15294c9a 100644 --- a/tests/test_equipment.py +++ b/tests/test_equipment.py @@ -184,4 +184,17 @@ def get_sim(essential_equipment): ) +def test_lookup_equipment_item_code_from_item_name(): + pass + +def test_lookup_equipment_item_code_from_pkg_name(): + pass + +def test_lookup_item_availability_by_hsi_event(): + pass + +def test_change_equipment_availability(): + """Test that we can change the availability of equipment midway through the simulation.""" + pass + From b1aa9aef99915ee20b3bf89219eba7210ab5c269 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Mon, 25 Mar 2024 16:04:42 +0000 Subject: [PATCH 072/118] [no_ci] RF_Equip: availabilities changed from probs to True/False values (all True for now) --- .../infrastructure_and_equipment/ResourceFile_Equipment.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv index 17c3f82ba7..5700c76b27 100644 --- a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv +++ b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0be4639d20a28222ecba14f04025c0270fd799e13195ccf76c937448c4c5e39a -size 42931 +oid sha256:20c07f76ec100a3c1a221c4c8ecc4bd6e37379f395b08ae67491501d04f04d6e +size 45343 From ffd5178ea922fc350fca0aa26a9fb6d38e5f766e Mon Sep 17 00:00:00 2001 From: Eva Janouskova <48157464+EvaJanouskova@users.noreply.github.com> Date: Tue, 26 Mar 2024 19:04:52 +0100 Subject: [PATCH 073/118] ac: rm comments Co-authored-by: Tim Hallett <39991060+tbhallett@users.noreply.github.com> --- src/tlo/methods/care_of_women_during_pregnancy.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/tlo/methods/care_of_women_during_pregnancy.py b/src/tlo/methods/care_of_women_during_pregnancy.py index d70ad7a29b..bc22b86993 100644 --- a/src/tlo/methods/care_of_women_during_pregnancy.py +++ b/src/tlo/methods/care_of_women_during_pregnancy.py @@ -205,8 +205,6 @@ def get_and_store_pregnancy_item_codes(self): get_list_of_items = pregnancy_helper_functions.get_list_of_items # ---------------------------------- BLOOD TEST EQUIPMENT --------------------------------------------------- - # TODO: As we now consider both consumables and equipment, using 'equipment' when meaning consumables is - # confusing self.item_codes_preg_consumables['blood_test_equipment'] = \ get_list_of_items(self, ['Blood collecting tube, 5 ml', 'Cannula iv (winged with injection pot) 18_each_CMST', From 7c8ea54984dc4c72a94ee429afe25a762f1efd35 Mon Sep 17 00:00:00 2001 From: Eva Janouskova <48157464+EvaJanouskova@users.noreply.github.com> Date: Tue, 26 Mar 2024 19:20:54 +0100 Subject: [PATCH 074/118] hs: rm equip_availability before sim default Co-authored-by: Tim Hallett <39991060+tbhallett@users.noreply.github.com> --- src/tlo/methods/healthsystem.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 1364f1196d..637a1b1327 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -440,7 +440,6 @@ def __init__( assert equip_availability in (None, 'default', 'all', 'none') self.arg_equip_availability = equip_availability - self.equip_availability = 'all' # provided so that there is a default even before simulation is run # `compute_squeeze_factor_to_district_level` is a Boolean indicating whether the computation of squeeze_factors # should be specific to each district (when `True`), or if the computation of squeeze_factors should be on the From 6ddd11e529719fe4198191d717075c57450657d0 Mon Sep 17 00:00:00 2001 From: Eva Janouskova <48157464+EvaJanouskova@users.noreply.github.com> Date: Tue, 26 Mar 2024 20:02:44 +0100 Subject: [PATCH 075/118] labour: rm comments Co-authored-by: Tim Hallett <39991060+tbhallett@users.noreply.github.com> --- src/tlo/methods/labour.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/tlo/methods/labour.py b/src/tlo/methods/labour.py index 5fdc3d3e52..42da5cc7d0 100644 --- a/src/tlo/methods/labour.py +++ b/src/tlo/methods/labour.py @@ -686,8 +686,6 @@ def get_and_store_labour_item_codes(self): get_list_of_items = pregnancy_helper_functions.get_list_of_items # ---------------------------------- IV DRUG ADMIN EQUIPMENT ------------------------------------------------- - # TODO: As we now consider both consumables and equipment, using 'equipment' when meaning consumables is - # confusing self.item_codes_lab_consumables['iv_drug_equipment'] = \ get_list_of_items(self, ['Cannula iv (winged with injection pot) 18_each_CMST', 'Giving set iv administration + needle 15 drops/ml_each_CMST', From 4f4d858b1085a373181e0ee24210f6d90114a832 Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Tue, 26 Mar 2024 18:46:36 +0000 Subject: [PATCH 076/118] hs: rm extra line --- src/tlo/methods/healthsystem.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 637a1b1327..a6c30c85a1 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -635,7 +635,6 @@ def pre_initialise_population(self): # Set up framework for considering a priority policy self.setup_priority_policy() - def initialise_population(self, population): self.bed_days.initialise_population(population.props) From f8b0ac1b9c9fd7ebb8b30630372109e5ca072bfe Mon Sep 17 00:00:00 2001 From: Eva Janouskova Date: Tue, 26 Mar 2024 18:56:47 +0000 Subject: [PATCH 077/118] hs: raise error if 1) ess equip not a set of ints, 2) invalid equip_availability --- src/tlo/methods/healthsystem.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index a6c30c85a1..5dfff499e6 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -1088,7 +1088,7 @@ def get_equip_availability(self) -> str: return _equip_availability - def get_equip_item_availability(self, equip_item_code: str) -> bool: + def get_equip_item_availability(self, equip_item_code: int) -> bool: # TODO: update with implementation of essential equipment availability for the HSI event to run # for now, always available if equip_item_code in [243, 41]: # 243 = Pulse oximeter, 41 = 'Lamp, Anglepoise' @@ -1096,16 +1096,22 @@ def get_equip_item_availability(self, equip_item_code: str) -> bool: return True # True of False def get_essential_equip_availability(self, essential_equip_set: Set[int]) -> bool: + if not isinstance(essential_equip_set, set) or any(not isinstance(item, int) for item in essential_equip_set): + raise ValueError( + "Argument to get_essential_equip_availability should be a set of integers." + ) if self.equip_availability == 'all': # Always all equipment available return True elif self.equip_availability == 'default': # True if all items of essential equipment available; False if any unavailable return all(self.get_equip_item_availability(item_code) for item_code in essential_equip_set) - else: # self.equip_availability == 'none': + elif self.equip_availability == 'none': # True if no essential equipment requested, otherwise False as assumed no equipment available # TODO: Should no equipment be logged then? return not bool(essential_equip_set) + else: + raise ValueError("Value for self.equip_availability invalid, it should be 'all', 'default', or 'none'.") def schedule_to_call_never_ran_on_date(self, hsi_event: 'HSI_Event', tdate: datetime.datetime): """Function to schedule never_ran being called on a given date""" From 16752aef35206117650f617ceec558d99eb17921 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Thu, 9 May 2024 10:24:22 +0100 Subject: [PATCH 078/118] roll back incidental changes Note to @Eva -- there were quite a ot of changes in `test_contraception.py` that seemed unrelated to this PR, but which she may like to cherry-pick and raise a PR for specifically. --- src/tlo/analysis/utils.py | 6 +- src/tlo/methods/alri.py | 4 - src/tlo/methods/breast_cancer.py | 3 +- src/tlo/methods/contraception.py | 44 +----- src/tlo/methods/newborn_outcomes.py | 2 - src/tlo/methods/rti.py | 4 +- tests/test_alri.py | 2 +- tests/test_contraception.py | 230 +++++----------------------- 8 files changed, 42 insertions(+), 253 deletions(-) diff --git a/src/tlo/analysis/utils.py b/src/tlo/analysis/utils.py index 64ba966aa5..3aeff2bb11 100644 --- a/src/tlo/analysis/utils.py +++ b/src/tlo/analysis/utils.py @@ -18,7 +18,6 @@ import numpy as np import pandas as pd import squarify -from pandas.api.types import is_numeric_dtype from tlo import Date, Simulation, logging, util from tlo.logging.reader import LogData @@ -294,10 +293,7 @@ def generate_series(dataframe: pd.DataFrame) -> pd.Series: df: pd.DataFrame = load_pickled_dataframes(results_folder, draw, run, module)[module][key] output_from_eval: pd.Series = generate_series(df) assert pd.Series == type(output_from_eval), 'Custom command does not generate a pd.Series' - if is_numeric_dtype(output_from_eval): - res[draw_run] = output_from_eval * get_multiplier(draw, run) - else: - res[draw_run] = output_from_eval + res[draw_run] = output_from_eval * get_multiplier(draw, run) except KeyError: # Some logs could not be found - probably because this run failed. diff --git a/src/tlo/methods/alri.py b/src/tlo/methods/alri.py index 447cef8026..277726e0ff 100644 --- a/src/tlo/methods/alri.py +++ b/src/tlo/methods/alri.py @@ -2316,8 +2316,6 @@ def __init__(self, module: Module, person_id: int, facility_level: str = "0", in self._treatment_id_stub = 'Alri_Pneumonia_Treatment' self._facility_levels = ("0", "1a", "1b", "2") # Health facility levels at which care may be provided assert facility_level in self._facility_levels - self.set_equipment_essential_to_run_event({'Pulse oximeter'}) - # TODO: CORRECT --- an example with ess. equipm. set (which may or may not be used at the end) self.is_followup_following_treatment_failure = is_followup_following_treatment_failure if not inpatient: @@ -2624,8 +2622,6 @@ def _get_disease_classification_for_treatment_decision(self, 'cough_or_cold' (symptoms-based assessment) }.""" - self.add_equipment({'Pulse oximeter'}) - child_is_younger_than_2_months = age_exact_years < (2.0 / 12.0) imci_classification_based_on_symptoms = self._get_imci_classification_based_on_symptoms( diff --git a/src/tlo/methods/breast_cancer.py b/src/tlo/methods/breast_cancer.py index 677b413fd2..8a3281243c 100644 --- a/src/tlo/methods/breast_cancer.py +++ b/src/tlo/methods/breast_cancer.py @@ -668,7 +668,6 @@ def __init__(self, module, person_id): self.TREATMENT_ID = "BreastCancer_Investigation" self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint({"Over5OPD": 1, "Mammography": 1}) self.ACCEPTED_FACILITY_LEVEL = '3' # Biopsy only available at level 3 and above. - # TODO: but the appt footprints suggests mammography to be provided def apply(self, person_id, squeeze_factor): df = self.sim.population.props @@ -685,7 +684,7 @@ def apply(self, person_id, squeeze_factor): if not pd.isnull(df.at[person_id, "brc_date_diagnosis"]): return hs.get_blank_appt_footprint() - df.at[person_id, 'brc_breast_lump_discernible_investigated'] = True + df.brc_breast_lump_discernible_investigated = True # Use a biopsy to diagnose whether the person has breast Cancer: # todo: request consumables needed for this diff --git a/src/tlo/methods/contraception.py b/src/tlo/methods/contraception.py index 38baf6d9b9..67d6684fce 100644 --- a/src/tlo/methods/contraception.py +++ b/src/tlo/methods/contraception.py @@ -91,7 +91,6 @@ class Contraception(Module): 'max_number_of_runs_of_hsi_if_consumable_not_available': Parameter( Types.INT, "The maximum number of time an HSI can run (repeats occur if the consumables are not " "available)."), - # TODO: We don't have anything like this for equipment, should we? 'max_days_delay_between_decision_to_change_method_and_hsi_scheduled': Parameter( Types.INT, "The maximum delay (in days) between the decision for a contraceptive to change and the `topen` " @@ -1111,20 +1110,6 @@ def __init__(self, module, person_id, new_contraceptive): self.TREATMENT_ID = "Contraception_Routine" self.ACCEPTED_FACILITY_LEVEL = _facility_level - # Set essential equipment based on the contraception method - if new_contraceptive == 'female_sterilization': - self.set_equipment_essential_to_run_event({ - 'Cusco’s/ bivalved Speculum (small, medium, large)', 'Lamp, Anglepoise' - }) - # + 'Minor Surgery' pkg - # TODO: How to set pkg as essential? - elif new_contraceptive == 'IUD': - self.set_equipment_essential_to_run_event({ - 'Cusco’s/ bivalved Speculum (small, medium, large)', 'Sponge Holding Forceps' - }) - else: - self.set_equipment_essential_to_run_event({''}) - @property def EXPECTED_APPT_FOOTPRINT(self): """Return the expected appt footprint based on contraception method and whether the HSI has been rescheduled.""" @@ -1159,12 +1144,6 @@ def apply(self, person_id, squeeze_factor): # Record the date that Family Planning Appointment happened for this person self.sim.population.props.at[person_id, "co_date_of_last_fp_appt"] = self.sim.date - # Measure weight, height and BP even if contraception not administrated - # TODO: Always or only if it's not a rescheduled appt? - self.add_equipment({ - 'Weighing scale', 'Height Pole (Stadiometer)', 'Blood pressure machine' - }) - # Determine essential and optional items # TODO: we don't distinguish essential X optional for contraception methods yet, will need to update once we do items_essential = self.module.cons_codes[self.new_contraceptive] @@ -1191,8 +1170,7 @@ def apply(self, person_id, squeeze_factor): items_all = {**items_essential, **items_optional} # Determine whether the contraception is administrated (ie all essential items are available), - # if so do log the availability of all items and update used equipment if any, if not set the contraception to - # "not_using": + # if so do log the availability of all items, if not set the contraception to "not_using": co_administrated = all(v for k, v in cons_available.items() if k in items_essential) if co_administrated: @@ -1216,22 +1194,6 @@ def apply(self, person_id, squeeze_factor): ) _new_contraceptive = self.new_contraceptive - - # Add equipment if any used with the method - if _new_contraceptive == 'female_sterilization': - self.add_equipment({ - 'Cusco’s/ bivalved Speculum (small, medium, large)', 'Lamp, Anglepoise' - }) - self.add_equipment_from_pkg({ - 'Minor Surgery' - }) - # TODO: this is just an example - update once figured out what we want in the pkgs - # (! Update also the RF_Equipment accordingly !) - elif _new_contraceptive == 'IUD': - self.add_equipment({ - 'Cusco’s/ bivalved Speculum (small, medium, large)', 'Sponge Holding Forceps' - }) - else: _new_contraceptive = "not_using" @@ -1246,10 +1208,6 @@ def apply(self, person_id, squeeze_factor): # If the intended change was not possible due to non-available consumable, reschedule the appointment if (not co_administrated) and ( - # TODO: the max nmb of runs is set to 1000, so they will be coming back every day for 1000 consecutive days? - # -- but only 8 footprints for did_not_run event when switch to f. steril when consumables available but - # equipment not available in test_contraception (test_record_of_appt_footprint_for_switching_to_methods). - # What am I missing? ???? self._number_of_times_run < self.module.parameters['max_number_of_runs_of_hsi_if_consumable_not_available'] ): self.reschedule() diff --git a/src/tlo/methods/newborn_outcomes.py b/src/tlo/methods/newborn_outcomes.py index f4fcf508b4..debfdb3530 100644 --- a/src/tlo/methods/newborn_outcomes.py +++ b/src/tlo/methods/newborn_outcomes.py @@ -380,8 +380,6 @@ def get_and_store_newborn_item_codes(self): get_list_of_items = pregnancy_helper_functions.get_list_of_items # ---------------------------------- IV DRUG ADMIN EQUIPMENT ------------------------------------------------- - # TODO: As we now consider both consumables and equipment, using 'equipment' when meaning consumables is - # confusing self.item_codes_nb_consumables['iv_drug_equipment'] = \ get_list_of_items(self, ['Cannula iv (winged with injection pot) 18_each_CMST', 'Giving set iv administration + needle 15 drops/ml_each_CMST', diff --git a/src/tlo/methods/rti.py b/src/tlo/methods/rti.py index 2b5a5f4efa..654378c4bf 100644 --- a/src/tlo/methods/rti.py +++ b/src/tlo/methods/rti.py @@ -4741,7 +4741,7 @@ def apply(self, person_id, squeeze_factor): get_item_code('Pethidine, 50 mg/ml, 2 ml ampoule'): 1, # administer antibiotic get_item_code("Ampicillin injection 500mg, PFR_each_CMST"): 1, - # consumables used by surgeon, gloves and facemask + # equipment used by surgeon, gloves and facemask get_item_code('Disposables gloves, powder free, 100 pieces per box'): 1, get_item_code('surgical face mask, disp., with metal nose piece_50_IDA'): 1, # request syringe @@ -5077,7 +5077,7 @@ def apply(self, person_id, squeeze_factor): get_item_code('Pethidine, 50 mg/ml, 2 ml ampoule'): 1, # administer antibiotic get_item_code("Ampicillin injection 500mg, PFR_each_CMST"): 1, - # consumables used by surgeon, gloves and facemask + # equipment used by surgeon, gloves and facemask get_item_code('Disposables gloves, powder free, 100 pieces per box'): 1, get_item_code('surgical face mask, disp., with metal nose piece_50_IDA'): 1, # request syringe diff --git a/tests/test_alri.py b/tests/test_alri.py index 0fba5fea8d..fb03312515 100644 --- a/tests/test_alri.py +++ b/tests/test_alri.py @@ -54,7 +54,7 @@ def _get_person_id(df, age_bounds: tuple = (0.0, np.inf)) -> int: ].index[0] -def get_sim(tmpdir, seed, cons_available, equip_available='all'): +def get_sim(tmpdir, seed, cons_available, equip_available): """Return simulation objection with Alri and other necessary modules registered.""" sim = Simulation( start_date=start_date, diff --git a/tests/test_contraception.py b/tests/test_contraception.py index b11402bfc2..388b834393 100644 --- a/tests/test_contraception.py +++ b/tests/test_contraception.py @@ -20,7 +20,6 @@ def run_sim(tmpdir, disable=False, healthsystem_disable_and_reject_all=False, consumables_available=True, - equipment_available=True, run=True, no_discontinuation=False, incr_prob_of_failure=False, @@ -55,14 +54,6 @@ def __check_dtypes(simulation): else: _cons_available = consumables_available - # Determine availability of equipment (True --> all available; False --> none available; other --> custom arg.) - if equipment_available is True: - _equip_available = 'all' - elif equipment_available is False: - _equip_available = 'none' - else: - _equip_available = equipment_available - resourcefilepath = Path(os.path.dirname(__file__)) / '../resources' start_date = Date(2010, 1, 1) @@ -89,7 +80,6 @@ def __check_dtypes(simulation): disable=disable, disable_and_reject_all=healthsystem_disable_and_reject_all, cons_availability=_cons_available, - equip_availability=_equip_available, ), # - modules for mechanistic representation of contraception -> pregnancy -> labour -> delivery etc. @@ -427,11 +417,8 @@ def test_record_of_appt_footprint_for_switching_to_methods(tmpdir, seed): """Check that the APPT_FOOTPRINTS recorded by the HealthSystem match the expectation: specifically, that the appointment depends on the nature of the switch and whether it is a reoccurrence.""" - def get_appt_footprints_when_did_not_or_did_run( - switch_from, switch_to, consumables_available, equipment_available - ) -> List[str]: - """Return a list of the APPT_FOOTPRINTS that are logged for one person for a particular switch, for HSI events - that 1) did not run, 2) did run.""" + def get_appt_footprints(switch_from, switch_to, consumables_available) -> List[str]: + """Return a list of the APPT_FOOTPRINTS that are logged for one person for a particular switch.""" person_id = 0 sim = run_sim(tmpdir, @@ -439,7 +426,6 @@ def get_appt_footprints_when_did_not_or_did_run( use_healthsystem=True, disable=False, consumables_available=consumables_available, - equipment_available=equipment_available, no_changes_in_contraception=True, no_discontinuation=True, equalised_risk_of_preg=0.0, @@ -450,7 +436,7 @@ def get_appt_footprints_when_did_not_or_did_run( # Set the person's initial sex, age and contraceptive method sim.population.props.at[person_id, 'sex'] = 'F' - sim.population.props.at[person_id, 'age_years'] = 31 + sim.population.props.at[person_id, 'age_years'] = 25 sim.population.props.at[person_id, 'co_contraception'] = switch_from # Schedule the initial HSI for the change @@ -464,193 +450,49 @@ def get_appt_footprints_when_did_not_or_did_run( sim.simulate(end_date=sim.start_date + pd.DateOffset(months=1)) hsi_run = parse_log_file(sim.log_filepath, level=logging.DEBUG)["tlo.methods.healthsystem"]["HSI_Event"] - return (hsi_run.loc[ - ~hsi_run.did_run + return hsi_run.loc[ + hsi_run.did_run & (hsi_run['Person_ID'] == person_id) & (hsi_run['TREATMENT_ID'] == 'Contraception_Routine'), 'Number_By_Appt_Type_Code' - ].to_list(), - hsi_run.loc[ - hsi_run.did_run - & (hsi_run['Person_ID'] == person_id) - & (hsi_run['TREATMENT_ID'] == 'Contraception_Routine'), 'Number_By_Appt_Type_Code' - ].to_list()) - - def assert_expected_conds_for_appt_footprints( - cond_did_not_run: list, cond_did_run: list, _switch_from: str, _switch_to: str - ) -> None: - """Returns true if footprints for did_not_run and did_run HSIs satisfy given conditions. - Possible Conditions: - ['appt_footprint_equal', 'appt_footprint']; - ['len_equal_0']; - ['len_greater_0_and_all_footprints_equal', 'appt_footprint']; - ['len_greater_1_and_first_footprint_equal_subsequent_blank', 'appt_footprint']; - ['none'] - """ - # print(f"\n{set_cons_avail=}") - # print(f"{set_equip_avail=}") - # print(f"{_switch_from=}") - # print(f"{_switch_to=}") - did_not_run_footprints, did_run_footprints = get_appt_footprints_when_did_not_or_did_run( - switch_from=_switch_from, switch_to=_switch_to, - consumables_available=set_cons_avail, equipment_available=set_equip_avail - ) - # Assert condition for did_not_run footprints - if cond_did_not_run[0] == 'len_equal_0': - assert 0 == len(did_not_run_footprints) - elif cond_did_not_run[0] == 'len_greater_0_and_all_footprints_equal': - assert 0 < len(did_not_run_footprints) and all([_x == cond_did_not_run[1] for _x in did_not_run_footprints]) - elif cond_did_not_run[0] == 'none': - assert [] == did_not_run_footprints - else: - warnings.warn(f'\nWarning: {cond_did_not_run=} does not exist.') - assert 0 - - # Assert condition for did_run footprints - if cond_did_run[0] == 'appt_footprint_equal': - assert [cond_did_run[1]] == did_run_footprints - elif cond_did_run[0] == 'len_greater_1_and_first_footprint_equal_subsequent_blank': - assert len(did_run_footprints) > 1 and did_run_footprints[0] == cond_did_run[1] and \ - (0 == len([_x for _i, _x in enumerate(did_run_footprints) if (_i != 0) and (_x != {})])) - elif cond_did_run[0] == 'none': - assert [] == did_run_footprints - else: - warnings.warn(f'\nWarning: {cond_did_run=} does not exist.') - assert 0 - - # 1) If both consumables and equipment available - set_cons_avail = True - set_equip_avail = True - # ... the HSI will only be run once, hence no footprint for did_not_run and exactly one footprint for did_run: - # - If switch to female_sterilization => 'MinorSurg' - assert_expected_conds_for_appt_footprints( - cond_did_not_run=['len_equal_0'], cond_did_run=['appt_footprint_equal', {'MinorSurg': 1}], - _switch_from='not_using', _switch_to='female_sterilization' - ) - # - If maintaining IUD => 'FamilyPlanning' - assert_expected_conds_for_appt_footprints( - cond_did_not_run=['len_equal_0'], cond_did_run=['appt_footprint_equal', {'FamPlan': 1}], - _switch_from='IUD', _switch_to='IUD' - ) - # - If switching to anything new => 'FamilyPlanning' - assert_expected_conds_for_appt_footprints( - cond_did_not_run=['len_equal_0'], cond_did_run=['appt_footprint_equal', {'FamPlan': 1}], - _switch_from='not_using', _switch_to='pill' - ) - # - If maintaining on implant => 'FamilyPlanning' - assert_expected_conds_for_appt_footprints( - cond_did_not_run=['len_equal_0'], cond_did_run=['appt_footprint_equal', {'FamPlan': 1}], - _switch_from='implant', _switch_to='implant' - ) - # - If maintaining on pill => 'PharmDispensing' - assert_expected_conds_for_appt_footprints( - cond_did_not_run=['len_equal_0'], cond_did_run=['appt_footprint_equal', {'PharmDispensing': 1}], - _switch_from='pill', _switch_to='pill' - ) + ].to_list() - # 2) If consumables available, but equipment not available - set_cons_avail = True - set_equip_avail = False - # ... when a method with essential equipment (f. sterilization or IUD) requested, the HSI will never run hence no - # appt_footprint will be returned for did_run HSI, but it will be returned for did_not_run HSI multiple times: - # - If switch to female_sterilization => 'MinorSurg' - assert_expected_conds_for_appt_footprints( - cond_did_not_run=['len_greater_0_and_all_footprints_equal', {'MinorSurg': 1}], cond_did_run=['none'], - _switch_from='not_using', _switch_to='female_sterilization' - ) - # - If maintaining IUD => 'FamilyPlanning' - assert_expected_conds_for_appt_footprints( - cond_did_not_run=['len_greater_0_and_all_footprints_equal', {'FamPlan': 1}], cond_did_run=['none'], - _switch_from='IUD', _switch_to='IUD' - ) - # ... otherwise the HSI will run and only once: + # 1) If consumables available, the HSI will only be run once: + # - If switch to female_sterilization => 'MinorSurg'" + assert [{'MinorSurg': 1}] == get_appt_footprints(switch_from='not_using', + switch_to='female_sterilization', + consumables_available=True) # - If switching to anything new => 'FamilyPlanning' - assert_expected_conds_for_appt_footprints( - cond_did_not_run=['len_equal_0'], cond_did_run=['appt_footprint_equal', {'FamPlan': 1}], - _switch_from='not_using', _switch_to='pill' - ) + assert [{'FamPlan': 1}] == get_appt_footprints(switch_from='not_using', + switch_to='pill', + consumables_available=True) # - If maintaining on implant => 'FamilyPlanning' - assert_expected_conds_for_appt_footprints( - cond_did_not_run=['len_equal_0'], cond_did_run=['appt_footprint_equal', {'FamPlan': 1}], - _switch_from='implant', _switch_to='implant' - ) + assert [{'FamPlan': 1}] == get_appt_footprints(switch_from='implant', + switch_to='implant', + consumables_available=True) # - If maintaining on pill => 'PharmDispensing' - assert_expected_conds_for_appt_footprints( - cond_did_not_run=['len_equal_0'], cond_did_run=['appt_footprint_equal', {'PharmDispensing': 1}], - _switch_from='pill', _switch_to='pill' - ) - - # 3) If consumables not available, but equipment available - set_cons_avail = False - set_equip_avail = True - # ... it does run and there should be multiple footprints, but only the first is non-blank: - # - If switch to female_sterilization => 'MinorSurg' - assert_expected_conds_for_appt_footprints( - cond_did_not_run=['none'], - cond_did_run=['len_greater_1_and_first_footprint_equal_subsequent_blank', {'MinorSurg': 1}], - _switch_from='not_using', _switch_to='female_sterilization' - ) - # - If maintaining IUD => 'FamilyPlanning' - assert_expected_conds_for_appt_footprints( - cond_did_not_run=['none'], - cond_did_run=['len_greater_1_and_first_footprint_equal_subsequent_blank', {'FamPlan': 1}], - _switch_from='IUD', _switch_to='IUD' - ) - # - If switching to anything new => 'FamilyPlanning' - assert_expected_conds_for_appt_footprints( - cond_did_not_run=['none'], - cond_did_run=['len_greater_1_and_first_footprint_equal_subsequent_blank', {'FamPlan': 1}], - _switch_from='not_using', _switch_to='pill' - ) - # - If maintaining on implant => 'FamilyPlanning' - assert_expected_conds_for_appt_footprints( - cond_did_not_run=['none'], - cond_did_run=['len_greater_1_and_first_footprint_equal_subsequent_blank', {'FamPlan': 1}], - _switch_from='implant', _switch_to='implant' - ) - # - If maintaining on pill => 'PharmDispensing' - assert_expected_conds_for_appt_footprints( - cond_did_not_run=['none'], - cond_did_run=['len_greater_1_and_first_footprint_equal_subsequent_blank', {'PharmDispensing': 1}], - _switch_from='pill', _switch_to='pill' - ) + assert [{'PharmDispensing': 1}] == get_appt_footprints(switch_from='pill', + switch_to='pill', + consumables_available=True) + + # 2) If consumables not available... there should be multiple footprints, but only the first is non-blank. + def is_list_longer_than_length_of_one_and_with_first_element_nonblank_and_subsequent_blank(x): + return ( + (len(x) > 1) + & (x[0] != {}) + & (0 == len([_x for _i, _x in enumerate(x) if (_i != 0) and (_x != {})])) + ) - # 4) If both consumables and equipment not available, ... - set_cons_avail = False - set_equip_avail = False - # ... when a method with essential equipment (f. sterilization or IUD) requested, the HSI will never run hence no - # appt_footprint will be returned for did_run HSI, but it will be returned for did_not_run HSI multiple times: - # - If switch to female_sterilization => 'MinorSurg' - assert_expected_conds_for_appt_footprints( - cond_did_not_run=['len_greater_0_and_all_footprints_equal', {'MinorSurg': 1}], cond_did_run=['none'], - _switch_from='not_using', _switch_to='female_sterilization' + assert is_list_longer_than_length_of_one_and_with_first_element_nonblank_and_subsequent_blank( + get_appt_footprints(switch_from='not_using', switch_to='female_sterilization', consumables_available=False) ) - # - If maintaining IUD => 'FamilyPlanning' - assert_expected_conds_for_appt_footprints( - cond_did_not_run=['len_greater_0_and_all_footprints_equal', {'FamPlan': 1}], cond_did_run=['none'], - _switch_from='IUD', _switch_to='IUD' + assert is_list_longer_than_length_of_one_and_with_first_element_nonblank_and_subsequent_blank( + get_appt_footprints(switch_from='not_using', switch_to='pill', consumables_available=False) ) - # ... otherwise the HSI will run and there should be multiple footprints, but only the first is non-blank: - # TODO: this is the current logic, but it doesn't sound right, if it is never performed due to missing consumables - # it should never have the footprint equal, or do we expect that they will be coming again and again and again - # and again ... until they will get it in very far future? - - # - If switching to anything new => 'FamilyPlanning' - assert_expected_conds_for_appt_footprints( - cond_did_not_run=['none'], - cond_did_run=['len_greater_1_and_first_footprint_equal_subsequent_blank', {'FamPlan': 1}], - _switch_from='not_using', _switch_to='pill' + assert is_list_longer_than_length_of_one_and_with_first_element_nonblank_and_subsequent_blank( + get_appt_footprints(switch_from='implant', switch_to='implant', consumables_available=False) ) - # - If maintaining on implant => 'FamilyPlanning' - assert_expected_conds_for_appt_footprints( - cond_did_not_run=['none'], - cond_did_run=['len_greater_1_and_first_footprint_equal_subsequent_blank', {'FamPlan': 1}], - _switch_from='implant', _switch_to='implant' - ) - # - If maintaining on pill => 'PharmDispensing' - assert_expected_conds_for_appt_footprints( - cond_did_not_run=['none'], - cond_did_run=['len_greater_1_and_first_footprint_equal_subsequent_blank', {'PharmDispensing': 1}], - _switch_from='pill', _switch_to='pill' + assert is_list_longer_than_length_of_one_and_with_first_element_nonblank_and_subsequent_blank( + get_appt_footprints(switch_from='pill', switch_to='pill', consumables_available=False) ) From cd2774e0fd1cdea503611ebb89a06ac0214686ad Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Thu, 9 May 2024 10:26:54 +0100 Subject: [PATCH 079/118] move `codes_to_items_list` to scripts/data-file-processing --- .../data_file_processing}/codes_to_items_list.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/{tlo/analysis => scripts/data_file_processing}/codes_to_items_list.py (100%) diff --git a/src/tlo/analysis/codes_to_items_list.py b/src/scripts/data_file_processing/codes_to_items_list.py similarity index 100% rename from src/tlo/analysis/codes_to_items_list.py rename to src/scripts/data_file_processing/codes_to_items_list.py From 7a5d991a9c21b084135d68e627df045b4d39ca81 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Thu, 9 May 2024 10:28:07 +0100 Subject: [PATCH 080/118] roll back parsing script --- .../equipment/equipment_catalogue.py | 255 ------------------ 1 file changed, 255 deletions(-) delete mode 100644 src/scripts/healthsystem/equipment/equipment_catalogue.py diff --git a/src/scripts/healthsystem/equipment/equipment_catalogue.py b/src/scripts/healthsystem/equipment/equipment_catalogue.py deleted file mode 100644 index 4dcd533903..0000000000 --- a/src/scripts/healthsystem/equipment/equipment_catalogue.py +++ /dev/null @@ -1,255 +0,0 @@ -import argparse -import warnings -from pathlib import Path - -import pandas as pd - -from tlo.analysis.utils import extract_results - -# TODO: make these to be arguments of called fnc -# %%% TO SET %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -# Declare whether to scale the counts to Malawi population size -# (True/False) -do_scaling = True -# Declare as a list by which hsi event details you want the equipment be grouped in the catalogue (choose any number) -# (event details: 'event_name', 'module_name', 'treatment_id', 'facility_level', 'appt_footprint', 'beddays_footprint') -catalog_by_details = ['treatment_id', 'facility_level'] -# Declare which time period you want the equipment be grouped in the catalogue (choose only one) -# (periods: 'monthly', 'annual') -catalog_by_time = 'annual' -# Suffix for output file names -suffix_file_names = '__2y_2Kpop_4runs_1draw' -# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - -# TODO: Could I have use the bin_hsi_event_details from src/tlo/analysis/utils.py instead? If so, how? -def get_monthly_hsi_event_counts(results_folder: Path) -> pd.DataFrame: - """Returned pd.DataFrame gives the monthly counts of all the hsi event details logged (details as keys) - for each simulated month. - NB. 'healthsystem.summary' logger required to have been set at the level INFO or higher.""" - - def get_hsi_event_counts(_df): - """Get the counts of all the hsi event details logged.""" - - def unpack_dict_in_series(_raw: pd.Series): - # Create an empty DataFrame to store the data - df = pd.DataFrame() - - # Iterate through the dictionary items - for _, mydict in _raw.items(): - for date, inner_dict in mydict.items(): - # Convert the inner_dict to a list of dictionaries with 'date' - data = [{'date': date, 'event_details_key': inner_dict_key, 'count': inner_dict_set} for - inner_dict_key, inner_dict_set in inner_dict.items()] - # Create a DataFrame from the list with date & fac_level as indexes - temp_df = pd.DataFrame(data) - temp_df.set_index(['date', 'event_details_key'], inplace=True) - temp_df.columns = [None] - - # Concatenate the temporary DataFrame to the result DataFrame - df = pd.concat([df, temp_df]) - - df.columns = [None] - - return df - - return _df \ - .set_index('date') \ - .pipe(unpack_dict_in_series) \ - .stack() \ - .droplevel(level=2) - - return extract_results( - results_folder, - module='tlo.methods.healthsystem.summary', - key='hsi_event_counts', - custom_generate_series=get_hsi_event_counts, - do_scaling=do_scaling - ) - - -def get_hsi_event_keys_all_runs(results_folder: Path) -> pd.DataFrame: - """Returned pd.DataFrame gives the dictionaries of hsi_event_details for each draw and run. - NB. 'healthsystem.summary' logger required to have been set at the level INFO or higher.""" - - def get_hsi_event_keys(_df): - """Get the hsi_event_keys for one particular run.""" - return _df['hsi_event_key_to_event_details'] - - return extract_results( - results_folder, - module='tlo.methods.healthsystem.summary', - key='hsi_event_details', - custom_generate_series=get_hsi_event_keys - ) - - -def create_equipment_catalogues(results_folder: Path, output_folder: Path): - # %%% Verify inputs are as expected - assert isinstance(do_scaling, bool), "The input parameter 'do_scaling' must be a boolean (True or False)" - assert isinstance(catalog_by_details, list), "The input parameter 'catalog_by_details' must be a list" - event_details = \ - {'event_name', 'module_name', 'treatment_id', 'facility_level', 'appt_footprint', 'beddays_footprint'} - for item in catalog_by_details: - assert isinstance(item, str) and item in event_details, \ - f"Each element in the input list 'catalog_by_details' must be a string and be one of the details:\n" \ - f"{event_details}" - assert catalog_by_time in {'monthly', 'annual'}, \ - "The input parameter 'catalog_by_time' must be one of the strings ('monthly' or 'annual')" - # --- - - # %%% Set output file names - # detailed CSV name - output_detailed_file_name = 'equipment_monthly_counts__all_event_details' + suffix_file_names + '.csv' - # requested details only CSV name - time_index = 'year' if catalog_by_time == 'annual' else 'date' - output_focused_file_name = \ - 'equipment_' + catalog_by_time + '_counts__by_' + time_index + '_' + '_'.join(catalog_by_details) + \ - suffix_file_names + '.csv' - output_summary_file_name = 'equipment_summary__module_name_event_name_treatment_id' + suffix_file_names + '.csv' - # --- - - # %%% Load RF - # Equipment - equip_resource_items_pkgs_df = pd.read_csv( - 'resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv' - ) - - # %% Catalog equipment by all HSI event details - sim_equipment = get_monthly_hsi_event_counts(results_folder) - sim_equipment_df = pd.DataFrame(sim_equipment) - hsi_event_keys = get_hsi_event_keys_all_runs(results_folder) - - final_df = pd.DataFrame() - - def details_col_to_str(details_col): - return details_col.apply(lambda x: ', '.join(map(str, x))) - - def get_equip_item_name_from_item_code(equip_item_code: int) -> str: - """Helper function to provide the equip item name (a string) when provided with the equip_item_code (an int).""" - lookup_df = equip_resource_items_pkgs_df - return str(pd.unique(lookup_df.loc[lookup_df["Equip_Code"] == equip_item_code, "Equip_Item"])[0]) - - def get_equip_item_code_from_item_name(equip_item_name: str) -> int: - """Helper function to provide the equip item code (an int) when provided with the equip_item_name (a string)""" - lookup_df = equip_resource_items_pkgs_df - return int(pd.unique(lookup_df.loc[lookup_df["Equip_Item"] == equip_item_name, "Equip_Code"])[0]) - - def lists_of_equip_item_codes_to_strings_of_list_of_equip_item_names(list_of_equip_item_codes_col): - return list_of_equip_item_codes_col.apply( - lambda x: - str(sorted([get_equip_item_name_from_item_code(item_code) for item_code in x])) - ) - - def strings_of_list_to_lists_of_strings(strings_of_list_col): - lists_of_strings_col = strings_of_list_col.apply(lambda x: x.strip('][').split("'")) - return lists_of_strings_col.apply(lambda x: [s for s in x if (s != '' and s != ', ')]) - - for col in hsi_event_keys.columns: - df_col = sim_equipment_df[col].dropna() - decoded_keys = df_col.index.get_level_values(1).astype(str).map(hsi_event_keys.at[0, col]) - - # %%% Verify the keys in dictionary and dataframe for the run 'col' are same - # Check if all keys in hsi_event_keys_set are in the 'event_details_key' of df_col - hsi_event_keys_set = set(hsi_event_keys.at[0, col].keys()) - missing_keys_df =\ - [key for key in hsi_event_keys_set if key not in df_col.index.get_level_values('event_details_key')] - - # Check if all keys in the 'event_details_key' of df_col are in hsi_event_keys_set - missing_keys_dict =\ - [key for key in df_col.index.get_level_values('event_details_key') if key not in hsi_event_keys_set] - - # Warn if some keys are missing - if missing_keys_df: - warnings.warn(UserWarning(f"Keys missing in sim_equipment_df for the run {col}: {missing_keys_df}")) - - if missing_keys_dict: - warnings.warn(UserWarning(f"Keys missing in hsi_event_keys for the run {col}: {missing_keys_dict}")) - # %%% - - df_col = pd.concat([df_col, pd.DataFrame(decoded_keys.tolist(), index=df_col.index)], axis=1) - # Make values in 'appt_footprint', 'beddays_footprint', and 'equipment' columns to be string - df_col['appt_footprint'] = details_col_to_str(df_col['appt_footprint']) - df_col['beddays_footprint'] = details_col_to_str(df_col['beddays_footprint']) - df_col['equipment'] = lists_of_equip_item_codes_to_strings_of_list_of_equip_item_names(df_col['equipment']) - df_col = (df_col.droplevel(level=1) - .set_index(['module_name', 'event_name', 'treatment_id', 'facility_level', 'appt_footprint', - 'beddays_footprint', 'equipment'], append=True)) - final_df = pd.concat([final_df, df_col], axis=1) - - # Replace NaN with 0 - final_df.fillna(0, inplace=True) - final_df.sort_index(inplace=True) - # Save the detailed equipment catalogue - final_df.to_csv(output_folder / output_detailed_file_name) - print(f'{output_detailed_file_name} saved.') - # --- - - # %% Catalog equipment summary - equipment_summary = final_df.copy() - equipment_summary = equipment_summary.groupby(['module_name', 'event_name', 'treatment_id', 'equipment']).sum() - equipment_summary = \ - equipment_summary.reset_index().set_index(['module_name', 'event_name', 'treatment_id', 'equipment']) - # Save the summary equipment catalogue - equipment_summary.index.to_frame().to_csv(output_folder / output_summary_file_name, index=False) - print(f'{output_summary_file_name} saved.') - # --- - - # %% Catalog equipment by requested details - equipment_counts_by_time_and_requested_details = final_df.copy() - - # Sum counts for each equipment set with the same date, treatment id, and facility level - # (remaining indexes removed), keeping only non-empty 'equipment' indexes - to_be_grouped_by = ['date'] + catalog_by_details + ['equipment'] - equipment_counts_by_time_and_requested_details = equipment_counts_by_time_and_requested_details.groupby( - to_be_grouped_by, - dropna=True - ).sum() - - if catalog_by_time == 'annual': - # Sum counts annually - equipment_counts_by_time_and_requested_details['year'] = \ - equipment_counts_by_time_and_requested_details.index.get_level_values('date').year - equipment_counts_by_time_and_requested_details.set_index('year', append=True, inplace=True) - equipment_counts_by_time_and_requested_details.index.droplevel('date') - to_be_grouped_by = ['year'] + catalog_by_details + ['equipment'] - equipment_counts_by_time_and_requested_details = equipment_counts_by_time_and_requested_details.groupby( - to_be_grouped_by - ).sum() - - # Remove rows with no equipment used - equipment_counts_by_time_and_requested_details.drop("[]", level='equipment', axis=0, inplace=True) - equipment_counts_by_time_and_requested_details['equipment'] = \ - equipment_counts_by_time_and_requested_details.index.get_level_values('equipment') - equipment_counts_by_time_and_requested_details.index = \ - equipment_counts_by_time_and_requested_details.index.droplevel('equipment') - equipment_counts_by_time_and_requested_details['equipment'] = strings_of_list_to_lists_of_strings( - equipment_counts_by_time_and_requested_details['equipment'] - ) - exploded_df = equipment_counts_by_time_and_requested_details.explode('equipment') - # Add column with equip item code - exploded_df['equip_code'] = exploded_df['equipment'].apply(lambda x: get_equip_item_code_from_item_name(x)) - exploded_df = exploded_df.set_index(['equipment', 'equip_code'], append=True) - - # Sum values with the same multi-index - exploded_df = exploded_df.groupby(level=exploded_df.index.names).sum() - - # Save the equipment counts CSV - exploded_df.to_csv(output_folder / output_focused_file_name) - print(f'{output_focused_file_name} saved.') - # --- - - return 0 - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("results_folder", type=Path) - args = parser.parse_args() - - create_equipment_catalogues( - results_folder=args.results_folder, - output_folder=args.results_folder, - ) -# NB. Edit run configuration, the Parameters: -# "./outputs/sejjej5@ucl.ac.uk/equip_jobs/long_run_all_diseases-2023-09-04T233551Z" From 478155510ab70b82e967ebbb89f39d88ef6611a0 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Thu, 9 May 2024 21:14:36 +0100 Subject: [PATCH 081/118] squash - basic outline of Equipment class --- .../ResourceFile_Equipment.csv | 3 - .../ResourceFile_EquipmentCatalogue.csv | 3 + ...eFile_Equipment_Availability_Estimates.csv | 3 + src/tlo/analysis/utils.py | 1 + src/tlo/methods/equipment.py | 204 ++++++++++++ src/tlo/methods/healthsystem.py | 159 ++++++---- src/tlo/methods/hsi_event.py | 161 ++-------- tests/test_equipment.py | 295 ++++++++++++------ 8 files changed, 550 insertions(+), 279 deletions(-) delete mode 100644 resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv create mode 100644 resources/healthsystem/infrastructure_and_equipment/ResourceFile_EquipmentCatalogue.csv create mode 100644 resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment_Availability_Estimates.csv create mode 100644 src/tlo/methods/equipment.py diff --git a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv deleted file mode 100644 index 5700c76b27..0000000000 --- a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment.csv +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:20c07f76ec100a3c1a221c4c8ecc4bd6e37379f395b08ae67491501d04f04d6e -size 45343 diff --git a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_EquipmentCatalogue.csv b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_EquipmentCatalogue.csv new file mode 100644 index 0000000000..2f3ca59328 --- /dev/null +++ b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_EquipmentCatalogue.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d21fb0f546325fd264b0598efd7afbea15ad48564ed28cb9df970509ce1d405 +size 33196 diff --git a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment_Availability_Estimates.csv b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment_Availability_Estimates.csv new file mode 100644 index 0000000000..829b95d1f9 --- /dev/null +++ b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment_Availability_Estimates.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7c0364af516509b47f161278a061817d22bcd06685f469c09be089454c02a22 +size 582777 diff --git a/src/tlo/analysis/utils.py b/src/tlo/analysis/utils.py index 3aeff2bb11..36cee558d4 100644 --- a/src/tlo/analysis/utils.py +++ b/src/tlo/analysis/utils.py @@ -1125,6 +1125,7 @@ def get_parameters_for_status_quo() -> Dict: "mode_appt_constraints": 1, "cons_availability": "default", "beds_availability": "default", + "equip_availability": "all", # <--- NB. Existing calibration is assuming all equipment is available }, } diff --git a/src/tlo/methods/equipment.py b/src/tlo/methods/equipment.py new file mode 100644 index 0000000000..2bb1f15267 --- /dev/null +++ b/src/tlo/methods/equipment.py @@ -0,0 +1,204 @@ +import warnings +from collections import defaultdict +from typing import Dict, Set, Iterable, Union, Counter, Optional, Any + +import numpy as np +import pandas as pd + +from tlo import logging + +logger_summary = logging.getLogger("tlo.methods.healthsystem.summary") + + +class Equipment: + """This is the Equipment Class. It maintains a current record of the availability of equipment in the + HealthSystem. It is expected that this is instantiated by the `HealthSystem` module. + + :param: 'catalogue': The database of all recognised item_codes. + + :param: `data_availability`: Specifies the probability with which each equipment (identified by an `item_code`) is + available at a facility level. Note that information is not necessarily provided for every item in the `catalogue`. + + :param: `rng`: The Random Number Generator object to use for random numbers. + + :param: `availability`: Determines the mode availability of the equipment. If 'default' then use the availability + specified in the ResourceFile; if 'none', then let no equipment be ever be available; if 'all', then all + equipment is always available. + + If an item_code is requested that is not recognised (not included in `data`), a `UserWarning` is issued, and the + result returned is on the basis of the average availability of other consumables in that facility in that month. + """ + + def __init__( + self, + catalogue: pd.DataFrame, + data_availability: pd.DataFrame, + rng: np.random, + master_facilities_list: pd.DataFrame, + availability: Optional[str] = "default", + ) -> None: + # Store arguments + self.catalogue = catalogue + self.rng = rng + self.data_availability = data_availability + self.master_facilities_list = master_facilities_list + + # Create internal storage structures + self._items_available: Dict = dict() # Will be the internal store of which items are available at each + # facility_id. This is of the form {facility_id: {items_available}}. + + self._record_of_equipment_used = defaultdict(Counter) # Will be the internal store of which items have been + # used at each facility_id. This is of the form + # {facility_id: {item_code: count}}. + + # Set up the internal stores of equipment items that are available, ready for calls. + self._set_equipment_items_available(availability=availability) + + # Set up internal lookup for item_descriptor -> item_code + self.item_code_lookup = self.catalogue.set_index('Description')['Item_Code'].to_dict() + + def on_simulation_end(self): + """Things to do when the simulation end: + * Log (to the summary logger) the equipment that has been used. + """ + self.write_to_log() + + def update_availability(self, availability) -> None: + """Update the availability of equipment. This is expected to be called midway through the simulation if + the assumption of the equipment availability needs to change.""" + self._set_equipment_items_available(availability=availability) + + def _set_equipment_items_available(self, availability: str): + """Update internal store of which items of equipment are available. This is called at the beginning of the + simulation and whenever an update in `availability` is needed.""" + + # For any facility_id in the data + all_fac_ids = self.master_facilities_list['Facility_ID'].unique() + + # All equipment items in the catalogue + all_eq_items = self.catalogue["Item_Code"].unique() + + # Create full dataset, where we force that there is probability of availability for every item_code at every + # observed facility + df = pd.Series( + index=pd.MultiIndex.from_product( + [all_fac_ids, all_eq_items], names=["Facility_ID", "Item_Code"] + ), + data=float("nan"), + ).combine_first( + self.data_availability.set_index(["Facility_ID", "Item_Code"])[ + "Pr_Available" + ] + ) + + # Merge in original dataset and use the mean in that facility_id to impute availability of missing item_code + df = df.groupby("Facility_ID").transform(lambda x: x.fillna(x.mean())) + # ... and also impute availability for any facility_ids for which no data, based on all other facilities + df = df.groupby("Item_Code").transform(lambda x: x.fillna(x.mean())) + + # Check no missing values + assert not df.isnull().any() + + # Over-write these data if `availability` argument specifies that `none` or `all` items should be available + if availability == "default": + pass + elif availability == "all": + df = (df + 1).clip(upper=1.0) + elif availability == "none": + df = df.mul(0.0) + else: + raise KeyError(f"Unknown equipment availability specified: {availability}") + + # Sample these probability to find which items are actually available + is_available = df > self.rng.random(size=len(df)) + + # Organise into dict of set, of the form: {facility_id: {items_available}} for known facility_ids + # (N.B. Has to be done this way around in order to guarantee that we have each known facility_id in the keys + # even if there are no item available.) + self._items_available: Dict = is_available.groupby("Facility_ID").agg( + lambda x: set(x[x].index.get_level_values("Item_Code")) + ).to_dict() + + def _parse_items(self, items: Union[int, str, Iterable[int | str]]) -> Set[int]: + """Parse equipment items specified as an item_code (integer), an item descriptor (string), or an iterable of + either, and return as a set of item_code (integers).""" + + def first_element_in_iterable(it: Iterable[Any]) -> Any | None: + for el in it: + return el + + if isinstance(items, str): + # Single descriptor + return set([self.item_code_lookup(items)]) + elif isinstance(items, int): + # Single item_code provided + return set([items]) + elif isinstance(items, Iterable) and len(items) == 0: + # Iterable of length 0 + return set() + elif isinstance(items, Iterable) and isinstance(first_element_in_iterable(items), str): + # Iterable of descriptors + return set(map(self.item_code_lookup, items)) + elif isinstance(items, Iterable) and isinstance(first_element_in_iterable(items), int): + # Iterable of item_cods + return set(items) + + else: + raise ValueError(f'Item_Code format not recognised: {items=}') + + if isinstance(items, int): + item_codes = set([items]) # If single int provided, place it into a list + elif isinstance(items, list): + item_codes = set(items) + + + def is_all_items_available( + self, item_codes: Union[int, str, Iterable[int | str]], facility_id: int + ) -> bool: + """Determine if all equipments are available at the given facility_id (or from the default if the faciluty_id + is not recognised). Returns True only if all items are available at the facility_id, otherwise returns False.""" + return self._parse_items(item_codes).issubset(self._items_available[facility_id]) + + + def record_use_of_equipment( + self, item_codes: Union[int, str, Iterable[int | str]], facility_id: int + ) -> None: + """Update internal record of the usage of items at equipment at the specified facility_id.""" + self._record_of_equipment_used[facility_id].update(self._parse_items(item_codes)) + + def write_to_log(self) -> None: + """Write to the log: + * Summary of the equipment that was _ever_ used at each facility_level + * For each facility_id, a set of the equipment items ever used. + """ + + mfl = self.master_facilities_list + + def set_of_keys_or_empty_set(x: Union[set, dict]): + if isinstance(x, set): + return x + elif isinstance(x, dict): + return set(x.keys()) + else: + return None + + set_of_equipment_ever_used_at_each_facility_id = pd.Series({ + fac_id: set_of_keys_or_empty_set(self._record_of_equipment_used.get(fac_id, set())) + for fac_id in mfl['Facility_ID'] + }, name='EquipmentEverUsed').astype(str) + + output = mfl.merge( + set_of_equipment_ever_used_at_each_facility_id, + left_on='Facility_ID', + right_index=True, + how='left', + ).drop(columns=['Facility_ID', 'Facility_Name']) + + # Log multi-row data-frame + for _, row in output.iterrows(): + logger_summary.info( + key='EquipmentEverUsed_ByFacilityID', + description='For each facility_id (the set of facilities of the same level in a district), the set' + 'equipment items that are ever used.', + data=row.to_dict(), + ) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 62e9f2fe01..a0a3e6b96f 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -27,6 +27,7 @@ get_item_codes_from_package_name, ) from tlo.methods.dxmanager import DxManager +from tlo.methods.equipment import Equipment from tlo.methods.hsi_event import ( LABEL_FOR_MERGED_FACILITY_LEVELS_1B_AND_2, FacilityInfo, @@ -193,13 +194,17 @@ class HealthSystem(Module): "Availability of beds. If 'default' then use the availability specified in the ResourceFile; if " "'none', then let no beds be ever be available; if 'all', then all beds are always available. NB. This " "parameter is over-ridden if an argument is provided to the module initialiser."), - 'Equipment': Parameter( - Types.DATA_FRAME, "Data on equipment items, packages, and availability probabilities by facility level."), + 'EquipmentCatalogue': Parameter( + Types.DATA_FRAME, "Data on equipment items and packages."), + 'equipment_availability_estimates': Parameter( + Types.DATA_FRAME, "Data on the availability of equipment items and packages." + ), 'equip_availability': Parameter( Types.STRING, - "Availability of equipment. If 'default' then use the availability specified in the ResourceFile;" - " if 'none', then let no equipment ever be available; if 'all', then all equipment is always available. NB." - " This parameter is over-ridden if an argument is provided to the module initialiser."), + "What to assume about the availability of equipment. If 'default' then use the availability specified in " + "the ResourceFile; if 'none', then let no equipment ever be available; if 'all', then all equipment is " + "always available. NB. This parameter is over-ridden if an argument is provided to the module initialiser." + ), # Service Availability 'Service_Availability': Parameter( @@ -542,8 +547,14 @@ def read_parameters(self, data_folder): path_to_resourcefiles_for_healthsystem / 'infrastructure_and_equipment' / 'ResourceFile_Bed_Capacity.csv') # Read in ResourceFile_Equipment - self.parameters['Equipment'] = pd.read_csv( - path_to_resourcefiles_for_healthsystem / 'infrastructure_and_equipment' / 'ResourceFile_Equipment.csv') + self.parameters['EquipmentCatalogue'] = pd.read_csv( + path_to_resourcefiles_for_healthsystem + / 'infrastructure_and_equipment' + / 'ResourceFile_EquipmentCatalogue.csv') + self.parameters['equipment_availability_estimates'] = pd.read_csv( + path_to_resourcefiles_for_healthsystem + / 'infrastructure_and_equipment' + / 'ResourceFile_Equipment_Availability_Estimates.csv') # Data on the priority of each Treatment_ID that should be adopted in the queueing system according to different # priority policies. Load all policies at this stage, and decide later which one to adopt. @@ -600,6 +611,7 @@ def pre_initialise_population(self): self.rng_for_hsi_queue = np.random.RandomState(self.rng.randint(2 ** 31 - 1)) self.rng_for_dx = np.random.RandomState(self.rng.randint(2 ** 31 - 1)) rng_for_consumables = np.random.RandomState(self.rng.randint(2 ** 31 - 1)) + rng_for_equipment = np.random.RandomState(self.rng.randint(2 ** 31 - 1)) # Determine mode_appt_constraints self.mode_appt_constraints = self.get_mode_appt_constraints() @@ -625,8 +637,13 @@ def pre_initialise_population(self): ) # Determine equip_availability - # todo - create Equipment class here - self.equip_availability = self.get_equip_availability() + self.equipment = Equipment( + catalogue=self.parameters['EquipmentCatalogue'], + data_availability=self.parameters['equipment_availability_estimates'], + rng=rng_for_equipment, + master_facilities_list=self.parameters['Master_Facilities_List'], + availability=self.get_equip_availability(), + ) self.tclose_overwrite = self.parameters['tclose_overwrite'] self.tclose_days_offset_overwrite = self.parameters['tclose_days_offset_overwrite'] @@ -713,6 +730,8 @@ def on_simulation_end(self): """Put out to the log the information from the tracker of the last day of the simulation""" self.bed_days.on_simulation_end() self.consumables.on_simulation_end() + self.equipment.on_simulation_end() + if self._hsi_event_count_log_period == "simulation": self._write_hsi_event_counts_to_log_and_reset() self._write_never_ran_hsi_event_counts_to_log_and_reset() @@ -2169,9 +2188,19 @@ def process_events_mode_0_and_1(self, hold_over: List[HSIEventQueueItem]) -> Non # Run the list of population-level HSI events self.module.run_population_level_events(list_of_population_hsi_event_tuples_due_today) - # Run the list of individual-level events + # For each individual level event, check whether the equipment it has already declared is available. If it + # is not, then call the HSI's never_run function, and do not take it forward for running; if it is then + # add it to the list of events to run. + list_of_individual_hsi_event_tuples_due_today_that_have_essential_equipment = list() + for item in list_of_individual_hsi_event_tuples_due_today: + if not item.hsi_event.is_all_declared_equipment_available: + self.module.call_and_record_never_ran_hsi_event(hsi_event=item.hsi_event, priority=item.priority) + else: + list_of_individual_hsi_event_tuples_due_today_that_have_essential_equipment.append(item) + + # Try to run the list of individual-level events that have their essential equipment _to_be_held_over = self.module.run_individual_level_events_in_mode_0_or_1( - list_of_individual_hsi_event_tuples_due_today, + list_of_individual_hsi_event_tuples_due_today_that_have_essential_equipment, ) hold_over.extend(_to_be_held_over) @@ -2198,7 +2227,6 @@ def process_events_mode_2(self, hold_over: List[HSIEventQueueItem]) -> None: list_of_population_hsi_event_tuples_due_today = list() list_of_events_not_due_today = list() - # todo - check if essential equipment available and do not run if not - in any mode # Traverse the queue and run events due today until have capabilities still available while len(self.module.HSI_EVENT_QUEUE) > 0: @@ -2308,58 +2336,66 @@ def process_events_mode_2(self, hold_over: List[HSIEventQueueItem]) -> None: assert event.facility_info is not None, \ f"Cannot run HSI {event.TREATMENT_ID} without facility_info being defined." - # Expected appt footprint before running event - _appt_footprint_before_running = event.EXPECTED_APPT_FOOTPRINT - # Run event & get actual footprint - actual_appt_footprint = event.run(squeeze_factor=squeeze_factor) + # Check if equipment declared is available. If not, call `never_ran` and do not run the + # event. + if not event.is_all_declared_equipment_available: + self.module.call_and_record_never_ran_hsi_event( + hsi_event=event, + priority=next_event_tuple.priority + ) + else: + # Expected appt footprint before running event + _appt_footprint_before_running = event.EXPECTED_APPT_FOOTPRINT + # Run event & get actual footprint + actual_appt_footprint = event.run(squeeze_factor=squeeze_factor) - # Check if the HSI event returned updated_appt_footprint, and if so adjust original_call - if actual_appt_footprint is not None: + # Check if the HSI event returned updated_appt_footprint, and if so adjust original_call + if actual_appt_footprint is not None: - # check its formatting: - assert self.module.appt_footprint_is_valid(actual_appt_footprint) + # check its formatting: + assert self.module.appt_footprint_is_valid(actual_appt_footprint) - # Update call that will be used to compute capabilities used - updated_call = self.module.get_appt_footprint_as_time_request( - facility_info=event.facility_info, - appt_footprint=actual_appt_footprint + # Update call that will be used to compute capabilities used + updated_call = self.module.get_appt_footprint_as_time_request( + facility_info=event.facility_info, + appt_footprint=actual_appt_footprint + ) + else: + actual_appt_footprint = _appt_footprint_before_running + updated_call = original_call + + # Recalculate call on officers based on squeeze factor. + for k in updated_call.keys(): + updated_call[k] = updated_call[k]/(squeeze_factor + 1.) + + # Subtract this from capabilities used so-far today + capabilities_monitor.subtract(updated_call) + + # If any of the officers have run out of time by performing this hsi, + # remove them from list of available officers. + for officer, call in updated_call.items(): + if capabilities_monitor[officer] <= 0: + if officer in set_capabilities_still_available: + set_capabilities_still_available.remove(officer) + else: + logger.warning( + key="message", + data=(f"{event.TREATMENT_ID} actual_footprint requires different" + f"officers than expected_footprint.") + ) + + # Update today's footprint based on actual call and squeeze factor + self.module.running_total_footprint -= original_call + self.module.running_total_footprint += updated_call + + # Write to the log + self.module.record_hsi_event( + hsi_event=event, + actual_appt_footprint=actual_appt_footprint, + squeeze_factor=squeeze_factor, + did_run=True, + priority=_priority ) - else: - actual_appt_footprint = _appt_footprint_before_running - updated_call = original_call - - # Recalculate call on officers based on squeeze factor. - for k in updated_call.keys(): - updated_call[k] = updated_call[k]/(squeeze_factor + 1.) - - # Subtract this from capabilities used so-far today - capabilities_monitor.subtract(updated_call) - - # If any of the officers have run out of time by performing this hsi, - # remove them from list of available officers. - for officer, call in updated_call.items(): - if capabilities_monitor[officer] <= 0: - if officer in set_capabilities_still_available: - set_capabilities_still_available.remove(officer) - else: - logger.warning( - key="message", - data=(f"{event.TREATMENT_ID} actual_footprint requires different" - f"officers than expected_footprint.") - ) - - # Update today's footprint based on actual call and squeeze factor - self.module.running_total_footprint -= original_call - self.module.running_total_footprint += updated_call - - # Write to the log - self.module.record_hsi_event( - hsi_event=event, - actual_appt_footprint=actual_appt_footprint, - squeeze_factor=squeeze_factor, - did_run=True, - priority=_priority - ) # Don't have any capabilities at all left for today, no # point in going through the queue to check what's left to do today. @@ -2714,8 +2750,7 @@ def apply(self, population): self.module.bed_days.availability = self._parameters['beds_availability'] if 'equip_availability' in self._parameters: - # todo - is this being directed to right place? - self.module.equip_availability = self._parameters['equip_availability'] + self.module.equipment.update_availability(self._parameters['equip_availability']) class DynamicRescalingHRCapabilities(RegularEvent, PopulationScopeEventMixin): diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index b2c20ea7a9..8598cb1294 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -1,10 +1,9 @@ from __future__ import annotations from collections import Counter -from typing import TYPE_CHECKING, Dict, Literal, NamedTuple, Optional, Set, Tuple, Union +from typing import TYPE_CHECKING, Dict, Literal, NamedTuple, Optional, Set, Tuple, Union, Iterable import numpy as np -import pandas as pd from tlo import Date, logging from tlo.events import Event @@ -79,7 +78,7 @@ class HSI_Event: """ module: Module - target: int # Will be overwritten by the mixin on derived classes + target: int # Will be overwritten by the mixin on derived classes TREATMENT_ID: str ACCEPTED_FACILITY_LEVEL: str @@ -87,6 +86,12 @@ class HSI_Event: # which have been loaded. BEDDAYS_FOOTPRINT: Dict[str, Union[float, int]] + _EQUIPMENT: Set[int] = set() # The set of equipment that is used in the HSI. If any items in this set are not + # available at the point when the HSI will be run, then the HSI is not run, and the + # `never_ran` method is called instead. This is a declaration of resource needs, but + # is private because users are expected to use `add_equipment` to declare equipment + # needs. + _received_info_about_bed_days: Dict[str, Union[float, int]] = None expected_time_requests: Counter = {} facility_info: FacilityInfo = None @@ -109,7 +114,7 @@ def __init__(self, module, *args, **kwargs): self.expected_time_requests = {} self.facility_info = None self.ESSENTIAL_EQUIPMENT = None - self.EQUIPMENT = set() + self.EQUIPMENT = set() # todo should this be private, and should we add setter methods self.TREATMENT_ID = "" self.ACCEPTED_FACILITY_LEVEL = None @@ -266,125 +271,25 @@ def make_appt_footprint(self, dict_of_appts) -> Counter: "values" ) - def get_equip_item_code_from_item_name(self, equip_item_name: str) -> int: - """Helper function to provide the equip_item_code (an int) when provided with the equip_item_name of the item""" - lookup_df = self.sim.modules['HealthSystem'].parameters['Equipment'] - return int(pd.unique(lookup_df.loc[lookup_df["Equip_Item"] == equip_item_name, "Equip_Code"])[0]) - - def get_equip_item_codes_from_pkg_name(self, equip_pkg_name: str) -> Set[int]: - """Helper function to provide the equip_item_codes (a set of ints) when provided with the equip_pkg_name of the - equipment package""" - lookup_df = self.sim.modules['HealthSystem'].parameters['Equipment'] - return set(lookup_df.loc[lookup_df["Equip_Pkg"] == equip_pkg_name, "Equip_Code"]) - - def ignore_unknown_equip_names(self, set_of_names: Set[str], type_in_set: str) -> Set[str]: - """Helper function to check if the equipment item or pkg names (depending on type_in_set: 'item' or 'pkg') from - the provided set are in the RF_Equipment. If they are not, they are added to a set to be warned about at the end - of the simulation. - - Only known (item or pkg) names are returned.""" - if set_of_names in [set(), None, {''}]: - return set() - - def add_unknown_names_to_dict(unknown_names_to_add: Set[str], dict_to_be_added_to: Dict) -> Dict: - if self.__class__.__name__ not in dict_to_be_added_to.keys(): - dict_to_be_added_to.update( - {self.__class__.__name__: unknown_names_to_add} - ) - else: - dict_to_be_added_to[self.__class__.__name__].update( - unknown_names_to_add - ) - return dict_to_be_added_to - - lookup_df = self.sim.modules['HealthSystem'].parameters['Equipment'] - if type_in_set == "item": - unknown_names = set_of_names.difference(set(lookup_df["Equip_Item"])) - if unknown_names: - self.sim.modules['HealthSystem']._equip_items_missing_in_RF = \ - add_unknown_names_to_dict( - unknown_names, self.sim.modules['HealthSystem']._equip_items_missing_in_RF - ) - - elif type_in_set == "pkg": - unknown_names = set_of_names.difference(set(lookup_df["Equip_Pkg"])) - if unknown_names: - self.sim.modules['HealthSystem']._equip_pkgs_missing_in_RF = \ - add_unknown_names_to_dict( - unknown_names, self.sim.modules['HealthSystem']._equip_pkgs_missing_in_RF - ) - # TODO: What happens if all equip in set_of_names has unknown name? - return set_of_names.difference(unknown_names) - - def set_equipment_essential_to_run_event(self, set_of_equip: Set[str]) -> None: - """Helper function to set essential equipment. - - Should be passed a set of equipment items names (strings) or an empty set. - """ - # Set EQUIPMENT if the given set_of_equip in correct format, ie a set of strings or an empty set - if not isinstance(set_of_equip, set) or any(not isinstance(item, str) for item in set_of_equip): - raise ValueError( - "Argument to set_equipment_essential_to_run_event should be an empty set or a set of strings of " - "equipment item names from ResourceFile_Equipment.csv." - ) - - set_of_equip = self.ignore_unknown_equip_names(set_of_equip, "item") - if set_of_equip: - equip_codes = set(self.get_equip_item_code_from_item_name(item_name) for item_name in set_of_equip) - self.ESSENTIAL_EQUIPMENT = equip_codes - else: - self.ESSENTIAL_EQUIPMENT = set() - - # todo add function to set essential equipment + def add_equipment(self, item_codes: Union[int, str, Iterable[int | str]]): + """Declare that piece(s) of equipment are used in this HSI_Event.""" + self._EQUIPMENT.update(self.healthcare_system.equipment._parse_items(item_codes)) - def add_equipment(self, set_of_equip: Set[str]) -> None: - """Helper function to update equipment. - - Should be passed a set of equipment item names (strings). - """ - # Update EQUIPMENT if the given set_of_equip in correct format, ie a non-empty set of strings - if ( - (not isinstance(set_of_equip, set)) - or - any(not isinstance(item, str) for item in set_of_equip) - or - (set_of_equip in [set(), None, {''}]) - ): - raise ValueError( - "Argument to add_equipment should be a non-empty set of strings of " - "equipment item names from ResourceFile_Equipment.csv." - ) - # from the set of equip item names create a set of equip item codes, ignore unknown equip names - # (ie not included in RF_Equipment) - set_of_equip = self.ignore_unknown_equip_names(set_of_equip, "item") - if set_of_equip: - equip_codes = set(self.get_equip_item_code_from_item_name(item_name) for item_name in set_of_equip) - self.EQUIPMENT.update(equip_codes) - - def add_equipment_from_pkg(self, set_of_pkgs: Set[str]) -> None: - """Helper function to update equipment with equipment from pkg(s). + @property + def is_all_declared_equipment_available(self) -> bool: + """Returns True if all the declared items of equipment are available. This is called before the HSI is run + so is looking only at those items that are declared when this instance was created.""" + return self.healthcare_system.equipment.is_all_items_available( + item_codes=self._EQUIPMENT, + facility_id=self.facility_info.id, + ) - Should be passed a set of equipment pkgs names (strings). - """ - # Update EQUIPMENT if the given set_of_pkgs in correct format, ie a non-empty set of strings - if not isinstance(set_of_pkgs, set) or any(not isinstance(item, str) for item in set_of_pkgs) or \ - (set_of_pkgs in [set(), None, {''}]): - raise ValueError( - "Argument to add_equipment_from_pkg should be a non-empty set of strings of " - "equipment pkg names from ResourceFile_Equipment.csv." - ) - # update EQUIPMENT with eqip item codes from equip pkgs with provided names, ignore unknown equip names - # (ie not included in RF_Equipment) - set_of_pkgs = self.ignore_unknown_equip_names(set_of_pkgs, "pkg") - if set_of_pkgs: - for pkg_name in set_of_pkgs: - self.EQUIPMENT.update(self.get_equip_item_codes_from_pkg_name(pkg_name)) - - def get_essential_equip_availability(self, set_of_pkgs: Set[str]) -> bool: - # TODO: Or, should it be called set_essential_equip_and_get_availability to be more transparent about what the - # fnc does? - self.set_equipment_essential_to_run_event(set_of_pkgs) - return self.sim.modules['HealthSystem'].get_essential_equip_availability(self.ESSENTIAL_EQUIPMENT) + def is_equipment_available(self, item_codes: Union[int, Iterable[int]]) -> bool: + """Check whether all the equipment item_codes are available.""" + return self.healthcare_system.equipment.is_all_items_available( + item_codes=item_codes, + facility_id=self.facility_info.id, + ) def initialise(self) -> None: """Initialise the HSI: @@ -422,11 +327,6 @@ def initialise(self) -> None: # Do checks self._check_if_appt_footprint_can_run() - # Set essential equip to empty set if not exists and warn about missing settings - if self.ESSENTIAL_EQUIPMENT is None: - self.set_equipment_essential_to_run_event({''}) - self.sim.modules['HealthSystem']._hsi_event_names_missing_ess_equip.update({self.__class__.__name__}) - def _check_if_appt_footprint_can_run(self) -> bool: """Check that event (if individual level) is able to run with this configuration of officers (i.e. check that this does not demand officers that are _never_ available), and issue warning if not. @@ -499,7 +399,14 @@ def as_namedtuple( beddays_footprint=tuple( sorted((k, v) for k, v in self.BEDDAYS_FOOTPRINT.items() if v > 0) ), - equipment=(tuple(sorted(self.EQUIPMENT))), + equipment=(tuple(sorted(self.EQUIPMENT))), # todo check if this is defined in HSIEventDetails and whether we want this here + ) + + def post_apply_hook(self): + """Record the equipment that has been added before and during the course of the HSI Event.""" + self.healthcare_system.equipment.record_use_of_equipment( + item_codes=self._EQUIPMENT, + facility_id=self.facility_info.id ) diff --git a/tests/test_equipment.py b/tests/test_equipment.py index 3d15294c9a..96faf660f8 100644 --- a/tests/test_equipment.py +++ b/tests/test_equipment.py @@ -1,47 +1,158 @@ """This file contains all the tests to do with Equipment use logging and availability checks.""" import os +import time from pathlib import Path from typing import Union, Dict, Iterable +import pytest + +import numpy as np import pandas as pd -from tlo import Simulation, Module, Date +from tlo import Simulation, Module, Date, logging from tlo.analysis.utils import parse_log_file from tlo.events import IndividualScopeEventMixin from tlo.methods import Metadata, demography, healthsystem +from tlo.methods.equipment import Equipment from tlo.methods.hsi_event import HSI_Event -resourcefilepath = Path(os.path.dirname(__file__)) / '../resources' +resourcefilepath = Path(os.path.dirname(__file__)) / "../resources" -equipment_item_code_that_is_available = [0, 1, ] -equipment_item_code_that_is_not_available = [2, 3, ] +def test_core_functionality_of_equipment_class(seed): + """Test that the core functionality of the equipment class works on toy data.""" + + # Create toy data + catalogue = pd.DataFrame( + [ + {"Description": "ItemZero", "Item_Code": 0}, + {"Description": "ItemOne", "Item_Code": 1}, + {"Description": "ItemTwo", "Item_Code": 2}, + {"Description": "ItemThree", "Item_Code": 3}, + ] + ) + data_availability = pd.DataFrame( + # item 0 is not available anywhere; item 1 is available everywhere; item 2 is available only at facility_id=1 + # No data for fac_id=2 + [ + {"Item_Code": 0, "Facility_ID": 0, "Pr_Available": 0.0}, + {"Item_Code": 0, "Facility_ID": 1, "Pr_Available": 0.0}, + {"Item_Code": 1, "Facility_ID": 0, "Pr_Available": 1.0}, + {"Item_Code": 1, "Facility_ID": 1, "Pr_Available": 1.0}, + {"Item_Code": 2, "Facility_ID": 0, "Pr_Available": 0.0}, + {"Item_Code": 2, "Facility_ID": 1, "Pr_Available": 1.0}, + ] + ) + + mfl = pd.DataFrame( + [ + {'District': 'D0', 'Facility_Level': '1a', 'Region': 'R0', 'Facility_ID': 0, 'Facility_Name': 'Fac0'}, + {'District': 'D0', 'Facility_Level': '1b', 'Region': 'R0', 'Facility_ID': 1, 'Facility_Name': 'Fac1'}, + {'District': 'D0', 'Facility_Level': '2', 'Region': 'R0', 'Facility_ID': 2, 'Facility_Name': 'Fac2'}, + ] + ) + + # Create instance of the Equipment class with these toy data and check availability of equipment... + # -- when using `default` behaviour: + eq_default = Equipment( + catalogue=catalogue, + data_availability=data_availability, + rng=np.random.RandomState(np.random.MT19937(np.random.SeedSequence(seed))), + master_facilities_list=mfl, + availability="default", + ) + # - using single integer for one item_code (item should be available) + assert eq_default.is_all_items_available(item_codes=1, facility_id=1) + # - using list of integers for item_codes (items should be available) + assert eq_default.is_all_items_available(item_codes=[1, 2], facility_id=1) + # - calling an item for which data on availability is not provided (should not raise error) + eq_default.is_all_items_available(item_codes=3, facility_id=1) + # - calling an item at a facility that for which data is not provided (should give average behaviour for other facilities) + assert not eq_default.is_all_items_available(item_codes=0, facility_id=2) + assert eq_default.is_all_items_available(item_codes=1, facility_id=2) + # - calling an item for which no data at a facility with no data (should not error) + eq_default.is_all_items_available(item_codes=3, facility_id=2) + # - calling when all items available (should be true) + assert eq_default.is_all_items_available(item_codes=[1, 2], facility_id=1) + # - calling when no items available (should be false) + assert not eq_default.is_all_items_available(item_codes=[0, 2], facility_id=0) + # - calling when some items available (should be false) + assert not eq_default.is_all_items_available(item_codes=[1, 2], facility_id=0) + + # -- when using `none` availability behaviour: everything should not be available! + eq_none = Equipment( + catalogue=catalogue, + data_availability=data_availability, + rng=np.random.RandomState(np.random.MT19937(np.random.SeedSequence(seed))), + availability="none", + master_facilities_list=mfl, + ) + # - calling when all items available (should be false because using 'none' behaviour) + assert not eq_none.is_all_items_available(item_codes=[1, 2], facility_id=1) + # - calling when no items available (should be false) + assert not eq_none.is_all_items_available(item_codes=[0, 2], facility_id=0) + # - calling when some items available (should be false) + assert not eq_none.is_all_items_available(item_codes=[1, 2], facility_id=0) + + # -- when using `all` availability behaviour: everything should not be available! + eq_all = Equipment( + catalogue=catalogue, + data_availability=data_availability, + rng=np.random.RandomState(np.random.MT19937(np.random.SeedSequence(seed))), + availability="all", + master_facilities_list=mfl, + ) + # - calling when all items available (should be true) + assert eq_all.is_all_items_available(item_codes=[1, 2], facility_id=1) + # - calling when no items available (should be true because using 'all' behaviour) + assert eq_all.is_all_items_available(item_codes=[0, 2], facility_id=0) + # - calling when some items available (should be true because using 'all' behaviour) + assert eq_all.is_all_items_available(item_codes=[1, 2], facility_id=0) + + # Check recording use of equipment + # - Add records, using calls with integers and list to different facility_id + eq_default.record_use_of_equipment(item_codes=1, facility_id=0) + eq_default.record_use_of_equipment(item_codes=[0, 1], facility_id=0) + eq_default.record_use_of_equipment(item_codes=[0, 1], facility_id=1) + # - Check that internal record is as expected + assert dict(eq_default._record_of_equipment_used) == {0: {0: 1, 1: 2}, 1: {0: 1, 1: 1}} -def run_simulation_return_log(seed, tmpdir, essential_equipment: Iterable[str], other_equipment: Iterable[str]) -> Dict: - """Returns a parsed logs from `tlo.methods.healthsystem.summary` from a simulation object, in which a single - event has been scheduled with the specified equipment usage, and the availability of equipment has been manipulated. - """ + # Do the logging + eq_default.write_to_log() + + +equipment_item_code_that_is_available = [0, 1, ] +equipment_item_code_that_is_not_available = [2, 3,] + +def run_simulation_and_return_log( + seed, tmpdir, essential_equipment: Iterable[str], other_equipment: Iterable[str] +) -> Dict: + """Returns the parsed logs from `tlo.methods.healthsystem.summary:EquipmentEverUsed_ByFacilityID` from a + simulation object in which a single event has been run with the specified equipment usage. The + availability of equipment has been manipulated so that the item_codes given in + `equipment_item_code_that_is_available` and `equipment_item_code_that_is_not_available` are as expected. """ class DummyHSIEvent(HSI_Event, IndividualScopeEventMixin): - def __init__(self, - module, - person_id, - level, - essential_equipment: Union[int, None], - other_equipment: Union[int, None] - ): + def __init__( + self, + module, + person_id, + level, + essential_equipment, + other_equipment, + ): super().__init__(module, person_id=person_id) self.TREATMENT_ID = "DummyHSIEvent" self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint({}) self.ACCEPTED_FACILITY_LEVEL = level - self.ESSENTIAL_EQUIPMENT = str(essential_equipment) if essential_equipment is not None else set() + self.add_equipment(essential_equipment) # Declaration at init will mean that these items are considered + # essential. self._other_equipment = other_equipment def apply(self, person_id, squeeze_factor): if self._other_equipment is not None: self.add_equipment(self._other_equipment) - class DummyModule(Module): METADATA = {Metadata.DISEASE_MODULE, Metadata.USES_HEALTHSYSTEM} @@ -58,11 +169,11 @@ def initialise_population(self, population): def initialise_simulation(self, sim): # Schedule the HSI_Event to occur on the first day of the simulation - sim.modules['HealthSystem'].schedule_hsi_event( + sim.modules["HealthSystem"].schedule_hsi_event( hsi_event=DummyHSIEvent( person_id=0, - level='2', - module=sim.modules['DummyModule'], + level="2", + module=sim.modules["DummyModule"], essential_equipment=self.essential_equipment, other_equipment=self.other_equipment, ), @@ -77,22 +188,22 @@ def initialise_simulation(self, sim): sim.register( demography.Demography(resourcefilepath=resourcefilepath), healthsystem.HealthSystem(resourcefilepath=resourcefilepath), - DummyModule(essential_equipment=essential_equipment, other_equipment=other_equipment), + DummyModule( + essential_equipment=essential_equipment, other_equipment=other_equipment + ), ) # Manipulate availability of equipment - df = sim.modules['HealthSystem'].parameters['Equipment'] - col_for_availability = df.columns[df.columns.str.startswith('Avail_')] - df.loc[df['Equip_Code'].isin(equipment_item_code_that_is_available), col_for_availability] = True - df.loc[df['Equip_Code'].isin(equipment_item_code_that_is_not_available), col_for_availability] = False - df['Equip_Item'] = df['Equip_Code'].astype(str) - sim.modules['HealthSystem'].parameters['Equipment'] = df.loc[df['Equip_Code'].isin(set(equipment_item_code_that_is_available) | set(equipment_item_code_that_is_not_available))] + df = sim.modules["HealthSystem"].parameters["equipment_availability_estimates"] + df.loc[df['Item_Code'].isin(equipment_item_code_that_is_available), 'Pr_Available'] = 1.0 + df.loc[df['Item_Code'].isin(equipment_item_code_that_is_not_available), 'Pr_Available'] = 0.0 + # Run the simulation sim.make_initial_population(n=100) - sim.simulate(end_date=pd.DateOffset(months=1)) - - return parse_log_file(sim.log_filepath)['tlo.methods.healthsystem.summary'] + sim.simulate(end_date=sim.start_date + pd.DateOffset(months=1)) + # Return the parsed log of `tlo.methods.healthsystem.summary` + return parse_log_file(sim.log_filepath)["tlo.methods.healthsystem.summary"] @@ -103,98 +214,108 @@ def test_equipment_use_is_logged(seed, tmpdir): * An HSI that declares use of equipment during its `apply` method (but no essential equipment); * An HSI that declare use of essential equipment but nothing in its `apply` method`; * An HSI that declare use of essential equipment and equipment during its `apply` method; - * An HSI that declares not use of any equipment (logs should be empty). + * An HSI that declares not use of any equipment. """ - - def logged_equipment_used(sim: Dict) -> pd.DataFrame: - """Read the log to work out what equipment usage has been logged.""" - # @Eva - I think this will somehow use the function that is currently in `src/scripts/healthsystem/equipment/equipment_catalogue.py` - pass - - def get_sim(essential_equipment, other_equipment): - """Pass-through to `run_simulation_return_log` to make call simpler.""" - return run_simulation_return_log( + the_item_code = equipment_item_code_that_is_available[0] + another_item_code = equipment_item_code_that_is_available[1] + + def all_equipment_ever_used(log: Dict) -> set: + """With the log of equipment used in the simulation, return a set of the equipments item that have been used + (at any facility).""" + s = set() + for i in log["EquipmentEverUsed_ByFacilityID"]['EquipmentEverUsed']: + s.update(eval(i)) + return s + + # * An HSI that declares no use of any equipment (logs should be empty). + assert set() == all_equipment_ever_used( + run_simulation_and_return_log( seed=seed, tmpdir=tmpdir, - essential_equipment=essential_equipment, - other_equipment=other_equipment, + essential_equipment=set(), + other_equipment=set(), ) - - # Check that the log matches expectation under each permutation - item_available_as_set_of_str = {str(equipment_item_code_that_is_available[0])} + ) # * An HSI that declares use of equipment during its `apply` method (but no essential equipment) - expected_df = pd.DataFrame() # <-- fill in what we expect it to look like - assert expected_df.equals(logged_equipment_used(get_sim( - essential_equipment={}, - other_equipment=item_available_as_set_of_str, - ))) + assert {the_item_code} == all_equipment_ever_used( + run_simulation_and_return_log( + seed=seed, + tmpdir=tmpdir, + essential_equipment=set(), + other_equipment=the_item_code, + ) + ) # * An HSI that declare use of essential equipment but nothing in its `apply` method`; - expected_df = pd.DataFrame() # <-- fill in what we expect it to look like - assert expected_df.equals(logged_equipment_used(get_sim( - essential_equipment=item_available_as_set_of_str, - other_equipment={}, - ))) + assert {the_item_code} == all_equipment_ever_used( + run_simulation_and_return_log( + seed=seed, + tmpdir=tmpdir, + essential_equipment=the_item_code, + other_equipment=set(), + ) + ) # * An HSI that declare use of essential equipment and equipment during its `apply` method; - expected_df = pd.DataFrame() # <-- fill in what we expect it to look like - assert expected_df.equals(logged_equipment_used(get_sim( - essential_equipment=item_available_as_set_of_str, - other_equipment=item_available_as_set_of_str, - ))) + assert {the_item_code, another_item_code} == all_equipment_ever_used( + run_simulation_and_return_log( + seed=seed, + tmpdir=tmpdir, + essential_equipment=the_item_code, + other_equipment=another_item_code, + ) + ) + - # * An HSI that declares not use of any equipment (logs should be empty). - expected_df = pd.DataFrame() # <-- fill in what we expect it to look like - assert expected_df.equals(logged_equipment_used(get_sim( - essential_equipment={}, - other_equipment={}, - ))) def test_hsi_does_not_run_if_essential_equipment_is_not_available(seed, tmpdir): """Check that an HSI which declares an item of equipment that is essential does run if that item is available and does not run if that item is not available.""" - def did_hsi_run(sim: Dict) -> bool: - """Read the log to work out if the Dummy HSI Event ran or not.""" - pass + def did_the_hsi_run(log: Dict) -> bool: + """Read the log to work out if the `DummyHSIEvent` ran or not.""" + it_did_run = len(log['hsi_event_counts'].iloc[0]['hsi_event_key_to_counts']) > 0 + it_did_not_run = len(log['never_ran_hsi_event_counts'].iloc[0]['never_ran_hsi_event_key_to_counts']) > 0 + + # Check that there if it did not run, it has had its "never_ran" function called + assert it_did_run != it_did_not_run + + # Return indication of whether it did run + return it_did_run - def get_sim(essential_equipment): - """Pass-through to `run_simulation_return_log` to make call simpler.""" - return run_simulation_return_log( - seed=seed, - tmpdir=tmpdir, - essential_equipment=essential_equipment, - other_equipment=None - ) # HSI_Event that requires equipment that is available --> will run - assert did_hsi_run( - get_sim( - essential_equipment=set(str(equipment_item_code_that_is_available[0])) + assert did_the_hsi_run( + run_simulation_and_return_log( + seed=seed, + tmpdir=tmpdir, + essential_equipment=equipment_item_code_that_is_available, + other_equipment=set(), ) ) # HSI_Event that requires equipment that is not available --> will not run - assert not did_hsi_run( - get_sim( - essential_equipment=set(str(equipment_item_code_that_is_not_available[0])) + assert not did_the_hsi_run( + run_simulation_and_return_log( + seed=seed, + tmpdir=tmpdir, + essential_equipment=equipment_item_code_that_is_not_available, + other_equipment=set(), ) ) def test_lookup_equipment_item_code_from_item_name(): + # todo incorporate into calls above, different forms of calling. pass + def test_lookup_equipment_item_code_from_pkg_name(): pass -def test_lookup_item_availability_by_hsi_event(): - pass def test_change_equipment_availability(): """Test that we can change the availability of equipment midway through the simulation.""" pass - - From 27779e67c130d24e5bedc42c796aa4d51936f649 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Fri, 10 May 2024 17:18:18 +0100 Subject: [PATCH 082/118] squash - basic outline of Equipment class --- .../ResourceFile_HealthSystem_parameters.csv | 4 +- .../ResourceFile_EquipmentCatalogue.csv | 4 +- src/tlo/methods/equipment.py | 100 +++++----- src/tlo/methods/healthsystem.py | 30 ++- src/tlo/methods/hsi_event.py | 22 ++- tests/test_equipment.py | 173 ++++++++++++++---- tests/test_healthsystem.py | 2 +- 7 files changed, 235 insertions(+), 100 deletions(-) diff --git a/resources/healthsystem/ResourceFile_HealthSystem_parameters.csv b/resources/healthsystem/ResourceFile_HealthSystem_parameters.csv index 25315a9917..6ca37170a7 100644 --- a/resources/healthsystem/ResourceFile_HealthSystem_parameters.csv +++ b/resources/healthsystem/ResourceFile_HealthSystem_parameters.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d9398dca0e70d64f0865f5f6d99ec0eec5baf9d2716ad037236959d3e7f479ed -size 632 +oid sha256:5df67165565fc88987f848e43363616e2ea4135de7c74d131e785ddc0178f123 +size 706 diff --git a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_EquipmentCatalogue.csv b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_EquipmentCatalogue.csv index 2f3ca59328..33ba052c64 100644 --- a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_EquipmentCatalogue.csv +++ b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_EquipmentCatalogue.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0d21fb0f546325fd264b0598efd7afbea15ad48564ed28cb9df970509ce1d405 -size 33196 +oid sha256:3e151e16f7eea2ae61d2fa637c26449aa533ddc6a7f0d83aff495f5f6c9d1f8d +size 33201 diff --git a/src/tlo/methods/equipment.py b/src/tlo/methods/equipment.py index 2bb1f15267..5f561dd8aa 100644 --- a/src/tlo/methods/equipment.py +++ b/src/tlo/methods/equipment.py @@ -25,8 +25,9 @@ class Equipment: specified in the ResourceFile; if 'none', then let no equipment be ever be available; if 'all', then all equipment is always available. - If an item_code is requested that is not recognised (not included in `data`), a `UserWarning` is issued, and the - result returned is on the basis of the average availability of other consumables in that facility in that month. + :param `: `master_facilities_list`: The pd.DataFrame with line-list of all the facilities in the HealthSystem. + + If an item_code is referred that is not recognised (not included in `catalogue`), a `UserWarning` is issued. """ def __init__( @@ -47,15 +48,18 @@ def __init__( self._items_available: Dict = dict() # Will be the internal store of which items are available at each # facility_id. This is of the form {facility_id: {items_available}}. - self._record_of_equipment_used = defaultdict(Counter) # Will be the internal store of which items have been - # used at each facility_id. This is of the form - # {facility_id: {item_code: count}}. + self._record_of_equipment_used_by_facility_id = defaultdict(Counter) # Will be the internal store of which + # items have been used at each facility_id This is of the form {facility_id: {item_code: count}}. # Set up the internal stores of equipment items that are available, ready for calls. self._set_equipment_items_available(availability=availability) - # Set up internal lookup for item_descriptor -> item_code - self.item_code_lookup = self.catalogue.set_index('Description')['Item_Code'].to_dict() + # Set up internal lookup for item_descriptor -> item_code, and validating codes/desciptors + self.item_code_lookup = self.catalogue.set_index('Item_Description')['Item_Code'].to_dict() + self._all_item_descriptors = set(self.item_code_lookup.keys()) + self._all_item_codes = set(self.item_code_lookup.values()) + + self._history_of_items_checked = set() # All the arguments provided to parse_items def on_simulation_end(self): """Things to do when the simulation end: @@ -119,57 +123,58 @@ def _set_equipment_items_available(self, availability: str): lambda x: set(x[x].index.get_level_values("Item_Code")) ).to_dict() - def _parse_items(self, items: Union[int, str, Iterable[int | str]]) -> Set[int]: + def parse_items(self, items: Union[int, str, Iterable[int | str]]) -> Set[int]: """Parse equipment items specified as an item_code (integer), an item descriptor (string), or an iterable of - either, and return as a set of item_code (integers).""" - - def first_element_in_iterable(it: Iterable[Any]) -> Any | None: - for el in it: - return el - - if isinstance(items, str): - # Single descriptor - return set([self.item_code_lookup(items)]) - elif isinstance(items, int): - # Single item_code provided - return set([items]) - elif isinstance(items, Iterable) and len(items) == 0: - # Iterable of length 0 - return set() - elif isinstance(items, Iterable) and isinstance(first_element_in_iterable(items), str): - # Iterable of descriptors - return set(map(self.item_code_lookup, items)) - elif isinstance(items, Iterable) and isinstance(first_element_in_iterable(items), int): - # Iterable of item_cods - return set(items) + either, and return as a set of item_code (integers). For any item_code/descriptor not recognised, a + `UserWarning` is issued.""" + + def check_item_code_recognised(item_codes: set[int]): + if not item_codes.issubset(self._all_item_codes): + warnings.warn(f'Item code(s) "{item_codes}" not recognised.') + def check_item_descriptors_recognised(item_descriptors: set[str]): + if not item_descriptors.issubset(self._all_item_descriptors): + warnings.warn(f'Item descriptor(s) "{item_descriptors}" not recognised.') + + # Make into a set if it is not one already + if isinstance(items, (str, int)): + items = set([items]) else: - raise ValueError(f'Item_Code format not recognised: {items=}') + items = set(items) - if isinstance(items, int): - item_codes = set([items]) # If single int provided, place it into a list - elif isinstance(items, list): - item_codes = set(items) + items_are_ints = all(isinstance(element, int) for element in items) + if items_are_ints: + check_item_code_recognised(items) + # In the return, any unrecognised item_codes are silently ignored. + return items.intersection(self._all_item_codes) + else: + check_item_descriptors_recognised(items) # Warn for any unrecognised descriptors + # In the return, any unrecognised descriptors are silently ignored. + return set(filter(lambda item: item is not None, map(self.item_code_lookup.get, items))) def is_all_items_available( - self, item_codes: Union[int, str, Iterable[int | str]], facility_id: int + self, item_codes: Set[int], facility_id: int ) -> bool: """Determine if all equipments are available at the given facility_id (or from the default if the faciluty_id is not recognised). Returns True only if all items are available at the facility_id, otherwise returns False.""" - return self._parse_items(item_codes).issubset(self._items_available[facility_id]) - + try: + return item_codes.issubset(self._items_available[facility_id]) + except KeyError: + raise ValueError(f'Not recognised {facility_id=}') def record_use_of_equipment( - self, item_codes: Union[int, str, Iterable[int | str]], facility_id: int + self, item_codes: Set[int], facility_id: int ) -> None: """Update internal record of the usage of items at equipment at the specified facility_id.""" - self._record_of_equipment_used[facility_id].update(self._parse_items(item_codes)) + self._record_of_equipment_used_by_facility_id[facility_id].update(item_codes) def write_to_log(self) -> None: """Write to the log: - * Summary of the equipment that was _ever_ used at each facility_level - * For each facility_id, a set of the equipment items ever used. + * Summary of the equipment that was _ever_ used at each district/facility level. + Note that the info-level health system logger (key: `hsi_event_counts`) contains logging of the equipment used + in each HSI event (if further finer splits are needed). Alternatively, different aggregations could be created + here for the summary logger, using the same pattern as used here. """ mfl = self.master_facilities_list @@ -183,7 +188,7 @@ def set_of_keys_or_empty_set(x: Union[set, dict]): return None set_of_equipment_ever_used_at_each_facility_id = pd.Series({ - fac_id: set_of_keys_or_empty_set(self._record_of_equipment_used.get(fac_id, set())) + fac_id: set_of_keys_or_empty_set(self._record_of_equipment_used_by_facility_id.get(fac_id, set())) for fac_id in mfl['Facility_ID'] }, name='EquipmentEverUsed').astype(str) @@ -202,3 +207,14 @@ def set_of_keys_or_empty_set(x: Union[set, dict]): 'equipment items that are ever used.', data=row.to_dict(), ) + + def lookup_item_codes_from_pkg_name(self, pkg_name: str) -> Set[int]: + """Convenience function to find the set of item_codes that are grouped under a package name in the catalogue. + It is expected that this is used by the disease module once and then the resulting equipment item_codes are + saved on th at module. Note that all interaction with the `Equipment` module is using set of item_codes.""" + df = self.catalogue + + if pkg_name not in df['Pkg_Name'].unique(): + raise ValueError(f'That Pkg_Name is not in the catalogue: {pkg_name=}') + + return set(df.loc[df['Pkg_Name'] == pkg_name, 'Item_Code'].values) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index a0a3e6b96f..eb44735d55 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -205,6 +205,16 @@ class HealthSystem(Module): "the ResourceFile; if 'none', then let no equipment ever be available; if 'all', then all equipment is " "always available. NB. This parameter is over-ridden if an argument is provided to the module initialiser." ), + 'equip_availability_postSwitch': Parameter( + Types.STRING, + "What to assume about the availability of equipment after the switch (see `year_equip_availability_switch`" + "). The options for this are the same as `equip_availability`." + ), + 'year_equip_availability_switch': Parameter( + Types.INT, + "Year in which the assumption for `equip_availability` changes (The change happens on 1st January of that " + "year.)" + ), # Service Availability 'Service_Availability': Parameter( @@ -708,6 +718,18 @@ def initialise_simulation(self, sim): Date(self.parameters["year_cons_availability_switch"], 1, 1) ) + # Schedule an equipment availability switch + sim.schedule_event( + HealthSystemChangeParameters( + self, + parameters={ + 'equip_availability': self.parameters['equip_availability_postSwitch'] + } + ), + Date(self.parameters["year_equip_availability_switch"], 1, 1) + ) + + # Schedule a one-off rescaling of _daily_capabilities broken down by officer type and level. # This occurs on 1st January of the year specified in the parameters. sim.schedule_event(ConstantRescalingHRCapabilities(self), @@ -1628,7 +1650,6 @@ def record_hsi_event(self, hsi_event, actual_appt_footprint=None, squeeze_factor squeeze_factor=_squeeze_factor, did_run=did_run, priority=priority, - equipment=hsi_event.EQUIPMENT, ) def write_to_hsi_log( @@ -1639,9 +1660,9 @@ def write_to_hsi_log( squeeze_factor: float, did_run: bool, priority: int, - equipment: set, ): """Write the log `HSI_Event` and add to the summary counter.""" + # Debug logger gives simple line-list for every HSI event logger.debug( key="HSI_Event", data={ @@ -1654,16 +1675,19 @@ def write_to_hsi_log( 'did_run': did_run, 'Facility_Level': event_details.facility_level if event_details.facility_level is not None else -99, 'Facility_ID': facility_id if facility_id is not None else -99, - 'equipment': sorted(equipment), + 'Equipment': sorted(event_details.equipment), }, description="record of each HSI event" ) if did_run: if self._hsi_event_count_log_period is not None: + # Do logging for HSI Event using counts of each 'unique type' of HSI event (as defined by + # `HSIEventDetails`). event_details_key = self._hsi_event_details.setdefault( event_details, len(self._hsi_event_details) ) self._hsi_event_counts_log_period[event_details_key] += 1 + # Do logging for 'summary logger' self._summary_counter.record_hsi_event( treatment_id=event_details.treatment_id, hsi_event_name=event_details.event_name, diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index 8598cb1294..f828a47968 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -113,8 +113,6 @@ def __init__(self, module, *args, **kwargs): self._received_info_about_bed_days = None self.expected_time_requests = {} self.facility_info = None - self.ESSENTIAL_EQUIPMENT = None - self.EQUIPMENT = set() # todo should this be private, and should we add setter methods self.TREATMENT_ID = "" self.ACCEPTED_FACILITY_LEVEL = None @@ -272,22 +270,26 @@ def make_appt_footprint(self, dict_of_appts) -> Counter: ) def add_equipment(self, item_codes: Union[int, str, Iterable[int | str]]): - """Declare that piece(s) of equipment are used in this HSI_Event.""" - self._EQUIPMENT.update(self.healthcare_system.equipment._parse_items(item_codes)) + """Declare that piece(s) of equipment are used in this HSI_Event. + Checks are done on the validity of the item_codes/item descriptions and warnings issued if they are not + recognised.""" + self._EQUIPMENT.update(self.healthcare_system.equipment.parse_items(item_codes)) @property def is_all_declared_equipment_available(self) -> bool: - """Returns True if all the declared items of equipment are available. This is called before the HSI is run - so is looking only at those items that are declared when this instance was created.""" + """Returns `True` if all the declared items of equipment are available. This is called before the HSI is run + and so is looking only at those items that are declared when this instance was created.""" return self.healthcare_system.equipment.is_all_items_available( item_codes=self._EQUIPMENT, facility_id=self.facility_info.id, ) - def is_equipment_available(self, item_codes: Union[int, Iterable[int]]) -> bool: - """Check whether all the equipment item_codes are available.""" + def is_equipment_available(self, item_codes: Union[int, str, Iterable[int | str]]) -> bool: + """Check whether all the equipment item_codes are available. This does not imply that the equipment is being + used and no logging happens. It is provided as a convenience to disease module authors in case the logic of + during an HSI Event depends on the availability of a piece of equipment.""" return self.healthcare_system.equipment.is_all_items_available( - item_codes=item_codes, + item_codes=self.healthcare_system.equipment.parse_items(item_codes), facility_id=self.facility_info.id, ) @@ -399,7 +401,7 @@ def as_namedtuple( beddays_footprint=tuple( sorted((k, v) for k, v in self.BEDDAYS_FOOTPRINT.items() if v > 0) ), - equipment=(tuple(sorted(self.EQUIPMENT))), # todo check if this is defined in HSIEventDetails and whether we want this here + equipment=(tuple(sorted(self._EQUIPMENT))), ) def post_apply_hook(self): diff --git a/tests/test_equipment.py b/tests/test_equipment.py index 96faf660f8..6dcc6c0699 100644 --- a/tests/test_equipment.py +++ b/tests/test_equipment.py @@ -25,10 +25,10 @@ def test_core_functionality_of_equipment_class(seed): # Create toy data catalogue = pd.DataFrame( [ - {"Description": "ItemZero", "Item_Code": 0}, - {"Description": "ItemOne", "Item_Code": 1}, - {"Description": "ItemTwo", "Item_Code": 2}, - {"Description": "ItemThree", "Item_Code": 3}, + {"Item_Description": "ItemZero", "Item_Code": 0, "Pkg_Name": float('nan')}, + {"Item_Description": "ItemOne", "Item_Code": 1, "Pkg_Name": float('nan')}, + {"Item_Description": "ItemTwo", "Item_Code": 2, "Pkg_Name": 'PkgWith2+3'}, + {"Item_Description": "ItemThree", "Item_Code": 3, "Pkg_Name": 'PkgWith2+3'}, ] ) data_availability = pd.DataFrame( @@ -61,23 +61,47 @@ def test_core_functionality_of_equipment_class(seed): master_facilities_list=mfl, availability="default", ) - # - using single integer for one item_code (item should be available) - assert eq_default.is_all_items_available(item_codes=1, facility_id=1) - # - using list of integers for item_codes (items should be available) - assert eq_default.is_all_items_available(item_codes=[1, 2], facility_id=1) - # - calling an item for which data on availability is not provided (should not raise error) - eq_default.is_all_items_available(item_codes=3, facility_id=1) - # - calling an item at a facility that for which data is not provided (should give average behaviour for other facilities) - assert not eq_default.is_all_items_available(item_codes=0, facility_id=2) - assert eq_default.is_all_items_available(item_codes=1, facility_id=2) - # - calling an item for which no data at a facility with no data (should not error) - eq_default.is_all_items_available(item_codes=3, facility_id=2) + + # Checks on parsing equipment items + # - using single integer for one item_code + assert {1} == eq_default.parse_items(1) + # - using list of integers for item_codes + assert {1, 2} == eq_default.parse_items([1, 2]) + # - using single string for one item descriptor + assert eq_default.parse_items('ItemOne') + # - using list of strings for item descriptors + assert eq_default.parse_items(['ItemOne', 'ItemTwo']) + # - an empty iterable of equipment should always be work whether expressed as list/tuple/set + assert set() == eq_default.parse_items(list()) + assert set() == eq_default.parse_items(tuple()) + assert set() == eq_default.parse_items(set()) + + # - Calling for unrecognised item_codes (should raise warning) + with pytest.warns(): + eq_default.parse_items(10001) + with pytest.warns(): + eq_default.parse_items('ItemThatIsNotDefined') + + # Testing checking on available of items # - calling when all items available (should be true) - assert eq_default.is_all_items_available(item_codes=[1, 2], facility_id=1) + assert eq_default.is_all_items_available(item_codes={1, 2}, facility_id=1) # - calling when no items available (should be false) - assert not eq_default.is_all_items_available(item_codes=[0, 2], facility_id=0) + assert not eq_default.is_all_items_available(item_codes={0, 2}, facility_id=0) # - calling when some items available (should be false) - assert not eq_default.is_all_items_available(item_codes=[1, 2], facility_id=0) + assert not eq_default.is_all_items_available(item_codes={1, 2}, facility_id=0) + # - calling for empty set of equipment (should always be available) + assert eq_default.is_all_items_available(item_codes=set(), facility_id=0) + + # - calling an item for which data on availability is not provided (should not raise error) + eq_default.is_all_items_available(item_codes={3}, facility_id=1) + # - calling an item at a facility that for which data is not provided (should give average behaviour for other facilities) + assert not eq_default.is_all_items_available(item_codes={0}, facility_id=2) + assert eq_default.is_all_items_available(item_codes={1}, facility_id=2) + # - calling a recognised item for which no data at a facility with no data (should not error) + eq_default.is_all_items_available(item_codes={3}, facility_id=2) + # -- calling for an unrecognised facility_id (should error) + with pytest.raises(ValueError): + eq_default.is_all_items_available(item_codes={1}, facility_id=1001) # -- when using `none` availability behaviour: everything should not be available! eq_none = Equipment( @@ -88,11 +112,13 @@ def test_core_functionality_of_equipment_class(seed): master_facilities_list=mfl, ) # - calling when all items available (should be false because using 'none' behaviour) - assert not eq_none.is_all_items_available(item_codes=[1, 2], facility_id=1) + assert not eq_none.is_all_items_available(item_codes={1, 2}, facility_id=1) # - calling when no items available (should be false) - assert not eq_none.is_all_items_available(item_codes=[0, 2], facility_id=0) + assert not eq_none.is_all_items_available(item_codes={0, 2}, facility_id=0) # - calling when some items available (should be false) - assert not eq_none.is_all_items_available(item_codes=[1, 2], facility_id=0) + assert not eq_none.is_all_items_available(item_codes={1, 2}, facility_id=0) + # - calling for empty set of equipment (should always be available) + assert eq_none.is_all_items_available(item_codes=set(), facility_id=0) # -- when using `all` availability behaviour: everything should not be available! eq_all = Equipment( @@ -103,22 +129,29 @@ def test_core_functionality_of_equipment_class(seed): master_facilities_list=mfl, ) # - calling when all items available (should be true) - assert eq_all.is_all_items_available(item_codes=[1, 2], facility_id=1) + assert eq_all.is_all_items_available(item_codes={1, 2}, facility_id=1) # - calling when no items available (should be true because using 'all' behaviour) - assert eq_all.is_all_items_available(item_codes=[0, 2], facility_id=0) + assert eq_all.is_all_items_available(item_codes={0, 2}, facility_id=0) # - calling when some items available (should be true because using 'all' behaviour) - assert eq_all.is_all_items_available(item_codes=[1, 2], facility_id=0) + assert eq_all.is_all_items_available(item_codes={1, 2}, facility_id=0) + # - calling for empty set of equipment (should always be available) + assert eq_all.is_all_items_available(item_codes=set(), facility_id=0) # Check recording use of equipment # - Add records, using calls with integers and list to different facility_id - eq_default.record_use_of_equipment(item_codes=1, facility_id=0) - eq_default.record_use_of_equipment(item_codes=[0, 1], facility_id=0) - eq_default.record_use_of_equipment(item_codes=[0, 1], facility_id=1) + eq_default.record_use_of_equipment(item_codes={1}, facility_id=0) + eq_default.record_use_of_equipment(item_codes={0, 1}, facility_id=0) + eq_default.record_use_of_equipment(item_codes={0, 1}, facility_id=1) # - Check that internal record is as expected - assert dict(eq_default._record_of_equipment_used) == {0: {0: 1, 1: 2}, 1: {0: 1, 1: 1}} + assert dict(eq_default._record_of_equipment_used_by_facility_id) == {0: {0: 1, 1: 2}, 1: {0: 1, 1: 1}} + + # Lookup the item_codes that belong in a particular package. + # - When package is recognised + assert {2, 3} == eq_default.lookup_item_codes_from_pkg_name(pkg_name='PkgWith2+3') # these items are in the same package + # - Error thrown when package is not recognised + with pytest.raises(ValueError): + eq_default.lookup_item_codes_from_pkg_name(pkg_name='') - # Do the logging - eq_default.write_to_log() equipment_item_code_that_is_available = [0, 1, ] @@ -268,8 +301,6 @@ def all_equipment_ever_used(log: Dict) -> set: ) - - def test_hsi_does_not_run_if_essential_equipment_is_not_available(seed, tmpdir): """Check that an HSI which declares an item of equipment that is essential does run if that item is available and does not run if that item is not available.""" @@ -307,15 +338,77 @@ def did_the_hsi_run(log: Dict) -> bool: ) -def test_lookup_equipment_item_code_from_item_name(): - # todo incorporate into calls above, different forms of calling. - pass +def test_change_equipment_availability(seed): + """Test that we can change the availability of equipment midway through the simulation.""" + # Set-up simulation that starts with `all` availability and then changes to `none` after one year. In the + # simulation a DummyModule schedules a DummyHSI that runs every month and tries to get a piece of equipment. + # Check that this piece of equipment is available for the first year but not the second year of the simulation. + class DummyHSIEvent(HSI_Event, IndividualScopeEventMixin): + def __init__( + self, + module, + person_id, + ): + super().__init__(module, person_id=person_id) + self.TREATMENT_ID = "DummyHSIEvent" + self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint({}) + self.ACCEPTED_FACILITY_LEVEL = '1a' + self.store_of_equipment_checks = dict() -def test_lookup_equipment_item_code_from_pkg_name(): - pass + def apply(self, person_id, squeeze_factor): + # Check availability of a piece of equipment, with item_code = 0 + self.store_of_equipment_checks.update( + { + self.sim.date: self.is_equipment_available(item_codes={0}) + } + ) + # Schedule recurrence of this event in one month's time + self.sim.modules["HealthSystem"].schedule_hsi_event( + hsi_event=self, + do_hsi_event_checks=False, + topen=self.sim.date + pd.DateOffset(months=1), + tclose=None, + priority=0, + ) -def test_change_equipment_availability(): - """Test that we can change the availability of equipment midway through the simulation.""" - pass + class DummyModule(Module): + METADATA = {Metadata.DISEASE_MODULE, Metadata.USES_HEALTHSYSTEM} + + def read_parameters(self, data_folder): + pass + + def initialise_population(self, population): + pass + + def initialise_simulation(self, sim): + # Schedule the HSI_Event to occur on the first day of the simulation (it will schedule its own repeats) + self.the_hsi_event = DummyHSIEvent(person_id=0, module=self) + + sim.modules["HealthSystem"].schedule_hsi_event( + hsi_event=self.the_hsi_event, + do_hsi_event_checks=False, + topen=sim.date, + tclose=None, + priority=0, + ) + + sim = Simulation(start_date=Date(2010, 1, 1), seed=seed) + sim.register( + demography.Demography(resourcefilepath=resourcefilepath), + healthsystem.HealthSystem(resourcefilepath=resourcefilepath), + DummyModule(), + ) + # Modify the parameters of the healthsystem to effect a change in the availability of equipment + sim.modules['HealthSystem'].parameters['equip_availability'] = 'all' + sim.modules['HealthSystem'].parameters['equip_availability_postSwitch'] = 'none' + sim.modules['HealthSystem'].parameters['year_equip_availability_switch'] = 2011 + + sim.make_initial_population(n=100) + sim.simulate(end_date=sim.start_date + pd.DateOffset(years=2)) + + # Get store & check for availabilities of the equipment + log = pd.Series(sim.modules['DummyModule'].the_hsi_event.store_of_equipment_checks) + assert log[log.index < Date(2011, 1, 1)].all() + assert not log[log.index >= Date(2011, 1, 1)].any() diff --git a/tests/test_healthsystem.py b/tests/test_healthsystem.py index 1bccb9e7c4..5dece9b2d9 100644 --- a/tests/test_healthsystem.py +++ b/tests/test_healthsystem.py @@ -948,7 +948,7 @@ def apply(self, person_id, squeeze_factor): detailed_consumables = log["tlo.methods.healthsystem"]['Consumables'] assert {'date', 'TREATMENT_ID', 'did_run', 'Squeeze_Factor', 'priority', 'Number_By_Appt_Type_Code', 'Person_ID', - 'Facility_Level', 'Facility_ID', 'Event_Name', 'equipment' + 'Facility_Level', 'Facility_ID', 'Event_Name', 'Equipment' } == set(detailed_hsi_event.columns) assert {'date', 'Frac_Time_Used_Overall', 'Frac_Time_Used_By_Facility_ID', 'Frac_Time_Used_By_OfficerType', } == set(detailed_capacity.columns) From 1988ae274846ed195d808af382b19eb2d9cf8d64 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Fri, 10 May 2024 17:33:52 +0100 Subject: [PATCH 083/118] linting --- src/tlo/methods/contraception.py | 3 ++- src/tlo/methods/equipment.py | 32 +++++++++++++++---------------- src/tlo/methods/healthsystem.py | 5 ++++- src/tlo/methods/hsi_event.py | 20 ++++++++++--------- src/tlo/methods/symptommanager.py | 3 ++- tests/test_equipment.py | 14 +++++++------- 6 files changed, 41 insertions(+), 36 deletions(-) diff --git a/src/tlo/methods/contraception.py b/src/tlo/methods/contraception.py index 67d6684fce..83fd86cab9 100644 --- a/src/tlo/methods/contraception.py +++ b/src/tlo/methods/contraception.py @@ -164,7 +164,8 @@ def read_parameters(self, data_folder): """Import the relevant sheets from the ResourceFile (excel workbook) and declare values for other parameters (CSV ResourceFile). """ - workbook = pd.read_excel(Path(self.resourcefilepath) / 'contraception' / 'ResourceFile_Contraception.xlsx', sheet_name=None) + workbook = pd.read_excel(Path(self.resourcefilepath) / 'contraception' / 'ResourceFile_Contraception.xlsx', + sheet_name=None) # Import selected sheets from the workbook as the parameters sheet_names = [ diff --git a/src/tlo/methods/equipment.py b/src/tlo/methods/equipment.py index 5f561dd8aa..3ba34eca4a 100644 --- a/src/tlo/methods/equipment.py +++ b/src/tlo/methods/equipment.py @@ -1,6 +1,6 @@ import warnings from collections import defaultdict -from typing import Dict, Set, Iterable, Union, Counter, Optional, Any +from typing import Counter, Dict, Iterable, Optional, Set, Union import numpy as np import pandas as pd @@ -45,21 +45,19 @@ def __init__( self.master_facilities_list = master_facilities_list # Create internal storage structures - self._items_available: Dict = dict() # Will be the internal store of which items are available at each + self._items_available: Dict = dict() # <-- Will be the internal store of which items are available at each # facility_id. This is of the form {facility_id: {items_available}}. - self._record_of_equipment_used_by_facility_id = defaultdict(Counter) # Will be the internal store of which + self._record_of_equipment_used_by_facility_id = defaultdict(Counter) # <-- Will be the internal store of which # items have been used at each facility_id This is of the form {facility_id: {item_code: count}}. - # Set up the internal stores of equipment items that are available, ready for calls. - self._set_equipment_items_available(availability=availability) - - # Set up internal lookup for item_descriptor -> item_code, and validating codes/desciptors - self.item_code_lookup = self.catalogue.set_index('Item_Description')['Item_Code'].to_dict() - self._all_item_descriptors = set(self.item_code_lookup.keys()) - self._all_item_codes = set(self.item_code_lookup.values()) + # Data structures for quick look-ups for items and descriptors + self._item_code_lookup = self.catalogue.set_index('Item_Description')['Item_Code'].to_dict() + self._all_item_descriptors = set(self._item_code_lookup.keys()) + self._all_item_codes = set(self._item_code_lookup.values()) - self._history_of_items_checked = set() # All the arguments provided to parse_items + # Initialise the internal stores of equipment items that are available, ready for calls. + self._set_equipment_items_available(availability=availability) def on_simulation_end(self): """Things to do when the simulation end: @@ -67,14 +65,14 @@ def on_simulation_end(self): """ self.write_to_log() - def update_availability(self, availability) -> None: + def update_availability(self, availability: str) -> None: """Update the availability of equipment. This is expected to be called midway through the simulation if the assumption of the equipment availability needs to change.""" self._set_equipment_items_available(availability=availability) def _set_equipment_items_available(self, availability: str): """Update internal store of which items of equipment are available. This is called at the beginning of the - simulation and whenever an update in `availability` is needed.""" + simulation and whenever an update in `availability` is done by `update_availability`.""" # For any facility_id in the data all_fac_ids = self.master_facilities_list['Facility_ID'].unique() @@ -128,7 +126,7 @@ def parse_items(self, items: Union[int, str, Iterable[int | str]]) -> Set[int]: either, and return as a set of item_code (integers). For any item_code/descriptor not recognised, a `UserWarning` is issued.""" - def check_item_code_recognised(item_codes: set[int]): + def check_item_codes_recognised(item_codes: set[int]): if not item_codes.issubset(self._all_item_codes): warnings.warn(f'Item code(s) "{item_codes}" not recognised.') @@ -145,18 +143,18 @@ def check_item_descriptors_recognised(item_descriptors: set[str]): items_are_ints = all(isinstance(element, int) for element in items) if items_are_ints: - check_item_code_recognised(items) + check_item_codes_recognised(items) # In the return, any unrecognised item_codes are silently ignored. return items.intersection(self._all_item_codes) else: check_item_descriptors_recognised(items) # Warn for any unrecognised descriptors # In the return, any unrecognised descriptors are silently ignored. - return set(filter(lambda item: item is not None, map(self.item_code_lookup.get, items))) + return set(filter(lambda item: item is not None, map(self._item_code_lookup.get, items))) def is_all_items_available( self, item_codes: Set[int], facility_id: int ) -> bool: - """Determine if all equipments are available at the given facility_id (or from the default if the faciluty_id + """Determine if all equipments are available at the given facility_id (or from the default if the facility_id is not recognised). Returns True only if all items are available at the facility_id, otherwise returns False.""" try: return item_codes.issubset(self._items_available[facility_id]) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index eb44735d55..28b7576aa0 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -1894,7 +1894,10 @@ def on_end_of_year(self) -> None: # If we are at the end of the year preceeding the mode switch, and if wanted # to rescale capabilities to capture effective availability as was recorded, on # average, in the past year, do so here. - if (self.sim.date.year == self.parameters['year_mode_switch'] - 1) and self.parameters['scale_to_effective_capabilities']: + if ( + (self.sim.date.year == self.parameters['year_mode_switch'] - 1) + and self.parameters['scale_to_effective_capabilities'] + ): self._rescale_capabilities_to_capture_effective_capability() self._summary_counter.write_to_log_and_reset_counters() self.consumables.on_end_of_year() diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index f828a47968..487d4fdabb 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections import Counter -from typing import TYPE_CHECKING, Dict, Literal, NamedTuple, Optional, Set, Tuple, Union, Iterable +from typing import TYPE_CHECKING, Dict, Iterable, Literal, NamedTuple, Optional, Set, Tuple, Union import numpy as np @@ -172,12 +172,21 @@ def never_ran(self) -> None: logger.debug(key="message", data=f"{self.__class__.__name__}: was never run.") def post_apply_hook(self) -> None: - """Impose the bed-days footprint (if target of the HSI is a person_id)""" + """ + Do things following the event's `apply` function running. + * Impose the bed-days footprint (if target of the HSI is a person_id) + * Record the equipment that has been added before and during the course of the HSI Event. + """ if isinstance(self.target, int): self.healthcare_system.bed_days.impose_beddays_footprint( person_id=self.target, footprint=self.bed_days_allocated_to_this_event ) + self.healthcare_system.equipment.record_use_of_equipment( + item_codes=self._EQUIPMENT, + facility_id=self.facility_info.id + ) + def run(self, squeeze_factor): """Make the event happen.""" updated_appt_footprint = self.apply(self.target, squeeze_factor) @@ -404,13 +413,6 @@ def as_namedtuple( equipment=(tuple(sorted(self._EQUIPMENT))), ) - def post_apply_hook(self): - """Record the equipment that has been added before and during the course of the HSI Event.""" - self.healthcare_system.equipment.record_use_of_equipment( - item_codes=self._EQUIPMENT, - facility_id=self.facility_info.id - ) - class HSIEventWrapper(Event): """This is wrapper that contains an HSI event. diff --git a/src/tlo/methods/symptommanager.py b/src/tlo/methods/symptommanager.py index 68edbf0840..26f6aa7ee4 100644 --- a/src/tlo/methods/symptommanager.py +++ b/src/tlo/methods/symptommanager.py @@ -272,7 +272,8 @@ def pre_initialise_population(self): SymptomManager.PROPERTIES = dict() for symptom_name in sorted(self.symptom_names): symptom_column_name = self.get_column_name_for_symptom(symptom_name) - SymptomManager.PROPERTIES[symptom_column_name] = Property(Types.BITSET, f'Presence of symptom {symptom_name}') + SymptomManager.PROPERTIES[symptom_column_name] = Property(Types.BITSET, + f'Presence of symptom {symptom_name}') def initialise_population(self, population): """ diff --git a/tests/test_equipment.py b/tests/test_equipment.py index 6dcc6c0699..44899f5c19 100644 --- a/tests/test_equipment.py +++ b/tests/test_equipment.py @@ -1,15 +1,13 @@ """This file contains all the tests to do with Equipment use logging and availability checks.""" import os -import time from pathlib import Path -from typing import Union, Dict, Iterable - -import pytest +from typing import Dict, Iterable import numpy as np import pandas as pd +import pytest -from tlo import Simulation, Module, Date, logging +from tlo import Date, Module, Simulation from tlo.analysis.utils import parse_log_file from tlo.events import IndividualScopeEventMixin from tlo.methods import Metadata, demography, healthsystem @@ -94,7 +92,8 @@ def test_core_functionality_of_equipment_class(seed): # - calling an item for which data on availability is not provided (should not raise error) eq_default.is_all_items_available(item_codes={3}, facility_id=1) - # - calling an item at a facility that for which data is not provided (should give average behaviour for other facilities) + # - calling an item at a facility that for which data is not provided (should give average behaviour for other + # facilities) assert not eq_default.is_all_items_available(item_codes={0}, facility_id=2) assert eq_default.is_all_items_available(item_codes={1}, facility_id=2) # - calling a recognised item for which no data at a facility with no data (should not error) @@ -147,7 +146,8 @@ def test_core_functionality_of_equipment_class(seed): # Lookup the item_codes that belong in a particular package. # - When package is recognised - assert {2, 3} == eq_default.lookup_item_codes_from_pkg_name(pkg_name='PkgWith2+3') # these items are in the same package + assert {2, 3} == eq_default.lookup_item_codes_from_pkg_name(pkg_name='PkgWith2+3') # these items are in the same + # package # - Error thrown when package is not recognised with pytest.raises(ValueError): eq_default.lookup_item_codes_from_pkg_name(pkg_name='') From 13d5a3c6d6c3e26557d1e9f8f6d2ec981f1fcdfc Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Fri, 10 May 2024 17:39:08 +0100 Subject: [PATCH 084/118] update call in bed-days --- src/tlo/methods/healthsystem.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 28b7576aa0..870735f27f 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -2537,14 +2537,14 @@ def apply(self, population): treatment_id='Inpatient_Care', facility_level=self.module._facility_by_facility_id[_fac_id].level, appt_footprint=tuple(sorted(_inpatient_appts.items())), - beddays_footprint=() + beddays_footprint=(), + equipment=set(), ), person_id=-1, facility_id=_fac_id, squeeze_factor=0.0, priority=-1, did_run=True, - equipment=set(), # TODO: explore more, should it be non-empty in some cases? ) # Restart the total footprint of all calls today, beginning with those due to existing in-patients. From 3d4b80df68f0198ddfe35241cd428cef3d97e20b Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Sun, 12 May 2024 10:36:06 +0100 Subject: [PATCH 085/118] use hashable type in HSIEventDetails --- src/tlo/methods/healthsystem.py | 2 +- src/tlo/methods/hsi_event.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 870735f27f..95b90d44ac 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -2538,7 +2538,7 @@ def apply(self, population): facility_level=self.module._facility_by_facility_id[_fac_id].level, appt_footprint=tuple(sorted(_inpatient_appts.items())), beddays_footprint=(), - equipment=set(), + equipment=tuple(), # Equipment is normally a set, but this has to be hashable. ), person_id=-1, facility_id=_fac_id, diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index 487d4fdabb..61fdf9d188 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -182,10 +182,12 @@ def post_apply_hook(self) -> None: person_id=self.target, footprint=self.bed_days_allocated_to_this_event ) - self.healthcare_system.equipment.record_use_of_equipment( - item_codes=self._EQUIPMENT, - facility_id=self.facility_info.id - ) + if self.facility_info is not None: + # If there is a facility_info (e.g., healthsystem not running in disabled mode), then record equipment used + self.healthcare_system.equipment.record_use_of_equipment( + item_codes=self._EQUIPMENT, + facility_id=self.facility_info.id + ) def run(self, squeeze_factor): """Make the event happen.""" From 3bb2581707c20fe55b468133d257fed976a28b16 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Sun, 12 May 2024 11:30:12 +0100 Subject: [PATCH 086/118] update logic and add docstring --- src/tlo/methods/equipment.py | 72 +++++++++++++++++++++++++----------- src/tlo/methods/hsi_event.py | 37 +++++++++++------- tests/test_equipment.py | 8 ++-- 3 files changed, 78 insertions(+), 39 deletions(-) diff --git a/src/tlo/methods/equipment.py b/src/tlo/methods/equipment.py index 3ba34eca4a..f70aada5a2 100644 --- a/src/tlo/methods/equipment.py +++ b/src/tlo/methods/equipment.py @@ -1,6 +1,6 @@ import warnings from collections import defaultdict -from typing import Counter, Dict, Iterable, Optional, Set, Union +from typing import Counter, Iterable, Optional, Set, Union import numpy as np import pandas as pd @@ -14,6 +14,33 @@ class Equipment: """This is the Equipment Class. It maintains a current record of the availability of equipment in the HealthSystem. It is expected that this is instantiated by the `HealthSystem` module. + The basic paradigm is that an HSI_Event can declare equipment that is required for delivering the healthcare + service that the HSI_Event represents. The HSI_Event uses `self.add_equipment()` to make these declaration, with + reference to the items of equipment that are defined in `ResourceFile_EquipmentCatalogue.csv`. (These declaration + can be in the form of the descriptor or the equipment item code). These declarations can be used when the HSI_Event + is created but before it is run (in `__init__`), or during execution of the HSI_Event (in `apply`). + + As the HSI_Event can declare equipment that is required before it is run, the HealthSystem _can_ use this to + prevent an HSI_Event running if the equipment declared is not available. Note that for equipment that is declared + whilst the HSI_Event is running, there are no checks on availabilty, and the HSI_Event is allowed to continue + running even if equipment is declared that is not available. For this reason, HSI_Event should declare equipment + that is _essential_ for the healthcare service in its `__init__` method. If the logic inside the `__apply__` method + of the HSI_Event depends on the availability of equipment, then the events can find the probability with which + item(s) will be available to it, using `self.probability_equipment_available()`. + + The data on the availability of equipment data refers to the proportion of facilities in a district of a + particular level (i.e., the Facility_ID) that do have that piece of equipment. In the model, we do not know + which facility the person is attending (there are many actual facilities grouped together into one Facility_ID in + the model). Therefore, the determination of whether equipment is available is made probabilistically for the + HSI_Event (i.e., the probability that the actual facility being attended by the person has the equipment is + represented by the proportion of such facilities that do have that equipment). It is assumed that the probabilities + of each item being available are independent of one other (so that the probability of all items being available is + the product of the probabilities of each item). This probabilistic determination of availability is only done + _once_ for the HSI_Event: i.e., if the equipment is not available for the instance of the HSI_Event, then it will + remain not available if the same event is re-scheduled/re-entered into the HealthSystem queue. This represents + that if the facility that a particular person attends for the HSI_Event does not have the equipment available, then + it will not be available on another day. + :param: 'catalogue': The database of all recognised item_codes. :param: `data_availability`: Specifies the probability with which each equipment (identified by an `item_code`) is @@ -45,18 +72,18 @@ def __init__( self.master_facilities_list = master_facilities_list # Create internal storage structures - self._items_available: Dict = dict() # <-- Will be the internal store of which items are available at each - # facility_id. This is of the form {facility_id: {items_available}}. - - self._record_of_equipment_used_by_facility_id = defaultdict(Counter) # <-- Will be the internal store of which - # items have been used at each facility_id This is of the form {facility_id: {item_code: count}}. + # - Probabilities of items being available at each facility_id + self._probabilities_of_items_available = pd.DataFrame() + # - Internal store of which items have been used at each facility_id This is of the form + # {facility_id: {item_code: count}}. + self._record_of_equipment_used_by_facility_id = defaultdict(Counter) # <-- Will be the - # Data structures for quick look-ups for items and descriptors + # - Data structures for quick look-ups for items and descriptors self._item_code_lookup = self.catalogue.set_index('Item_Description')['Item_Code'].to_dict() self._all_item_descriptors = set(self._item_code_lookup.keys()) self._all_item_codes = set(self._item_code_lookup.values()) - # Initialise the internal stores of equipment items that are available, ready for calls. + # Initialise the internal stores of the probability with which equipment items that are available. self._set_equipment_items_available(availability=availability) def on_simulation_end(self): @@ -111,15 +138,7 @@ def _set_equipment_items_available(self, availability: str): else: raise KeyError(f"Unknown equipment availability specified: {availability}") - # Sample these probability to find which items are actually available - is_available = df > self.rng.random(size=len(df)) - - # Organise into dict of set, of the form: {facility_id: {items_available}} for known facility_ids - # (N.B. Has to be done this way around in order to guarantee that we have each known facility_id in the keys - # even if there are no item available.) - self._items_available: Dict = is_available.groupby("Facility_ID").agg( - lambda x: set(x[x].index.get_level_values("Item_Code")) - ).to_dict() + self._probabilities_of_items_available = df def parse_items(self, items: Union[int, str, Iterable[int | str]]) -> Set[int]: """Parse equipment items specified as an item_code (integer), an item descriptor (string), or an iterable of @@ -151,16 +170,25 @@ def check_item_descriptors_recognised(item_descriptors: set[str]): # In the return, any unrecognised descriptors are silently ignored. return set(filter(lambda item: item is not None, map(self._item_code_lookup.get, items))) - def is_all_items_available( + def probability_equipment_available( self, item_codes: Set[int], facility_id: int - ) -> bool: - """Determine if all equipments are available at the given facility_id (or from the default if the facility_id - is not recognised). Returns True only if all items are available at the facility_id, otherwise returns False.""" + ) -> float: + """Returns the probability that all the equipment item_codes are available. It does so by looking at the + probabilities of each equipment item being available and multiplying these together to find the probability + that _all_ are available.""" try: - return item_codes.issubset(self._items_available[facility_id]) + return self._probabilities_of_items_available.loc[(facility_id, list(item_codes))].prod() except KeyError: raise ValueError(f'Not recognised {facility_id=}') + def is_all_items_available( + self, item_codes: Set[int], facility_id: int + ) -> bool: + """Determine if all equipments are available at the given facility_id. Returns True only if all items are + available at the facility_id, otherwise returns False.""" + return self.rng.random_sample() < self.probability_equipment_available(item_codes=item_codes, + facility_id=facility_id) + def record_use_of_equipment( self, item_codes: Set[int], facility_id: int ) -> None: diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index 61fdf9d188..cd00451087 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -109,10 +109,11 @@ def __init__(self, module, *args, **kwargs): self.module = module super().__init__(*args, **kwargs) - # Information that will later be received about this HSI + # Information that will later be received/computed about this HSI self._received_info_about_bed_days = None self.expected_time_requests = {} self.facility_info = None + self._is_all_declared_equipment_available = None self.TREATMENT_ID = "" self.ACCEPTED_FACILITY_LEVEL = None @@ -288,18 +289,28 @@ def add_equipment(self, item_codes: Union[int, str, Iterable[int | str]]): @property def is_all_declared_equipment_available(self) -> bool: - """Returns `True` if all the declared items of equipment are available. This is called before the HSI is run - and so is looking only at those items that are declared when this instance was created.""" - return self.healthcare_system.equipment.is_all_items_available( - item_codes=self._EQUIPMENT, - facility_id=self.facility_info.id, - ) - - def is_equipment_available(self, item_codes: Union[int, str, Iterable[int | str]]) -> bool: - """Check whether all the equipment item_codes are available. This does not imply that the equipment is being - used and no logging happens. It is provided as a convenience to disease module authors in case the logic of - during an HSI Event depends on the availability of a piece of equipment.""" - return self.healthcare_system.equipment.is_all_items_available( + """Returns `True` if all the declared items of equipment are available. This is called by the HealthSystem + before the HSI is run and so is looking only at those items that are declared when this instance was created. + The evaluation of whether equipment is available is only done _once_ for this instance of the event: i.e., if + the equipment is not available for the instance of the HSI_Event, then it will remain not available if the + same event is re-scheduled/re-entered into the HealthSystem queue. This is representing that if the facility + that a particular person attends for the HSI_Event does not have the equipment available, then it will not + be available on another day.""" + + if self._is_all_declared_equipment_available is None: + # Availability has not already been evaluated: determine availability + self._is_all_declared_equipment_available = self.healthcare_system.equipment.is_all_items_available( + item_codes=self._EQUIPMENT, + facility_id=self.facility_info.id, + ) + return self._is_all_declared_equipment_available + + def probability_equipment_available(self, item_codes: Union[int, str, Iterable[int | str]]) -> float: + """Returns the probability that all the equipment item_codes are available. This does not imply that the + equipment is being used and no logging happens. It is provided as a convenience to disease module authors in + case the logic of during an HSI Event depends on the availability of a piece of equipment. This function + accepts the item codes/descriptions in a variety of formats, so this needs to be parsed.""" + return self.healthcare_system.equipment.probability_equipment_available( item_codes=self.healthcare_system.equipment.parse_items(item_codes), facility_id=self.facility_info.id, ) diff --git a/tests/test_equipment.py b/tests/test_equipment.py index 44899f5c19..45884a4034 100644 --- a/tests/test_equipment.py +++ b/tests/test_equipment.py @@ -339,7 +339,7 @@ def did_the_hsi_run(log: Dict) -> bool: def test_change_equipment_availability(seed): - """Test that we can change the availability of equipment midway through the simulation.""" + """Test that we can change the probability of the availability of equipment midway through the simulation.""" # Set-up simulation that starts with `all` availability and then changes to `none` after one year. In the # simulation a DummyModule schedules a DummyHSI that runs every month and tries to get a piece of equipment. # Check that this piece of equipment is available for the first year but not the second year of the simulation. @@ -360,7 +360,7 @@ def apply(self, person_id, squeeze_factor): # Check availability of a piece of equipment, with item_code = 0 self.store_of_equipment_checks.update( { - self.sim.date: self.is_equipment_available(item_codes={0}) + self.sim.date: self.probability_equipment_available(item_codes={0}) } ) @@ -410,5 +410,5 @@ def initialise_simulation(self, sim): # Get store & check for availabilities of the equipment log = pd.Series(sim.modules['DummyModule'].the_hsi_event.store_of_equipment_checks) - assert log[log.index < Date(2011, 1, 1)].all() - assert not log[log.index >= Date(2011, 1, 1)].any() + assert (1.0 == log[log.index < Date(2011, 1, 1)]).all() + assert (0.0 == log[log.index >= Date(2011, 1, 1)]).all() From 3c16898ea90809aab8e9c6fd73e4bd0df29f77f7 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Mon, 13 May 2024 09:34:27 +0100 Subject: [PATCH 087/118] linting --- src/tlo/methods/equipment.py | 96 +++++++++++++++++++----------------- src/tlo/methods/hsi_event.py | 27 +++++----- 2 files changed, 66 insertions(+), 57 deletions(-) diff --git a/src/tlo/methods/equipment.py b/src/tlo/methods/equipment.py index f70aada5a2..50bd866856 100644 --- a/src/tlo/methods/equipment.py +++ b/src/tlo/methods/equipment.py @@ -14,47 +14,54 @@ class Equipment: """This is the Equipment Class. It maintains a current record of the availability of equipment in the HealthSystem. It is expected that this is instantiated by the `HealthSystem` module. - The basic paradigm is that an HSI_Event can declare equipment that is required for delivering the healthcare - service that the HSI_Event represents. The HSI_Event uses `self.add_equipment()` to make these declaration, with - reference to the items of equipment that are defined in `ResourceFile_EquipmentCatalogue.csv`. (These declaration - can be in the form of the descriptor or the equipment item code). These declarations can be used when the HSI_Event - is created but before it is run (in `__init__`), or during execution of the HSI_Event (in `apply`). + The basic paradigm is that an `HSI_Event` can declare equipment that is required for delivering the healthcare + service that the `HSI_Event` represents. The `HSI_Event` uses `self.add_equipment()` to make these declaration, + with reference to the items of equipment that are defined in `ResourceFile_EquipmentCatalogue.csv`. (These + declaration can be in the form of the descriptor or the equipment item code). These declarations can be used when + the `HSI_Event` is created but before it is run (in `__init__`), or during execution of the HSI_Event (in `apply`). As the HSI_Event can declare equipment that is required before it is run, the HealthSystem _can_ use this to prevent an HSI_Event running if the equipment declared is not available. Note that for equipment that is declared - whilst the HSI_Event is running, there are no checks on availabilty, and the HSI_Event is allowed to continue - running even if equipment is declared that is not available. For this reason, HSI_Event should declare equipment - that is _essential_ for the healthcare service in its `__init__` method. If the logic inside the `__apply__` method - of the HSI_Event depends on the availability of equipment, then the events can find the probability with which - item(s) will be available to it, using `self.probability_equipment_available()`. + whilst the HSI_Event is running, there are no checks on availability, and the HSI_Event is allowed to continue + running even if equipment is declared is not available. For this reason, the `HSI_Event` should declare equipment + that is _essential_ for the healthcare service in its `__init__` method. If the logic inside the `apply` method + of the `HSI_Event` depends on the availability of equipment, then it can find the probability with which + item(s) will be available using `self.probability_equipment_available()`. The data on the availability of equipment data refers to the proportion of facilities in a district of a - particular level (i.e., the Facility_ID) that do have that piece of equipment. In the model, we do not know - which facility the person is attending (there are many actual facilities grouped together into one Facility_ID in - the model). Therefore, the determination of whether equipment is available is made probabilistically for the - HSI_Event (i.e., the probability that the actual facility being attended by the person has the equipment is - represented by the proportion of such facilities that do have that equipment). It is assumed that the probabilities - of each item being available are independent of one other (so that the probability of all items being available is - the product of the probabilities of each item). This probabilistic determination of availability is only done - _once_ for the HSI_Event: i.e., if the equipment is not available for the instance of the HSI_Event, then it will - remain not available if the same event is re-scheduled/re-entered into the HealthSystem queue. This represents - that if the facility that a particular person attends for the HSI_Event does not have the equipment available, then - it will not be available on another day. + particular level (i.e., the `Facility_ID`) that do have that piece of equipment. In the model, we do not know + which actual facility the person is attending (there are many actual facilities grouped together into one + `Facility_ID` in the model). Therefore, the determination of whether equipment is available is made + probabilistically for the `HSI_Event` (i.e., the probability that the actual facility being attended by the + person has the equipment is represented by the proportion of such facilities that do have that equipment). It is + assumed that the probabilities of each item being available are independent of one other (so that the + probability of all items being available is the product of the probabilities for each item). This probabilistic + determination of availability is only done _once_ for the `HSI_Event`: i.e., if the equipment is determined to + not be available for the instance of the `HSI_Event`, then it will remain not available if the same event is + re-scheduled / re-entered into the HealthSystem queue. This represents that if the facility that a particular + person attends for the `HSI_Event` does not have the equipment available, then it will still not be available on + another day. + + Where data on availability is not provided for an item, the probability of availability is inferred from the + average availability of other items in that `Facility_ID`. Likewise, the probability of an item being available + at `Facility_ID` is inferred from the average availability of that item at other facilities. If an item_code is + referred that is not recognised (not included in `catalogue`), a `UserWarning` is issued. If a facility_id is + referred that is not recognised (not included in `master_facilities_list`), an `Error` is raised. :param: 'catalogue': The database of all recognised item_codes. :param: `data_availability`: Specifies the probability with which each equipment (identified by an `item_code`) is - available at a facility level. Note that information is not necessarily provided for every item in the `catalogue`. + available at a facility level. Note that information is not necessarily provided for every item in the `catalogue` + or every facility_id in the `master_facilities_list`. :param: `rng`: The Random Number Generator object to use for random numbers. :param: `availability`: Determines the mode availability of the equipment. If 'default' then use the availability - specified in the ResourceFile; if 'none', then let no equipment be ever be available; if 'all', then all + specified in the `data_availability`; if 'none', then let no equipment be ever be available; if 'all', then all equipment is always available. - :param `: `master_facilities_list`: The pd.DataFrame with line-list of all the facilities in the HealthSystem. + :param `: `master_facilities_list`: The pd.DataFrame with the line-list of all the facilities in the HealthSystem. - If an item_code is referred that is not recognised (not included in `catalogue`), a `UserWarning` is issued. """ def __init__( @@ -87,27 +94,27 @@ def __init__( self._set_equipment_items_available(availability=availability) def on_simulation_end(self): - """Things to do when the simulation end: + """Things to do when the simulation ends: * Log (to the summary logger) the equipment that has been used. """ self.write_to_log() def update_availability(self, availability: str) -> None: """Update the availability of equipment. This is expected to be called midway through the simulation if - the assumption of the equipment availability needs to change.""" + the assumption of the equipment availability is changed.""" self._set_equipment_items_available(availability=availability) def _set_equipment_items_available(self, availability: str): - """Update internal store of which items of equipment are available. This is called at the beginning of the - simulation and whenever an update in `availability` is done by `update_availability`.""" + """Update internal store of probabilities of items of equipment being available. This is called at the beginning + of the simulation and whenever an update in `availability` is done by `update_availability`.""" - # For any facility_id in the data + # All facility_id in the simulation all_fac_ids = self.master_facilities_list['Facility_ID'].unique() # All equipment items in the catalogue all_eq_items = self.catalogue["Item_Code"].unique() - # Create full dataset, where we force that there is probability of availability for every item_code at every + # Create "full" dataset, where we force that there is probability of availability for every item_code at every # observed facility df = pd.Series( index=pd.MultiIndex.from_product( @@ -120,7 +127,7 @@ def _set_equipment_items_available(self, availability: str): ] ) - # Merge in original dataset and use the mean in that facility_id to impute availability of missing item_code + # Merge in original dataset and use the mean in that facility_id to impute availability of missing item_codes df = df.groupby("Facility_ID").transform(lambda x: x.fillna(x.mean())) # ... and also impute availability for any facility_ids for which no data, based on all other facilities df = df.groupby("Item_Code").transform(lambda x: x.fillna(x.mean())) @@ -132,12 +139,13 @@ def _set_equipment_items_available(self, availability: str): if availability == "default": pass elif availability == "all": - df = (df + 1).clip(upper=1.0) + df = (df + 1).clip(upper=1.0) # All probabilities -> 1.0 elif availability == "none": - df = df.mul(0.0) + df = df.mul(0.0) # All probabilities -> 0.0 else: - raise KeyError(f"Unknown equipment availability specified: {availability}") + raise KeyError(f"Unknown equipment availability specified: {availability=}") + # Save self._probabilities_of_items_available = df def parse_items(self, items: Union[int, str, Iterable[int | str]]) -> Set[int]: @@ -170,7 +178,7 @@ def check_item_descriptors_recognised(item_descriptors: set[str]): # In the return, any unrecognised descriptors are silently ignored. return set(filter(lambda item: item is not None, map(self._item_code_lookup.get, items))) - def probability_equipment_available( + def probability_all_equipment_available( self, item_codes: Set[int], facility_id: int ) -> float: """Returns the probability that all the equipment item_codes are available. It does so by looking at the @@ -184,10 +192,10 @@ def probability_equipment_available( def is_all_items_available( self, item_codes: Set[int], facility_id: int ) -> bool: - """Determine if all equipments are available at the given facility_id. Returns True only if all items are + """Determine if all equipment items are available at the given facility_id. Returns True only if all items are available at the facility_id, otherwise returns False.""" - return self.rng.random_sample() < self.probability_equipment_available(item_codes=item_codes, - facility_id=facility_id) + return self.rng.random_sample() < self.probability_all_equipment_available(item_codes=item_codes, + facility_id=facility_id) def record_use_of_equipment( self, item_codes: Set[int], facility_id: int @@ -199,8 +207,8 @@ def write_to_log(self) -> None: """Write to the log: * Summary of the equipment that was _ever_ used at each district/facility level. Note that the info-level health system logger (key: `hsi_event_counts`) contains logging of the equipment used - in each HSI event (if further finer splits are needed). Alternatively, different aggregations could be created - here for the summary logger, using the same pattern as used here. + in each HSI event (if finer splits are needed). Alternatively, different aggregations could be created here for + the summary logger, using the same pattern as used here. """ mfl = self.master_facilities_list @@ -211,7 +219,7 @@ def set_of_keys_or_empty_set(x: Union[set, dict]): elif isinstance(x, dict): return set(x.keys()) else: - return None + return set() set_of_equipment_ever_used_at_each_facility_id = pd.Series({ fac_id: set_of_keys_or_empty_set(self._record_of_equipment_used_by_facility_id.get(fac_id, set())) @@ -229,7 +237,7 @@ def set_of_keys_or_empty_set(x: Union[set, dict]): for _, row in output.iterrows(): logger_summary.info( key='EquipmentEverUsed_ByFacilityID', - description='For each facility_id (the set of facilities of the same level in a district), the set' + description='For each facility_id (the set of facilities of the same level in a district), the set of' 'equipment items that are ever used.', data=row.to_dict(), ) @@ -237,7 +245,7 @@ def set_of_keys_or_empty_set(x: Union[set, dict]): def lookup_item_codes_from_pkg_name(self, pkg_name: str) -> Set[int]: """Convenience function to find the set of item_codes that are grouped under a package name in the catalogue. It is expected that this is used by the disease module once and then the resulting equipment item_codes are - saved on th at module. Note that all interaction with the `Equipment` module is using set of item_codes.""" + saved on the module.""" df = self.catalogue if pkg_name not in df['Pkg_Name'].unique(): diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index cd00451087..64d2ee5930 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -281,21 +281,22 @@ def make_appt_footprint(self, dict_of_appts) -> Counter: "values" ) - def add_equipment(self, item_codes: Union[int, str, Iterable[int | str]]): - """Declare that piece(s) of equipment are used in this HSI_Event. - Checks are done on the validity of the item_codes/item descriptions and warnings issued if they are not + def add_equipment(self, item_codes: Union[int, str, Iterable[int | str]]) -> None: + """Declare that piece(s) of equipment are used in this HSI_Event. Equipment items can be identified by their + item_codes (int) or descriptors (str); a singular item or an iterable of items can be defined at once. Checks + are done on the validity of the item_codes/item descriptions and a warning issued if any are not recognised.""" self._EQUIPMENT.update(self.healthcare_system.equipment.parse_items(item_codes)) @property def is_all_declared_equipment_available(self) -> bool: - """Returns `True` if all the declared items of equipment are available. This is called by the HealthSystem - before the HSI is run and so is looking only at those items that are declared when this instance was created. - The evaluation of whether equipment is available is only done _once_ for this instance of the event: i.e., if - the equipment is not available for the instance of the HSI_Event, then it will remain not available if the - same event is re-scheduled/re-entered into the HealthSystem queue. This is representing that if the facility - that a particular person attends for the HSI_Event does not have the equipment available, then it will not - be available on another day.""" + """Returns `True` if all the (currently) declared items of equipment are available. This is called by the + `HealthSystem` module before the HSI is run and so is looking only at those items that are declared when this + instance was created. The evaluation of whether equipment is available is only done _once_ for this instance of + the event: i.e., if the equipment is not available for the instance of this `HSI_Event`, then it will remain not + available if the same event is re-scheduled/re-entered into the HealthSystem queue. This is representing that + if the facility that a particular person attends for the HSI_Event does not have the equipment available, then + it will also not be available on another day.""" if self._is_all_declared_equipment_available is None: # Availability has not already been evaluated: determine availability @@ -308,9 +309,9 @@ def is_all_declared_equipment_available(self) -> bool: def probability_equipment_available(self, item_codes: Union[int, str, Iterable[int | str]]) -> float: """Returns the probability that all the equipment item_codes are available. This does not imply that the equipment is being used and no logging happens. It is provided as a convenience to disease module authors in - case the logic of during an HSI Event depends on the availability of a piece of equipment. This function - accepts the item codes/descriptions in a variety of formats, so this needs to be parsed.""" - return self.healthcare_system.equipment.probability_equipment_available( + case the logic during an `HSI_Event` depends on the availability of a piece of equipment. This function + accepts the item codes/descriptions in a variety of formats, so the argument needs to be parsed.""" + return self.healthcare_system.equipment.probability_all_equipment_available( item_codes=self.healthcare_system.equipment.parse_items(item_codes), facility_id=self.facility_info.id, ) From 5bcdb3885dd7e81f925a95bd5be82ff900167731 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Mon, 13 May 2024 09:54:29 +0100 Subject: [PATCH 088/118] linting --- src/tlo/methods/healthsystem.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 95b90d44ac..f6ced798b9 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -1105,8 +1105,9 @@ def get_beds_availability(self) -> str: return _beds_availability def get_equip_availability(self) -> str: - """Returns equipment availability. (Should be equal to what is specified by the parameter, but overwrite with - what was provided in argument if an argument was specified -- provided for backward compatibility/debugging.)""" + """Returns equipment availability. (Should be equal to what is specified by the parameter, but can be + overwritten with what was provided in argument if an argument was specified -- provided for backward + compatibility/debugging.)""" if self.arg_equip_availability is None: _equip_availability = self.parameters['equip_availability'] @@ -1894,10 +1895,7 @@ def on_end_of_year(self) -> None: # If we are at the end of the year preceeding the mode switch, and if wanted # to rescale capabilities to capture effective availability as was recorded, on # average, in the past year, do so here. - if ( - (self.sim.date.year == self.parameters['year_mode_switch'] - 1) - and self.parameters['scale_to_effective_capabilities'] - ): + if (self.sim.date.year == self.parameters['year_mode_switch'] - 1) and self.parameters['scale_to_effective_capabilities']: self._rescale_capabilities_to_capture_effective_capability() self._summary_counter.write_to_log_and_reset_counters() self.consumables.on_end_of_year() @@ -1962,7 +1960,7 @@ def run_individual_level_events_in_mode_0_or_1(self, # Mode 0: All HSI Event run, with no squeeze # Mode 1: All HSI Events run with squeeze provided latter is not inf ok_to_run = True - # todo - also consider whether essential equipment is available + if self.mode_appt_constraints == 1 and squeeze_factor == float('inf'): ok_to_run = False From 3a8d3da97dad990ed5a12e0acc1a2181077e0a5e Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Mon, 13 May 2024 10:06:38 +0100 Subject: [PATCH 089/118] linting --- tests/test_equipment.py | 50 ++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/tests/test_equipment.py b/tests/test_equipment.py index 45884a4034..061fbf2526 100644 --- a/tests/test_equipment.py +++ b/tests/test_equipment.py @@ -1,4 +1,4 @@ -"""This file contains all the tests to do with Equipment use logging and availability checks.""" +"""This file contains all the tests to do with Equipment.""" import os from pathlib import Path from typing import Dict, Iterable @@ -158,9 +158,9 @@ def test_core_functionality_of_equipment_class(seed): equipment_item_code_that_is_not_available = [2, 3,] def run_simulation_and_return_log( - seed, tmpdir, essential_equipment: Iterable[str], other_equipment: Iterable[str] + seed, tmpdir, equipment_in_init, equipment_in_apply ) -> Dict: - """Returns the parsed logs from `tlo.methods.healthsystem.summary:EquipmentEverUsed_ByFacilityID` from a + """Returns the parsed logs from `tlo.methods.healthsystem.summary` from a simulation object in which a single event has been run with the specified equipment usage. The availability of equipment has been manipulated so that the item_codes given in `equipment_item_code_that_is_available` and `equipment_item_code_that_is_not_available` are as expected. """ @@ -222,7 +222,7 @@ def initialise_simulation(self, sim): demography.Demography(resourcefilepath=resourcefilepath), healthsystem.HealthSystem(resourcefilepath=resourcefilepath), DummyModule( - essential_equipment=essential_equipment, other_equipment=other_equipment + essential_equipment=equipment_in_init, other_equipment=equipment_in_apply ), ) @@ -253,7 +253,7 @@ def test_equipment_use_is_logged(seed, tmpdir): another_item_code = equipment_item_code_that_is_available[1] def all_equipment_ever_used(log: Dict) -> set: - """With the log of equipment used in the simulation, return a set of the equipments item that have been used + """With the log of equipment used in the simulation, return a set of the equipment item that have been used (at any facility).""" s = set() for i in log["EquipmentEverUsed_ByFacilityID"]['EquipmentEverUsed']: @@ -265,45 +265,45 @@ def all_equipment_ever_used(log: Dict) -> set: run_simulation_and_return_log( seed=seed, tmpdir=tmpdir, - essential_equipment=set(), - other_equipment=set(), + equipment_in_init=set(), + equipment_in_apply=set(), ) ) - # * An HSI that declares use of equipment during its `apply` method (but no essential equipment) + # * An HSI that declares use of equipment only during its `apply` method. assert {the_item_code} == all_equipment_ever_used( run_simulation_and_return_log( seed=seed, tmpdir=tmpdir, - essential_equipment=set(), - other_equipment=the_item_code, + equipment_in_init=set(), + equipment_in_apply=the_item_code, ) ) - # * An HSI that declare use of essential equipment but nothing in its `apply` method`; + # * An HSI that declare use of equipment only in its `__init__` method assert {the_item_code} == all_equipment_ever_used( run_simulation_and_return_log( seed=seed, tmpdir=tmpdir, - essential_equipment=the_item_code, - other_equipment=set(), + equipment_in_init=the_item_code, + equipment_in_apply=set(), ) ) - # * An HSI that declare use of essential equipment and equipment during its `apply` method; + # * An HSI that declare use of equipment in `__init__` _and_ `apply`. assert {the_item_code, another_item_code} == all_equipment_ever_used( run_simulation_and_return_log( seed=seed, tmpdir=tmpdir, - essential_equipment=the_item_code, - other_equipment=another_item_code, + equipment_in_init=the_item_code, + equipment_in_apply=another_item_code, ) ) -def test_hsi_does_not_run_if_essential_equipment_is_not_available(seed, tmpdir): - """Check that an HSI which declares an item of equipment that is essential does run if that item is available - and does not run if that item is not available.""" +def test_hsi_does_not_run_if_equipment_declared_in_init_is_not_available(seed, tmpdir): + """Check that an HSI which declares an item of equipment that is declared in the HSI_Event's __init__ does run if + that item is available and does not run if that item is not available.""" def did_the_hsi_run(log: Dict) -> bool: """Read the log to work out if the `DummyHSIEvent` ran or not.""" @@ -322,8 +322,8 @@ def did_the_hsi_run(log: Dict) -> bool: run_simulation_and_return_log( seed=seed, tmpdir=tmpdir, - essential_equipment=equipment_item_code_that_is_available, - other_equipment=set(), + equipment_in_init=equipment_item_code_that_is_available, + equipment_in_apply=set(), ) ) @@ -332,8 +332,8 @@ def did_the_hsi_run(log: Dict) -> bool: run_simulation_and_return_log( seed=seed, tmpdir=tmpdir, - essential_equipment=equipment_item_code_that_is_not_available, - other_equipment=set(), + equipment_in_init=equipment_item_code_that_is_not_available, + equipment_in_apply=set(), ) ) @@ -341,8 +341,8 @@ def did_the_hsi_run(log: Dict) -> bool: def test_change_equipment_availability(seed): """Test that we can change the probability of the availability of equipment midway through the simulation.""" # Set-up simulation that starts with `all` availability and then changes to `none` after one year. In the - # simulation a DummyModule schedules a DummyHSI that runs every month and tries to get a piece of equipment. - # Check that this piece of equipment is available for the first year but not the second year of the simulation. + # simulation a DummyModule schedules a DummyHSI that runs every month and tries to get a piece of equipment; + # then, check that the probability that this piece of equipment is available each month during the simulation. class DummyHSIEvent(HSI_Event, IndividualScopeEventMixin): def __init__( From dc9d9b99f068f93c621617244da69dd5db70046c Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Mon, 13 May 2024 11:17:42 +0100 Subject: [PATCH 090/118] linting --- src/tlo/methods/healthsystem.py | 5 ++++- tests/test_equipment.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index f6ced798b9..f4c22f5ead 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -1895,7 +1895,10 @@ def on_end_of_year(self) -> None: # If we are at the end of the year preceeding the mode switch, and if wanted # to rescale capabilities to capture effective availability as was recorded, on # average, in the past year, do so here. - if (self.sim.date.year == self.parameters['year_mode_switch'] - 1) and self.parameters['scale_to_effective_capabilities']: + if ( + (self.sim.date.year == self.parameters['year_mode_switch'] - 1) + and self.parameters['scale_to_effective_capabilities'] + ): self._rescale_capabilities_to_capture_effective_capability() self._summary_counter.write_to_log_and_reset_counters() self.consumables.on_end_of_year() diff --git a/tests/test_equipment.py b/tests/test_equipment.py index 061fbf2526..7a679fa611 100644 --- a/tests/test_equipment.py +++ b/tests/test_equipment.py @@ -1,7 +1,7 @@ """This file contains all the tests to do with Equipment.""" import os from pathlib import Path -from typing import Dict, Iterable +from typing import Dict import numpy as np import pandas as pd From ea4027bc9ecb5fec06bd256332373761c5913863 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Mon, 13 May 2024 11:20:13 +0100 Subject: [PATCH 091/118] provide default for 'equip_availability' in test_alri:get_sim() --- tests/test_alri.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_alri.py b/tests/test_alri.py index fb03312515..0fba5fea8d 100644 --- a/tests/test_alri.py +++ b/tests/test_alri.py @@ -54,7 +54,7 @@ def _get_person_id(df, age_bounds: tuple = (0.0, np.inf)) -> int: ].index[0] -def get_sim(tmpdir, seed, cons_available, equip_available): +def get_sim(tmpdir, seed, cons_available, equip_available='all'): """Return simulation objection with Alri and other necessary modules registered.""" sim = Simulation( start_date=start_date, From 55e3352d44064ed8ce27f1e4b1a3f8d8ba3672d2 Mon Sep 17 00:00:00 2001 From: willGraham01 <1willgraham@gmail.com> Date: Mon, 13 May 2024 11:59:23 +0100 Subject: [PATCH 092/118] Prevent recomputing of probabilities every time availability is updated --- src/tlo/methods/equipment.py | 110 ++++++++++++++++++++++++----------- 1 file changed, 77 insertions(+), 33 deletions(-) diff --git a/src/tlo/methods/equipment.py b/src/tlo/methods/equipment.py index 50bd866856..c579e7b64a 100644 --- a/src/tlo/methods/equipment.py +++ b/src/tlo/methods/equipment.py @@ -1,6 +1,6 @@ import warnings from collections import defaultdict -from typing import Counter, Iterable, Optional, Set, Union +from typing import Counter, Iterable, Literal, Optional, Set, Union import numpy as np import pandas as pd @@ -70,17 +70,18 @@ def __init__( data_availability: pd.DataFrame, rng: np.random, master_facilities_list: pd.DataFrame, - availability: Optional[str] = "default", + availability: Optional[Literal["all", "default", "none"]] = "default", ) -> None: # Store arguments self.catalogue = catalogue self.rng = rng self.data_availability = data_availability + self.availability = availability self.master_facilities_list = master_facilities_list # Create internal storage structures # - Probabilities of items being available at each facility_id - self._probabilities_of_items_available = pd.DataFrame() + self._probabilities_of_items_available = self._calculate_equipment_availability_probabilities() # - Internal store of which items have been used at each facility_id This is of the form # {facility_id: {item_code: count}}. self._record_of_equipment_used_by_facility_id = defaultdict(Counter) # <-- Will be the @@ -90,8 +91,12 @@ def __init__( self._all_item_descriptors = set(self._item_code_lookup.keys()) self._all_item_codes = set(self._item_code_lookup.values()) - # Initialise the internal stores of the probability with which equipment items that are available. - self._set_equipment_items_available(availability=availability) + # Initialise the internal stores of the probability with which + # equipment items that are available. + # This DF will not be used if availability is not all or none, + # but creating it here prevent _re_ computing it should the + # availability be updated mid-simulation. + self._calculate_equipment_availability_probabilities() def on_simulation_end(self): """Things to do when the simulation ends: @@ -99,15 +104,28 @@ def on_simulation_end(self): """ self.write_to_log() - def update_availability(self, availability: str) -> None: - """Update the availability of equipment. This is expected to be called midway through the simulation if - the assumption of the equipment availability is changed.""" - self._set_equipment_items_available(availability=availability) + def update_availability( + self, availability: Literal["all", "default", "none"] + ) -> None: + """ + Update the availability of equipment. + + This is expected to be called midway through the simulation if the + assumption of the equipment availability is changed. + """ + assert availability in ["all", "none", "default"], f"New availability parameter {availability} not recognised." + self.availability = availability - def _set_equipment_items_available(self, availability: str): - """Update internal store of probabilities of items of equipment being available. This is called at the beginning - of the simulation and whenever an update in `availability` is done by `update_availability`.""" + def _calculate_equipment_availability_probabilities(self) -> pd.DataFrame: + """ + Compute the probabilities that each equipment item is available (at a given + facility), for use when the equipment availability is set to "default". + The probabilities computed in this method are constant throughout the simulation, + however they will not be used when the equipment availability is "all" or "none". + Computing them once and storing the result allows us to avoid repeating this + calculation if the equipment availability change event occurs during the simulation. + """ # All facility_id in the simulation all_fac_ids = self.master_facilities_list['Facility_ID'].unique() @@ -135,15 +153,15 @@ def _set_equipment_items_available(self, availability: str): # Check no missing values assert not df.isnull().any() - # Over-write these data if `availability` argument specifies that `none` or `all` items should be available - if availability == "default": - pass - elif availability == "all": - df = (df + 1).clip(upper=1.0) # All probabilities -> 1.0 - elif availability == "none": - df = df.mul(0.0) # All probabilities -> 0.0 - else: - raise KeyError(f"Unknown equipment availability specified: {availability=}") + # # Over-write these data if `availability` argument specifies that `none` or `all` items should be available + # if availability == "default": + # pass + # elif availability == "all": + # df = (df + 1).clip(upper=1.0) # All probabilities -> 1.0 + # elif availability == "none": + # df = df.mul(0.0) # All probabilities -> 0.0 + # else: + # raise KeyError(f"Unknown equipment availability specified: {availability=}") # Save self._probabilities_of_items_available = df @@ -179,23 +197,49 @@ def check_item_descriptors_recognised(item_descriptors: set[str]): return set(filter(lambda item: item is not None, map(self._item_code_lookup.get, items))) def probability_all_equipment_available( - self, item_codes: Set[int], facility_id: int + self, facility_id: int, *item_codes: int ) -> float: - """Returns the probability that all the equipment item_codes are available. It does so by looking at the - probabilities of each equipment item being available and multiplying these together to find the probability - that _all_ are available.""" - try: - return self._probabilities_of_items_available.loc[(facility_id, list(item_codes))].prod() - except KeyError: - raise ValueError(f'Not recognised {facility_id=}') + """ + Returns the probability that all the equipment item_codes are available + at the given facility. + + It does so by looking at the probabilities of each equipment item being + available and multiplying these together to find the probability that _all_ + are available. + + :param facility_id: Facility at which to check for the equipment. + :param item_codes: Integer item codes corresponding to the equipment to check. + """ + # NOTE: Preserving the current implementation here - IE we always error if + # the facility ID or any of the item codes is not recognised. + # Not sure if this is intended behaviour or not + assert ( + facility_id in self._probabilities_of_items_available.index + ), f"Unrecognised facility ID: {facility_id}" + assert all( + [item_codes in self._probabilities_of_items_available.columns] + ), "At least one item code was unrecognised." + + if self.availability == "all": + return 1 + elif self.availability == "none": + return 0 + return self._probabilities_of_items_available.loc[ + (facility_id, list(item_codes)) + ].prod() def is_all_items_available( self, item_codes: Set[int], facility_id: int ) -> bool: - """Determine if all equipment items are available at the given facility_id. Returns True only if all items are - available at the facility_id, otherwise returns False.""" - return self.rng.random_sample() < self.probability_all_equipment_available(item_codes=item_codes, - facility_id=facility_id) + """ + Determine if all equipment items are available at the given facility_id. + Returns True only if all items are available at the facility_id, + otherwise returns False. + """ + return self.rng.random_sample() < self.probability_all_equipment_available( + facility_id=facility_id, + *item_codes + ) def record_use_of_equipment( self, item_codes: Set[int], facility_id: int From eba5e39c6bbff75444dabfc4c2a6337c9c32598f Mon Sep 17 00:00:00 2001 From: willGraham01 <1willgraham@gmail.com> Date: Mon, 13 May 2024 12:01:24 +0100 Subject: [PATCH 093/118] Remove commentted-out code and pass back return value --- src/tlo/methods/equipment.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/src/tlo/methods/equipment.py b/src/tlo/methods/equipment.py index c579e7b64a..e1468e9e0f 100644 --- a/src/tlo/methods/equipment.py +++ b/src/tlo/methods/equipment.py @@ -116,7 +116,7 @@ def update_availability( assert availability in ["all", "none", "default"], f"New availability parameter {availability} not recognised." self.availability = availability - def _calculate_equipment_availability_probabilities(self) -> pd.DataFrame: + def _calculate_equipment_availability_probabilities(self) -> pd.Series: """ Compute the probabilities that each equipment item is available (at a given facility), for use when the equipment availability is set to "default". @@ -153,18 +153,7 @@ def _calculate_equipment_availability_probabilities(self) -> pd.DataFrame: # Check no missing values assert not df.isnull().any() - # # Over-write these data if `availability` argument specifies that `none` or `all` items should be available - # if availability == "default": - # pass - # elif availability == "all": - # df = (df + 1).clip(upper=1.0) # All probabilities -> 1.0 - # elif availability == "none": - # df = df.mul(0.0) # All probabilities -> 0.0 - # else: - # raise KeyError(f"Unknown equipment availability specified: {availability=}") - - # Save - self._probabilities_of_items_available = df + return df def parse_items(self, items: Union[int, str, Iterable[int | str]]) -> Set[int]: """Parse equipment items specified as an item_code (integer), an item descriptor (string), or an iterable of From d3f926f55d1eb6ba02d13f13109713d8b4ee52af Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Mon, 13 May 2024 13:40:23 +0100 Subject: [PATCH 094/118] small updates following chnages from Will --- src/tlo/methods/equipment.py | 83 ++++++++++++++++-------------------- src/tlo/methods/hsi_event.py | 2 +- tests/test_equipment.py | 4 +- 3 files changed, 40 insertions(+), 49 deletions(-) diff --git a/src/tlo/methods/equipment.py b/src/tlo/methods/equipment.py index e1468e9e0f..4d8eab86a8 100644 --- a/src/tlo/methods/equipment.py +++ b/src/tlo/methods/equipment.py @@ -45,8 +45,9 @@ class Equipment: Where data on availability is not provided for an item, the probability of availability is inferred from the average availability of other items in that `Facility_ID`. Likewise, the probability of an item being available at `Facility_ID` is inferred from the average availability of that item at other facilities. If an item_code is - referred that is not recognised (not included in `catalogue`), a `UserWarning` is issued. If a facility_id is - referred that is not recognised (not included in `master_facilities_list`), an `Error` is raised. + referred in `add_equipment() that is not recognised (not included in `catalogue`), a `UserWarning` is issued, but + that item is then silently ignored. If a facility_id is ever referred that is not recognised (not included in + `master_facilities_list`), an `AssertionError` is raised. :param: 'catalogue': The database of all recognised item_codes. @@ -72,31 +73,26 @@ def __init__( master_facilities_list: pd.DataFrame, availability: Optional[Literal["all", "default", "none"]] = "default", ) -> None: - # Store arguments + # - Store arguments self.catalogue = catalogue self.rng = rng self.data_availability = data_availability self.availability = availability self.master_facilities_list = master_facilities_list - # Create internal storage structures - # - Probabilities of items being available at each facility_id - self._probabilities_of_items_available = self._calculate_equipment_availability_probabilities() - # - Internal store of which items have been used at each facility_id This is of the form - # {facility_id: {item_code: count}}. - self._record_of_equipment_used_by_facility_id = defaultdict(Counter) # <-- Will be the - # - Data structures for quick look-ups for items and descriptors self._item_code_lookup = self.catalogue.set_index('Item_Description')['Item_Code'].to_dict() self._all_item_descriptors = set(self._item_code_lookup.keys()) self._all_item_codes = set(self._item_code_lookup.values()) + self._all_fac_ids = self.master_facilities_list['Facility_ID'].unique() + + # - Probabilities of items being available at each facility_id + self._probabilities_of_items_available = self._calculate_equipment_availability_probabilities() + + # - Internal store of which items have been used at each facility_id This is of the form + # {facility_id: {item_code: count}}. + self._record_of_equipment_used_by_facility_id = defaultdict(Counter) - # Initialise the internal stores of the probability with which - # equipment items that are available. - # This DF will not be used if availability is not all or none, - # but creating it here prevent _re_ computing it should the - # availability be updated mid-simulation. - self._calculate_equipment_availability_probabilities() def on_simulation_end(self): """Things to do when the simulation ends: @@ -109,7 +105,7 @@ def update_availability( ) -> None: """ Update the availability of equipment. - + This is expected to be called midway through the simulation if the assumption of the equipment availability is changed. """ @@ -126,17 +122,11 @@ def _calculate_equipment_availability_probabilities(self) -> pd.Series: Computing them once and storing the result allows us to avoid repeating this calculation if the equipment availability change event occurs during the simulation. """ - # All facility_id in the simulation - all_fac_ids = self.master_facilities_list['Facility_ID'].unique() - - # All equipment items in the catalogue - all_eq_items = self.catalogue["Item_Code"].unique() - # Create "full" dataset, where we force that there is probability of availability for every item_code at every # observed facility - df = pd.Series( + dat = pd.Series( index=pd.MultiIndex.from_product( - [all_fac_ids, all_eq_items], names=["Facility_ID", "Item_Code"] + [self._all_fac_ids, self._all_item_codes], names=["Facility_ID", "Item_Code"] ), data=float("nan"), ).combine_first( @@ -146,14 +136,14 @@ def _calculate_equipment_availability_probabilities(self) -> pd.Series: ) # Merge in original dataset and use the mean in that facility_id to impute availability of missing item_codes - df = df.groupby("Facility_ID").transform(lambda x: x.fillna(x.mean())) + dat = dat.groupby("Facility_ID").transform(lambda x: x.fillna(x.mean())) # ... and also impute availability for any facility_ids for which no data, based on all other facilities - df = df.groupby("Item_Code").transform(lambda x: x.fillna(x.mean())) + dat = dat.groupby("Item_Code").transform(lambda x: x.fillna(x.mean())) # Check no missing values - assert not df.isnull().any() + assert not dat.isnull().any() - return df + return dat def parse_items(self, items: Union[int, str, Iterable[int | str]]) -> Set[int]: """Parse equipment items specified as an item_code (integer), an item descriptor (string), or an iterable of @@ -186,33 +176,29 @@ def check_item_descriptors_recognised(item_descriptors: set[str]): return set(filter(lambda item: item is not None, map(self._item_code_lookup.get, items))) def probability_all_equipment_available( - self, facility_id: int, *item_codes: int + self, facility_id: int, item_codes: Set[int] ) -> float: """ Returns the probability that all the equipment item_codes are available at the given facility. - + It does so by looking at the probabilities of each equipment item being available and multiplying these together to find the probability that _all_ are available. + NOTE: This will error if the facility ID or any of the item codes is not recognised. + :param facility_id: Facility at which to check for the equipment. :param item_codes: Integer item codes corresponding to the equipment to check. """ - # NOTE: Preserving the current implementation here - IE we always error if - # the facility ID or any of the item codes is not recognised. - # Not sure if this is intended behaviour or not - assert ( - facility_id in self._probabilities_of_items_available.index - ), f"Unrecognised facility ID: {facility_id}" - assert all( - [item_codes in self._probabilities_of_items_available.columns] - ), "At least one item code was unrecognised." + + assert facility_id in self._all_fac_ids, f"Unrecognised facility ID: {facility_id=}" + assert item_codes.issubset(self._all_item_codes), f"At least one item code was unrecognised: {item_codes=}" if self.availability == "all": - return 1 + return 1.0 elif self.availability == "none": - return 0 + return 0.0 return self._probabilities_of_items_available.loc[ (facility_id, list(item_codes)) ].prod() @@ -225,10 +211,15 @@ def is_all_items_available( Returns True only if all items are available at the facility_id, otherwise returns False. """ - return self.rng.random_sample() < self.probability_all_equipment_available( - facility_id=facility_id, - *item_codes - ) + if item_codes: + return self.rng.random_sample() < self.probability_all_equipment_available( + facility_id=facility_id, + item_codes=item_codes, + ) + else: + # In the case of an empty set, default to True without doing anything else ('no equipment' is always + # "available"). This is the most common case, so optimising for speed. + return True def record_use_of_equipment( self, item_codes: Set[int], facility_id: int diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index 64d2ee5930..c49cf8570a 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -306,7 +306,7 @@ def is_all_declared_equipment_available(self) -> bool: ) return self._is_all_declared_equipment_available - def probability_equipment_available(self, item_codes: Union[int, str, Iterable[int | str]]) -> float: + def probability_all_equipment_available(self, item_codes: Union[int, str, Iterable[int | str]]) -> float: """Returns the probability that all the equipment item_codes are available. This does not imply that the equipment is being used and no logging happens. It is provided as a convenience to disease module authors in case the logic during an `HSI_Event` depends on the availability of a piece of equipment. This function diff --git a/tests/test_equipment.py b/tests/test_equipment.py index 7a679fa611..0a3a713dbb 100644 --- a/tests/test_equipment.py +++ b/tests/test_equipment.py @@ -99,7 +99,7 @@ def test_core_functionality_of_equipment_class(seed): # - calling a recognised item for which no data at a facility with no data (should not error) eq_default.is_all_items_available(item_codes={3}, facility_id=2) # -- calling for an unrecognised facility_id (should error) - with pytest.raises(ValueError): + with pytest.raises(AssertionError): eq_default.is_all_items_available(item_codes={1}, facility_id=1001) # -- when using `none` availability behaviour: everything should not be available! @@ -360,7 +360,7 @@ def apply(self, person_id, squeeze_factor): # Check availability of a piece of equipment, with item_code = 0 self.store_of_equipment_checks.update( { - self.sim.date: self.probability_equipment_available(item_codes={0}) + self.sim.date: self.probability_all_equipment_available(item_codes={0}) } ) From 092a417dc51f764e44e5e2538fabe12f42d5ac2d Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Mon, 13 May 2024 20:48:48 +0100 Subject: [PATCH 095/118] Update src/tlo/methods/hsi_event.py Co-authored-by: Matt Graham --- src/tlo/methods/hsi_event.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index c49cf8570a..a7c916413e 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -424,7 +424,7 @@ def as_namedtuple( beddays_footprint=tuple( sorted((k, v) for k, v in self.BEDDAYS_FOOTPRINT.items() if v > 0) ), - equipment=(tuple(sorted(self._EQUIPMENT))), + equipment=tuple(sorted(self._EQUIPMENT)), ) From af05d26861a7cf2d710d35abf44475d57bf95b13 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Mon, 13 May 2024 20:49:47 +0100 Subject: [PATCH 096/118] Update src/tlo/methods/hsi_event.py Co-authored-by: Matt Graham --- src/tlo/methods/hsi_event.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index a7c916413e..dfc705496d 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -41,7 +41,7 @@ class HSIEventDetails(NamedTuple): facility_level: Optional[str] appt_footprint: Tuple[Tuple[str, int]] beddays_footprint: Tuple[Tuple[str, int]] - equipment: set + equipment: Tuple[str] class HSIEventQueueItem(NamedTuple): From 8be77826f4192184d9b1dff9843465d1aa8b9e5c Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Mon, 13 May 2024 20:50:17 +0100 Subject: [PATCH 097/118] Update src/tlo/methods/equipment.py Co-authored-by: Matt Graham --- src/tlo/methods/equipment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tlo/methods/equipment.py b/src/tlo/methods/equipment.py index 4d8eab86a8..ee50a09daa 100644 --- a/src/tlo/methods/equipment.py +++ b/src/tlo/methods/equipment.py @@ -145,7 +145,7 @@ def _calculate_equipment_availability_probabilities(self) -> pd.Series: return dat - def parse_items(self, items: Union[int, str, Iterable[int | str]]) -> Set[int]: + def parse_items(self, items: Union[int, str, Iterable[Union[int, str]]]) -> Set[int]: """Parse equipment items specified as an item_code (integer), an item descriptor (string), or an iterable of either, and return as a set of item_code (integers). For any item_code/descriptor not recognised, a `UserWarning` is issued.""" From e0d2d0054f6bea1baf47404bc84a043405a2392c Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Mon, 13 May 2024 20:52:31 +0100 Subject: [PATCH 098/118] Update src/tlo/methods/equipment.py Co-authored-by: Matt Graham --- src/tlo/methods/equipment.py | 75 +++++++++++++++++------------------- 1 file changed, 35 insertions(+), 40 deletions(-) diff --git a/src/tlo/methods/equipment.py b/src/tlo/methods/equipment.py index ee50a09daa..c21b8d771a 100644 --- a/src/tlo/methods/equipment.py +++ b/src/tlo/methods/equipment.py @@ -11,58 +11,53 @@ class Equipment: - """This is the Equipment Class. It maintains a current record of the availability of equipment in the - HealthSystem. It is expected that this is instantiated by the `HealthSystem` module. + """This is the equipment class. It maintains a current record of the availability of equipment in the + health system. It is expected that this is instantiated by the :py:class:`~.HealthSystem` module. - The basic paradigm is that an `HSI_Event` can declare equipment that is required for delivering the healthcare - service that the `HSI_Event` represents. The `HSI_Event` uses `self.add_equipment()` to make these declaration, - with reference to the items of equipment that are defined in `ResourceFile_EquipmentCatalogue.csv`. (These + The basic paradigm is that an :py:class:`~.HSI_Event` can declare equipment that is required for delivering the healthcare + service that the ``HSI_Event`` represents. The ``HSI_Event`` uses :py:meth:`HSI_event.add_equipment` to make these declarations, + with reference to the items of equipment that are defined in ``ResourceFile_EquipmentCatalogue.csv``. (These declaration can be in the form of the descriptor or the equipment item code). These declarations can be used when - the `HSI_Event` is created but before it is run (in `__init__`), or during execution of the HSI_Event (in `apply`). + the ``HSI_Event`` is created but before it is run (in ``__init__``), or during execution of the ``HSI_Event`` (in :py:meth:`.HSI_Event.apply`). - As the HSI_Event can declare equipment that is required before it is run, the HealthSystem _can_ use this to - prevent an HSI_Event running if the equipment declared is not available. Note that for equipment that is declared - whilst the HSI_Event is running, there are no checks on availability, and the HSI_Event is allowed to continue - running even if equipment is declared is not available. For this reason, the `HSI_Event` should declare equipment - that is _essential_ for the healthcare service in its `__init__` method. If the logic inside the `apply` method - of the `HSI_Event` depends on the availability of equipment, then it can find the probability with which - item(s) will be available using `self.probability_equipment_available()`. + As the ``HSI_Event`` can declare equipment that is required before it is run, the HealthSystem *can* use this to + prevent an ``HSI_Event`` running if the equipment declared is not available. Note that for equipment that is declared + whilst the ``HSI_Event`` is running, there are no checks on availability, and the ``HSI_Event`` is allowed to continue + running even if equipment is declared is not available. For this reason, the ``HSI_Event`` should declare equipment + that is *essential* for the healthcare service in its ``__init__`` method. If the logic inside the ``apply`` method + of the ``HSI_Event`` depends on the availability of equipment, then it can find the probability with which + item(s) will be available using :py:meth:`.HSI_Event.probability_equipment_available`. The data on the availability of equipment data refers to the proportion of facilities in a district of a - particular level (i.e., the `Facility_ID`) that do have that piece of equipment. In the model, we do not know + particular level (i.e., the ``Facility_ID``) that do have that piece of equipment. In the model, we do not know which actual facility the person is attending (there are many actual facilities grouped together into one - `Facility_ID` in the model). Therefore, the determination of whether equipment is available is made - probabilistically for the `HSI_Event` (i.e., the probability that the actual facility being attended by the + ``Facility_ID`` in the model). Therefore, the determination of whether equipment is available is made + probabilistically for the ``HSI_Event`` (i.e., the probability that the actual facility being attended by the person has the equipment is represented by the proportion of such facilities that do have that equipment). It is assumed that the probabilities of each item being available are independent of one other (so that the probability of all items being available is the product of the probabilities for each item). This probabilistic - determination of availability is only done _once_ for the `HSI_Event`: i.e., if the equipment is determined to - not be available for the instance of the `HSI_Event`, then it will remain not available if the same event is - re-scheduled / re-entered into the HealthSystem queue. This represents that if the facility that a particular - person attends for the `HSI_Event` does not have the equipment available, then it will still not be available on + determination of availability is only done _once_ for the ``HSI_Event``: i.e., if the equipment is determined to + not be available for the instance of the ``HSI_Event``, then it will remain not available if the same event is + re-scheduled / re-entered into the ``HealthSystem`` queue. This represents that if the facility that a particular + person attends for the ``HSI_Event`` does not have the equipment available, then it will still not be available on another day. Where data on availability is not provided for an item, the probability of availability is inferred from the - average availability of other items in that `Facility_ID`. Likewise, the probability of an item being available - at `Facility_ID` is inferred from the average availability of that item at other facilities. If an item_code is - referred in `add_equipment() that is not recognised (not included in `catalogue`), a `UserWarning` is issued, but - that item is then silently ignored. If a facility_id is ever referred that is not recognised (not included in - `master_facilities_list`), an `AssertionError` is raised. - - :param: 'catalogue': The database of all recognised item_codes. - - :param: `data_availability`: Specifies the probability with which each equipment (identified by an `item_code`) is - available at a facility level. Note that information is not necessarily provided for every item in the `catalogue` - or every facility_id in the `master_facilities_list`. - - :param: `rng`: The Random Number Generator object to use for random numbers. - - :param: `availability`: Determines the mode availability of the equipment. If 'default' then use the availability - specified in the `data_availability`; if 'none', then let no equipment be ever be available; if 'all', then all - equipment is always available. - - :param `: `master_facilities_list`: The pd.DataFrame with the line-list of all the facilities in the HealthSystem. - + average availability of other items in that facility ID. Likewise, the probability of an item being available + at a facility ID is inferred from the average availability of that item at other facilities. If an item code is + referred in ``add_equipment`` that is not recognised (not included in :py:attr:`catalogue`), a :py:exc:`UserWarning` is issued, but + that item is then silently ignored. If a facility ID is ever referred that is not recognised (not included in + :py:attr:`master_facilities_list`), an :py:exc:`AssertionError` is raised. + + :param catalogue: The database of all recognised item_codes. + :param data_availability: Specifies the probability with which each equipment (identified by an ``item_code``) is + available at a facility level. Note that information is not necessarily provided for every item in the :py:attr`catalogue` + or every facility ID in the :py:attr`master_facilities_list`. + :param: rng: The random number generator object to use for random numbers. + :param availability: Determines the mode availability of the equipment. If 'default' then use the availability + specified in :py:attr:`data_availability`; if 'none', then let no equipment be ever be available; if 'all', then all + equipment is always available. + :param master_facilities_list: The :py:class:`~pandas.DataFrame` with the line-list of all the facilities in the health system. """ def __init__( From 1a6fdc92f6ebbd337736088d21e666c81c917d13 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Mon, 13 May 2024 20:52:45 +0100 Subject: [PATCH 099/118] Update src/tlo/methods/equipment.py Co-authored-by: Matt Graham --- src/tlo/methods/equipment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tlo/methods/equipment.py b/src/tlo/methods/equipment.py index c21b8d771a..d21f375c95 100644 --- a/src/tlo/methods/equipment.py +++ b/src/tlo/methods/equipment.py @@ -64,7 +64,7 @@ def __init__( self, catalogue: pd.DataFrame, data_availability: pd.DataFrame, - rng: np.random, + rng: np.random.RandomState, master_facilities_list: pd.DataFrame, availability: Optional[Literal["all", "default", "none"]] = "default", ) -> None: From 2579b869ec651c034ba1d33b961c00a7d013f655 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Mon, 13 May 2024 20:54:42 +0100 Subject: [PATCH 100/118] Update src/tlo/methods/equipment.py --- src/tlo/methods/equipment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tlo/methods/equipment.py b/src/tlo/methods/equipment.py index d21f375c95..5e9683902b 100644 --- a/src/tlo/methods/equipment.py +++ b/src/tlo/methods/equipment.py @@ -66,7 +66,7 @@ def __init__( data_availability: pd.DataFrame, rng: np.random.RandomState, master_facilities_list: pd.DataFrame, - availability: Optional[Literal["all", "default", "none"]] = "default", + availability: Literal["all", "default", "none"] = "default", ) -> None: # - Store arguments self.catalogue = catalogue From f2ea9cf2d1c37591480720cacc6433a597c863ad Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Mon, 13 May 2024 20:57:14 +0100 Subject: [PATCH 101/118] Update src/tlo/methods/equipment.py Co-authored-by: Matt Graham --- src/tlo/methods/equipment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tlo/methods/equipment.py b/src/tlo/methods/equipment.py index 5e9683902b..d191c79e6a 100644 --- a/src/tlo/methods/equipment.py +++ b/src/tlo/methods/equipment.py @@ -168,7 +168,7 @@ def check_item_descriptors_recognised(item_descriptors: set[str]): else: check_item_descriptors_recognised(items) # Warn for any unrecognised descriptors # In the return, any unrecognised descriptors are silently ignored. - return set(filter(lambda item: item is not None, map(self._item_code_lookup.get, items))) + return set(self._item_code_lookup[i] for i in items if i in self._item_code_lookup) def probability_all_equipment_available( self, facility_id: int, item_codes: Set[int] From cf1d8a504f9005fd30f1a4c85b8732eb29e4cd9a Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Mon, 13 May 2024 20:57:34 +0100 Subject: [PATCH 102/118] Update src/tlo/methods/equipment.py Co-authored-by: Matt Graham --- src/tlo/methods/equipment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tlo/methods/equipment.py b/src/tlo/methods/equipment.py index d191c79e6a..b9842da0ae 100644 --- a/src/tlo/methods/equipment.py +++ b/src/tlo/methods/equipment.py @@ -178,7 +178,7 @@ def probability_all_equipment_available( at the given facility. It does so by looking at the probabilities of each equipment item being - available and multiplying these together to find the probability that _all_ + available and multiplying these together to find the probability that *all* are available. NOTE: This will error if the facility ID or any of the item codes is not recognised. From 8e0a353758d864c710b7a11c14e26c12d838a050 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Mon, 13 May 2024 21:09:58 +0100 Subject: [PATCH 103/118] use property syntax and a setter for `availability` rather than `update_availability` --- src/tlo/methods/equipment.py | 9 +++++++++ src/tlo/methods/healthsystem.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/tlo/methods/equipment.py b/src/tlo/methods/equipment.py index b9842da0ae..eb5dc0e28a 100644 --- a/src/tlo/methods/equipment.py +++ b/src/tlo/methods/equipment.py @@ -95,6 +95,15 @@ def on_simulation_end(self): """ self.write_to_log() + @property + def availability(self): + return self._availability + + @availability.setter + def availability(self, value: Literal["all", "default", "none"]): + assert value in {"all", "none", "default"}, f"New availability value {value} not recognised." + self._availability = value + def update_availability( self, availability: Literal["all", "default", "none"] ) -> None: diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index f4c22f5ead..ed1adb4526 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -2778,7 +2778,7 @@ def apply(self, population): self.module.bed_days.availability = self._parameters['beds_availability'] if 'equip_availability' in self._parameters: - self.module.equipment.update_availability(self._parameters['equip_availability']) + self.module.equipment.availability = self._parameters['equip_availability'] class DynamicRescalingHRCapabilities(RegularEvent, PopulationScopeEventMixin): From 6176413e3019d82493bfc1da047c8e6bb74d6920 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Mon, 13 May 2024 21:12:17 +0100 Subject: [PATCH 104/118] update typehint and docstring on `parse_items` to be more accurate --- src/tlo/methods/equipment.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tlo/methods/equipment.py b/src/tlo/methods/equipment.py index eb5dc0e28a..27536d00a2 100644 --- a/src/tlo/methods/equipment.py +++ b/src/tlo/methods/equipment.py @@ -149,10 +149,10 @@ def _calculate_equipment_availability_probabilities(self) -> pd.Series: return dat - def parse_items(self, items: Union[int, str, Iterable[Union[int, str]]]) -> Set[int]: + def parse_items(self, items: Union[int, str, Iterable[int], Iterable[str]]) -> Set[int]: """Parse equipment items specified as an item_code (integer), an item descriptor (string), or an iterable of - either, and return as a set of item_code (integers). For any item_code/descriptor not recognised, a - `UserWarning` is issued.""" + item_codes or descriptors (but not a mix of the two), and return as a set of item_code (integers). For any + item_code/descriptor not recognised, a ``UserWarning`` is issued.""" def check_item_codes_recognised(item_codes: set[int]): if not item_codes.issubset(self._all_item_codes): From fa214a76fc1ac5850b273c8538ee839531c43bfe Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Mon, 13 May 2024 21:19:57 +0100 Subject: [PATCH 105/118] fix typo in `test_HealthSystemChangeParameters` --- tests/test_healthsystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_healthsystem.py b/tests/test_healthsystem.py index 5dece9b2d9..91adb7bea1 100644 --- a/tests/test_healthsystem.py +++ b/tests/test_healthsystem.py @@ -1370,7 +1370,7 @@ def apply(self, population): _params['capabilities_coefficient'] = hs.capabilities_coefficient _params['cons_availability'] = hs.consumables.cons_availability _params['beds_availability'] = hs.bed_days.availability - _params['equip_availability'] = hs.equip_availability + _params['equip_availability'] = hs.equipment.availability logger = logging.getLogger('tlo.methods.healthsystem') logger.info(key='CheckHealthSystemParameters', data=_params) From 3be83633970bd06ee609015c15c901af2b47543e Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Tue, 14 May 2024 08:11:54 +0100 Subject: [PATCH 106/118] linting --- src/tlo/methods/equipment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tlo/methods/equipment.py b/src/tlo/methods/equipment.py index 27536d00a2..e3f3d0b1ae 100644 --- a/src/tlo/methods/equipment.py +++ b/src/tlo/methods/equipment.py @@ -1,6 +1,6 @@ import warnings from collections import defaultdict -from typing import Counter, Iterable, Literal, Optional, Set, Union +from typing import Counter, Iterable, Literal, Set, Union import numpy as np import pandas as pd From 959c243afa878af3d1f501cc22b7aaa9e8eb2e87 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Tue, 14 May 2024 10:10:19 +0100 Subject: [PATCH 107/118] remove update_availability --- src/tlo/methods/equipment.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/tlo/methods/equipment.py b/src/tlo/methods/equipment.py index e3f3d0b1ae..4de29107f9 100644 --- a/src/tlo/methods/equipment.py +++ b/src/tlo/methods/equipment.py @@ -104,18 +104,6 @@ def availability(self, value: Literal["all", "default", "none"]): assert value in {"all", "none", "default"}, f"New availability value {value} not recognised." self._availability = value - def update_availability( - self, availability: Literal["all", "default", "none"] - ) -> None: - """ - Update the availability of equipment. - - This is expected to be called midway through the simulation if the - assumption of the equipment availability is changed. - """ - assert availability in ["all", "none", "default"], f"New availability parameter {availability} not recognised." - self.availability = availability - def _calculate_equipment_availability_probabilities(self) -> pd.Series: """ Compute the probabilities that each equipment item is available (at a given From cc56948598fa5ecc69da11e0b810b1cc3a6e997d Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Tue, 14 May 2024 19:09:52 +0100 Subject: [PATCH 108/118] use continue rather than nested if/else statment --- src/tlo/methods/healthsystem.py | 101 ++++++++++++++++---------------- 1 file changed, 51 insertions(+), 50 deletions(-) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index ed1adb4526..1e71c04f46 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -2365,65 +2365,66 @@ def process_events_mode_2(self, hold_over: List[HSIEventQueueItem]) -> None: f"Cannot run HSI {event.TREATMENT_ID} without facility_info being defined." # Check if equipment declared is available. If not, call `never_ran` and do not run the - # event. + # event. (`continue` returns flow to beginning of the `while` loop) if not event.is_all_declared_equipment_available: self.module.call_and_record_never_ran_hsi_event( hsi_event=event, priority=next_event_tuple.priority ) - else: - # Expected appt footprint before running event - _appt_footprint_before_running = event.EXPECTED_APPT_FOOTPRINT - # Run event & get actual footprint - actual_appt_footprint = event.run(squeeze_factor=squeeze_factor) + continue - # Check if the HSI event returned updated_appt_footprint, and if so adjust original_call - if actual_appt_footprint is not None: + # Expected appt footprint before running event + _appt_footprint_before_running = event.EXPECTED_APPT_FOOTPRINT + # Run event & get actual footprint + actual_appt_footprint = event.run(squeeze_factor=squeeze_factor) - # check its formatting: - assert self.module.appt_footprint_is_valid(actual_appt_footprint) + # Check if the HSI event returned updated_appt_footprint, and if so adjust original_call + if actual_appt_footprint is not None: - # Update call that will be used to compute capabilities used - updated_call = self.module.get_appt_footprint_as_time_request( - facility_info=event.facility_info, - appt_footprint=actual_appt_footprint - ) - else: - actual_appt_footprint = _appt_footprint_before_running - updated_call = original_call - - # Recalculate call on officers based on squeeze factor. - for k in updated_call.keys(): - updated_call[k] = updated_call[k]/(squeeze_factor + 1.) - - # Subtract this from capabilities used so-far today - capabilities_monitor.subtract(updated_call) - - # If any of the officers have run out of time by performing this hsi, - # remove them from list of available officers. - for officer, call in updated_call.items(): - if capabilities_monitor[officer] <= 0: - if officer in set_capabilities_still_available: - set_capabilities_still_available.remove(officer) - else: - logger.warning( - key="message", - data=(f"{event.TREATMENT_ID} actual_footprint requires different" - f"officers than expected_footprint.") - ) - - # Update today's footprint based on actual call and squeeze factor - self.module.running_total_footprint -= original_call - self.module.running_total_footprint += updated_call - - # Write to the log - self.module.record_hsi_event( - hsi_event=event, - actual_appt_footprint=actual_appt_footprint, - squeeze_factor=squeeze_factor, - did_run=True, - priority=_priority + # check its formatting: + assert self.module.appt_footprint_is_valid(actual_appt_footprint) + + # Update call that will be used to compute capabilities used + updated_call = self.module.get_appt_footprint_as_time_request( + facility_info=event.facility_info, + appt_footprint=actual_appt_footprint ) + else: + actual_appt_footprint = _appt_footprint_before_running + updated_call = original_call + + # Recalculate call on officers based on squeeze factor. + for k in updated_call.keys(): + updated_call[k] = updated_call[k]/(squeeze_factor + 1.) + + # Subtract this from capabilities used so-far today + capabilities_monitor.subtract(updated_call) + + # If any of the officers have run out of time by performing this hsi, + # remove them from list of available officers. + for officer, call in updated_call.items(): + if capabilities_monitor[officer] <= 0: + if officer in set_capabilities_still_available: + set_capabilities_still_available.remove(officer) + else: + logger.warning( + key="message", + data=(f"{event.TREATMENT_ID} actual_footprint requires different" + f"officers than expected_footprint.") + ) + + # Update today's footprint based on actual call and squeeze factor + self.module.running_total_footprint -= original_call + self.module.running_total_footprint += updated_call + + # Write to the log + self.module.record_hsi_event( + hsi_event=event, + actual_appt_footprint=actual_appt_footprint, + squeeze_factor=squeeze_factor, + did_run=True, + priority=_priority + ) # Don't have any capabilities at all left for today, no # point in going through the queue to check what's left to do today. From 69e28ca7fa6361b0508ec3f26a1fdf72e3fabc7f Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Tue, 14 May 2024 19:13:01 +0100 Subject: [PATCH 109/118] initialise self._EQUIPMENT in the __init__ so that it specific to the instance and not the class --- src/tlo/methods/hsi_event.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index dfc705496d..b50c4de6aa 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -86,12 +86,6 @@ class HSI_Event: # which have been loaded. BEDDAYS_FOOTPRINT: Dict[str, Union[float, int]] - _EQUIPMENT: Set[int] = set() # The set of equipment that is used in the HSI. If any items in this set are not - # available at the point when the HSI will be run, then the HSI is not run, and the - # `never_ran` method is called instead. This is a declaration of resource needs, but - # is private because users are expected to use `add_equipment` to declare equipment - # needs. - _received_info_about_bed_days: Dict[str, Union[float, int]] = None expected_time_requests: Counter = {} facility_info: FacilityInfo = None @@ -119,6 +113,11 @@ def __init__(self, module, *args, **kwargs): self.ACCEPTED_FACILITY_LEVEL = None # Set "dynamic" default value self.BEDDAYS_FOOTPRINT = self.make_beddays_footprint({}) + self._EQUIPMENT: Set[int] = set() # The set of equipment that is used in the HSI. If any items in this set are + # not available at the point when the HSI will be run, then the HSI is not + # run, and the `never_ran` method is called instead. This is a declaration + # of resource needs, but is private because users are expected to use + # `add_equipment` to declare equipment needs. @property def bed_days_allocated_to_this_event(self): From d340e0918685ad8ac4293b4cf9868409766bea89 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Tue, 14 May 2024 19:13:18 +0100 Subject: [PATCH 110/118] Update src/tlo/methods/hsi_event.py Co-authored-by: Matt Graham --- src/tlo/methods/hsi_event.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index dfc705496d..1b831be7d1 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -281,7 +281,7 @@ def make_appt_footprint(self, dict_of_appts) -> Counter: "values" ) - def add_equipment(self, item_codes: Union[int, str, Iterable[int | str]]) -> None: + def add_equipment(self, item_codes: Union[int, str, Iterable[int], Iterable[str]]) -> None: """Declare that piece(s) of equipment are used in this HSI_Event. Equipment items can be identified by their item_codes (int) or descriptors (str); a singular item or an iterable of items can be defined at once. Checks are done on the validity of the item_codes/item descriptions and a warning issued if any are not From 45b3b41c71ad4816e2772907c748fe51cbaf6f86 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Tue, 14 May 2024 19:13:46 +0100 Subject: [PATCH 111/118] Update src/tlo/methods/hsi_event.py Co-authored-by: Matt Graham --- src/tlo/methods/hsi_event.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index 1b831be7d1..664cf509b8 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -306,7 +306,7 @@ def is_all_declared_equipment_available(self) -> bool: ) return self._is_all_declared_equipment_available - def probability_all_equipment_available(self, item_codes: Union[int, str, Iterable[int | str]]) -> float: + def probability_all_equipment_available(self, item_codes: Union[int, str, Iterable[int], Iterable[str]]) -> float: """Returns the probability that all the equipment item_codes are available. This does not imply that the equipment is being used and no logging happens. It is provided as a convenience to disease module authors in case the logic during an `HSI_Event` depends on the availability of a piece of equipment. This function From da5222ea76f9b25de9bf80d90f08d17dbd749ed6 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Tue, 14 May 2024 19:14:06 +0100 Subject: [PATCH 112/118] Update src/tlo/methods/hsi_event.py Co-authored-by: Matt Graham --- src/tlo/methods/hsi_event.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index 664cf509b8..755644fcdf 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -309,7 +309,7 @@ def is_all_declared_equipment_available(self) -> bool: def probability_all_equipment_available(self, item_codes: Union[int, str, Iterable[int], Iterable[str]]) -> float: """Returns the probability that all the equipment item_codes are available. This does not imply that the equipment is being used and no logging happens. It is provided as a convenience to disease module authors in - case the logic during an `HSI_Event` depends on the availability of a piece of equipment. This function + case the logic during an ``HSI_Event`` depends on the availability of a piece of equipment. This function accepts the item codes/descriptions in a variety of formats, so the argument needs to be parsed.""" return self.healthcare_system.equipment.probability_all_equipment_available( item_codes=self.healthcare_system.equipment.parse_items(item_codes), From e7b40b0f23bc3e22ee342e5243f89ca59f10c472 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Tue, 14 May 2024 19:14:28 +0100 Subject: [PATCH 113/118] Update src/tlo/methods/hsi_event.py Co-authored-by: Matt Graham --- src/tlo/methods/hsi_event.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index 755644fcdf..b124de5dd3 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -290,12 +290,12 @@ def add_equipment(self, item_codes: Union[int, str, Iterable[int], Iterable[str] @property def is_all_declared_equipment_available(self) -> bool: - """Returns `True` if all the (currently) declared items of equipment are available. This is called by the - `HealthSystem` module before the HSI is run and so is looking only at those items that are declared when this - instance was created. The evaluation of whether equipment is available is only done _once_ for this instance of - the event: i.e., if the equipment is not available for the instance of this `HSI_Event`, then it will remain not + """Returns ``True`` if all the (currently) declared items of equipment are available. This is called by the + ``HealthSystem`` module before the HSI is run and so is looking only at those items that are declared when this + instance was created. The evaluation of whether equipment is available is only done *once* for this instance of + the event: i.e., if the equipment is not available for the instance of this ``HSI_Event``, then it will remain not available if the same event is re-scheduled/re-entered into the HealthSystem queue. This is representing that - if the facility that a particular person attends for the HSI_Event does not have the equipment available, then + if the facility that a particular person attends for the ``HSI_Event`` does not have the equipment available, then it will also not be available on another day.""" if self._is_all_declared_equipment_available is None: From 204877e94c6a3d8669799d8db60522b9ce52d39e Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Tue, 14 May 2024 19:15:14 +0100 Subject: [PATCH 114/118] Update src/tlo/methods/hsi_event.py Co-authored-by: Matt Graham --- src/tlo/methods/hsi_event.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index b124de5dd3..b05ec7b5a1 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -283,9 +283,9 @@ def make_appt_footprint(self, dict_of_appts) -> Counter: def add_equipment(self, item_codes: Union[int, str, Iterable[int], Iterable[str]]) -> None: """Declare that piece(s) of equipment are used in this HSI_Event. Equipment items can be identified by their - item_codes (int) or descriptors (str); a singular item or an iterable of items can be defined at once. Checks - are done on the validity of the item_codes/item descriptions and a warning issued if any are not - recognised.""" + item_codes (int) or descriptors (str); a singular item or an iterable of items (either codes or descriptors but + not a mix of both) can be defined at once. Checks are done on the validity of the item_codes/item + descriptions and a warning issued if any are not recognised.""" self._EQUIPMENT.update(self.healthcare_system.equipment.parse_items(item_codes)) @property From 55f9c8860aa81598943be1d7e7f7705608672c01 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Tue, 14 May 2024 19:21:12 +0100 Subject: [PATCH 115/118] make checks in `test_core_functionality_of_equipment_class` test for exactly the right resonse, rather than truthiness only. --- tests/test_equipment.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_equipment.py b/tests/test_equipment.py index 0a3a713dbb..c352ce2941 100644 --- a/tests/test_equipment.py +++ b/tests/test_equipment.py @@ -66,9 +66,9 @@ def test_core_functionality_of_equipment_class(seed): # - using list of integers for item_codes assert {1, 2} == eq_default.parse_items([1, 2]) # - using single string for one item descriptor - assert eq_default.parse_items('ItemOne') + assert {1} == eq_default.parse_items('ItemOne') # - using list of strings for item descriptors - assert eq_default.parse_items(['ItemOne', 'ItemTwo']) + assert {1, 2} == eq_default.parse_items(['ItemOne', 'ItemTwo']) # - an empty iterable of equipment should always be work whether expressed as list/tuple/set assert set() == eq_default.parse_items(list()) assert set() == eq_default.parse_items(tuple()) @@ -142,7 +142,7 @@ def test_core_functionality_of_equipment_class(seed): eq_default.record_use_of_equipment(item_codes={0, 1}, facility_id=0) eq_default.record_use_of_equipment(item_codes={0, 1}, facility_id=1) # - Check that internal record is as expected - assert dict(eq_default._record_of_equipment_used_by_facility_id) == {0: {0: 1, 1: 2}, 1: {0: 1, 1: 1}} + assert {0: {0: 1, 1: 2}, 1: {0: 1, 1: 1}} == dict(eq_default._record_of_equipment_used_by_facility_id) # Lookup the item_codes that belong in a particular package. # - When package is recognised From 75e02a96c286ed2644adc37bd9e531b9b51b7633 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Tue, 14 May 2024 20:14:11 +0100 Subject: [PATCH 116/118] check logging from multiple HSI_events in test_logging_of_equipment_from_multiple_hsi --- tests/test_equipment.py | 64 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/tests/test_equipment.py b/tests/test_equipment.py index c352ce2941..3e4daecc44 100644 --- a/tests/test_equipment.py +++ b/tests/test_equipment.py @@ -412,3 +412,67 @@ def initialise_simulation(self, sim): log = pd.Series(sim.modules['DummyModule'].the_hsi_event.store_of_equipment_checks) assert (1.0 == log[log.index < Date(2011, 1, 1)]).all() assert (0.0 == log[log.index >= Date(2011, 1, 1)]).all() + + +def test_logging_of_equipment_from_multiple_hsi(seed, tmpdir): + """Test that we correctly capture in the log the equipment declared by different HSI_Events that run at different + levels.""" + + item_code_needed_at_each_level = { + '0': set({0}), '1a': set({10}), '2': set({30}), '3': set({44}), '4': set() + } + + class DummyHSIEvent(HSI_Event, IndividualScopeEventMixin): + def __init__( + self, + module, + person_id, + level, + equipment_item_code + ): + super().__init__(module, person_id=person_id) + self.TREATMENT_ID = f"DummyHSIEvent_Level:{level}" + self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint({}) + self.ACCEPTED_FACILITY_LEVEL = level + self.add_equipment(equipment_item_code) + + def apply(self, person_id, squeeze_factor): + pass + + class DummyModule(Module): + METADATA = {Metadata.DISEASE_MODULE, Metadata.USES_HEALTHSYSTEM} + + def read_parameters(self, data_folder): + pass + + def initialise_population(self, population): + pass + + def initialise_simulation(self, sim): + # Schedule the HSI_Events to occur, with the level determining the item_code used + for level, item_code in item_code_needed_at_each_level.items(): + sim.modules["HealthSystem"].schedule_hsi_event( + hsi_event=DummyHSIEvent(person_id=0, module=self, level=level, equipment_item_code=item_code), + do_hsi_event_checks=False, + topen=sim.date, + tclose=None, + priority=0, + ) + + log_config = {"filename": "log", "directory": tmpdir} + sim = Simulation(start_date=Date(2010, 1, 1), seed=seed, log_config=log_config) + sim.register( + demography.Demography(resourcefilepath=resourcefilepath), + healthsystem.HealthSystem(resourcefilepath=resourcefilepath), + DummyModule(), + ) + sim.make_initial_population(n=100) + sim.simulate(end_date=sim.start_date + pd.DateOffset(days=1)) + + # Read log to find what equipment used + df = parse_log_file(sim.log_filepath)["tlo.methods.healthsystem.summary"]['EquipmentEverUsed_ByFacilityID'] + df = df.drop(index=df.index[~df['Facility_Level'].isin(item_code_needed_at_each_level.keys())]) + df['EquipmentEverUsed'] = df['EquipmentEverUsed'].apply(eval).apply(list) + + # Check that equipment used at each level matches expectations + assert item_code_needed_at_each_level == df.groupby('Facility_Level')['EquipmentEverUsed'].sum().apply(set).to_dict() From 81b8c58dd195c28883f361a0949cb7f33d8ac965 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Tue, 14 May 2024 20:39:39 +0100 Subject: [PATCH 117/118] roll back incidental changes --- src/tlo/methods/contraception.py | 3 +-- src/tlo/methods/symptommanager.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/tlo/methods/contraception.py b/src/tlo/methods/contraception.py index 83fd86cab9..67d6684fce 100644 --- a/src/tlo/methods/contraception.py +++ b/src/tlo/methods/contraception.py @@ -164,8 +164,7 @@ def read_parameters(self, data_folder): """Import the relevant sheets from the ResourceFile (excel workbook) and declare values for other parameters (CSV ResourceFile). """ - workbook = pd.read_excel(Path(self.resourcefilepath) / 'contraception' / 'ResourceFile_Contraception.xlsx', - sheet_name=None) + workbook = pd.read_excel(Path(self.resourcefilepath) / 'contraception' / 'ResourceFile_Contraception.xlsx', sheet_name=None) # Import selected sheets from the workbook as the parameters sheet_names = [ diff --git a/src/tlo/methods/symptommanager.py b/src/tlo/methods/symptommanager.py index 26f6aa7ee4..68edbf0840 100644 --- a/src/tlo/methods/symptommanager.py +++ b/src/tlo/methods/symptommanager.py @@ -272,8 +272,7 @@ def pre_initialise_population(self): SymptomManager.PROPERTIES = dict() for symptom_name in sorted(self.symptom_names): symptom_column_name = self.get_column_name_for_symptom(symptom_name) - SymptomManager.PROPERTIES[symptom_column_name] = Property(Types.BITSET, - f'Presence of symptom {symptom_name}') + SymptomManager.PROPERTIES[symptom_column_name] = Property(Types.BITSET, f'Presence of symptom {symptom_name}') def initialise_population(self, population): """ From e42e66090aeccf78ba38fb533e5e323b654d93b5 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Tue, 14 May 2024 20:40:33 +0100 Subject: [PATCH 118/118] remove `codes_to_items_list.py` from this PR --- .../codes_to_items_list.py | 75 ------------------- 1 file changed, 75 deletions(-) delete mode 100644 src/scripts/data_file_processing/codes_to_items_list.py diff --git a/src/scripts/data_file_processing/codes_to_items_list.py b/src/scripts/data_file_processing/codes_to_items_list.py deleted file mode 100644 index da169a06e5..0000000000 --- a/src/scripts/data_file_processing/codes_to_items_list.py +++ /dev/null @@ -1,75 +0,0 @@ -""" -(1) Can be used for a list of items without item codes yet saved in a csv file named 'csv_file_to_update_name'. - -This script will assign unique code to each unique item name which has no code assigned yet. The codes are -assigned in order from the sequence 0, 1, 2, .... - -Duplicated items are allowed, the same code will be assigned to the same items. - -(2) Can be used when new items are added later without item codes but some items with codes are already in the list. - -This script will keep the existing codes for items with already assigned code and for items without existing -code will assign new code (continue in sequence, i.e. if the highest code is 5, it assigns new codes from the continuing -sequence 6, 7, 8, ...). - ------- -NB. Make sure the 'csv_file_to_update_name' is the file you want to update. The output will be named -'csv_file_to_update_name' + '_new.csv' to avoid unintentionally losing the previous version. ------- -""" - -from pathlib import Path - -import pandas as pd - -# Get the path of the current script file -script_path = Path(__file__) -print(script_path) - -# ############################# -# ## CHANGE THIS FOR YOUR FILE -# Specify name of the csv file -csv_file_to_update_name = 'ResourceFile_Equipment_withoutEquipmentCodes' -# Specify the file path to csv file -file_path = script_path.parent.parent.parent.parent / 'resources/healthsystem/infrastructure_and_equipment' -# Specify the names of columns containing the item names and item codes -item_col_name = 'Equip_Item' -code_col_name = 'Equip_Code' -# ############################# - -# Load the CSV RF into a DataFrame -df = pd.read_csv(Path(file_path) / str(csv_file_to_update_name + '.csv')) - -# Find unique values in Equipment that have no code and are not None or empty -unique_values =\ - df.loc[df[code_col_name].isna() & df[item_col_name].notna() & (df[item_col_name] != ''), item_col_name].unique() - -# Create a mapping of unique values to codes -value_to_code = {} -# Initialize the starting code value -if not df[code_col_name].isna().all(): - next_code = int(df[code_col_name].max()) + 1 -else: - next_code = 0 - -# Iterate through unique values -for value in unique_values: - # Check if there is at least one existing code for this value - matching_rows = df.loc[df[item_col_name] == value, code_col_name].dropna() - if not matching_rows.empty: - # Use the existing code for this value - existing_code = int(matching_rows.iloc[0]) - # TODO: verify all the codes are the same - else: - # If no existing codes, start with the next available code - existing_code = next_code - next_code += 1 - value_to_code[value] = existing_code - # Update the code_col_name column for matching rows - df.loc[df[item_col_name] == value, code_col_name] = existing_code - -# Convert code_col_name column to integers -df[code_col_name] = df[code_col_name].astype('Int64') # Convert to nullable integer type - -# Save CSV with equipment codes -df.to_csv(Path(file_path) / str(csv_file_to_update_name + '_new.csv'), index=False)