Merge branch 'master' into mnjowe/update-copd-write-up

Author: mnjowe

.github/workflows/checks.yml

@@ -27,7 +27,7 @@ jobs:
         with:
           lfs: false
       - name: Cache tox
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: .tox
           key: tox-${{hashFiles('requirements/*.txt', 'tox.ini')}}

.github/workflows/docs.yml

@@ -35,7 +35,7 @@ jobs:
         with:
           lfs: true
       - name: Cache tox
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: .tox
           key: tox-${{hashFiles('requirements/*.txt', 'tox.ini')}}

.github/workflows/run-on-comment.yml

@@ -126,7 +126,7 @@ jobs:
     - name: Get comment-bot token
       if: always() && steps.has_permissions.outputs.result == 'true'
       id: get_comment_bot_token
-      uses: peter-murray/workflow-application-token-action@b1ad34e28f4ef30582df1bd5f45df2044eb7a6b9
+      uses: peter-murray/workflow-application-token-action@dc0413987a085fa17d19df9e47d4677cf81ffef3
       with:
         application_id: ${{ secrets.application-id }}
         application_private_key: ${{ secrets.application-private-key }}

.github/workflows/run-profiling.yaml

@@ -91,6 +91,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           ref: ${{ github.sha }}
+          lfs: true
 
       ## The profile environment produces outputs in the /results directory
       - name: Run profiling in dev environment
@@ -150,7 +151,7 @@ jobs:
           user_name:  rc-softdev-admin
 
       - name: Trigger website rebuild
-        uses: peter-evans/repository-dispatch@v2
+        uses: peter-evans/repository-dispatch@v3
         with:
           token: ${{ secrets.PROFILING_REPO_ACCESS }}
           repository: UCL/TLOmodel-profiling

src/scripts/calibration_analyses/analysis_scripts/analysis_cause_of_death_and_disability_calibrations.py

@@ -11,7 +11,9 @@
 import pandas as pd
 from matplotlib import pyplot as plt
 
+from tlo import Date
 from tlo.analysis.utils import (
+    CAUSE_OF_DEATH_OR_DALY_LABEL_TO_COLOR_MAP,
     extract_results,
     format_gbd,
     get_color_cause_of_death_or_daly_label,
@@ -166,7 +168,7 @@ def _sort_columns(df):
 
                 fig, ax = plt.subplots()
                 plot_clustered_stacked(ax=ax,
-                                       dfall=_dat,
+                                       dfall=({k: v/1e3 for k, v in _dat.items()} if what == 'DALYs' else _dat),
                                        color_for_column_map=get_color_cause_of_death_or_daly_label,
                                        scaled=scaled,
                                        legends=False,
@@ -178,7 +180,6 @@ def _sort_columns(df):
                 ax.set_xlabel('Age Group')
                 ax.set_xticklabels(ax.get_xticklabels(), rotation=90)
 
-                # ax.set_xlim([0, 17.5])
                 if scaled:
                     ax.set_ylim([0, 1.05])
                 else:
@@ -188,23 +189,22 @@ def _sort_columns(df):
                         ax.set_ylim([0, 25_000])
                         ax.set_yticks(np.arange(0, 30_000, 5_000))
                     else:
-                        ax.set_ylabel(f"{what} per year\n")
+                        ax.set_ylabel(f"{what} per year (/1000)\n")
+                        ax.set_ylim([0, 2000.0])
 
                 # Create figure legend and remove duplicated entries, but keep the first entries
                 handles, labels = ax.get_legend_handles_labels()
                 lgd = dict()
                 for k, v in zip(labels, handles):
                     lgd.setdefault(k, v)
-                ax.legend(reversed(lgd.values()), reversed(lgd.keys()), loc="upper right", ncol=2, fontsize=8)
+                # ax.legend(reversed(lgd.values()), reversed(lgd.keys()), loc="upper right", ncol=2, fontsize=8)
+                # ax.text(
+                # 5.2, 11_000, 'GBD || Model', horizontalalignment='left',  verticalalignment='bottom', fontsize=8)
+                ax.legend().set_visible(False)  # Hide legend
 
                 fig.tight_layout()
                 fig.savefig(make_graph_file_name(
                     f"{what}_{period}_{sex}_StackedBars_ModelvsGBD_{'scaled' if scaled else ''}"))
-
-                # ax.text(
-                # 5.2, 11_000, 'GBD || Model', horizontalalignment='left',  verticalalignment='bottom', fontsize=8)
-                ax.legend().set_visible(False)
-
                 plt.close(fig)
 
         # Simple pie-charts of just TLO estimates
@@ -296,9 +296,9 @@ def shift_row_to_top(df, index_to_shift):
                 dat: outcome_by_age_pt[dat].sum(axis=0) for dat in outcome_by_age_pt.keys()
             }, axis=1
         )
-        # todo N.B. For GBD, should really use all ages and all sex numbers from GBD to get correct uncertainty bounds
-        #  (the addition of the bounds for the sub-categories - as done here - is not strictly correct.)
-        #  ... OR use formula to make my own explicit assumption about correlation of uncertainty in different age-grps.
+        # todo N.B. For GBD, would ideally use all ages and all sex numbers from GBD to get correct uncertainty bounds
+        #  (the addition of the bounds for the sub-categories - as done here - might over-state the uncertainty.) This
+        #  plot should be taken as indicative only.
 
         select_labels = []
 
@@ -311,6 +311,11 @@ def shift_row_to_top(df, index_to_shift):
 
         for cause in all_causes:
 
+            if (cause == 'Other') and (what == 'DALYs'):
+                # Skip 'Other' when plotting DALYS as it's misleading. We don't have "Other" (non-modelled) causes
+                # of disability.
+                continue
+
             vals = tot_outcomes_by_cause.loc[(slice(None), cause), ] / 1e3
 
             x = vals.at[('mean', cause), 'GBD']
@@ -367,19 +372,22 @@ def shift_row_to_top(df, index_to_shift):
         ])
 
         outcomes = outcome_by_age_pt['GBD'][("mean")]
-        fraction_causes_modelled = (1.0 - outcomes['Other'] / outcomes.sum(axis=1))
+        fraction_causes_modelled_overall = (1.0 - outcomes['Other'].sum() / outcomes.sum().sum())
+        fraction_causes_modelled_by_sex_and_age = (1.0 - outcomes['Other'] / outcomes.sum(axis=1))
         fig, ax = plt.subplots()
         for sex in sexes:
-            fraction_causes_modelled.loc[(sex, slice(None))].plot(
+            fraction_causes_modelled_by_sex_and_age.loc[(sex, slice(None))].plot(
                 ax=ax,
                 color=get_color_cause_of_death_or_daly_label('Other'),
                 linestyle=':' if sex == 'F' else '-',
                 label=sexname(sex),
                 lw=5,
             )
-        ax.legend()
+        ax.axhline(fraction_causes_modelled_overall, color='b',
+                   label=f'Overall: {round(100 * fraction_causes_modelled_overall)}%')
+        ax.legend(loc='upper right')
         ax.set_ylim(0, 1.0)
-        xticks = fraction_causes_modelled.index.levels[1]
+        xticks = fraction_causes_modelled_by_sex_and_age.index.levels[1]
         ax.set_xticks(range(len(xticks)))
         ax.set_xticklabels(xticks, rotation=90)
         ax.grid(axis='y')
@@ -392,6 +400,62 @@ def shift_row_to_top(df, index_to_shift):
         plt.savefig(make_graph_file_name(f"C_{what}_{period}_coverage"))
         plt.close(fig)
 
+        # Describe the burden with respect to wealth quintile:
+        TARGET_PERIOD = (Date(2015, 1, 1), Date(2019, 12, 31))
+
+        def get_total_num_dalys_by_wealth_and_label(_df):
+            """Return the total number of DALYS in the TARGET_PERIOD by wealth and cause label."""
+            wealth_cats = {5: '0-19%', 4: '20-39%', 3: '40-59%', 2: '60-79%', 1: '80-100%'}
+
+            return _df \
+                .loc[_df['year'].between(*[d.year for d in TARGET_PERIOD])] \
+                .drop(columns=['date', 'year']) \
+                .assign(
+                    li_wealth=lambda x: x['li_wealth'].map(wealth_cats).astype(
+                        pd.CategoricalDtype(wealth_cats.values(), ordered=True)
+                    )
+                ).melt(id_vars=['li_wealth'], var_name='label') \
+                 .groupby(by=['li_wealth', 'label'])['value'] \
+                 .sum()
+
+        total_num_dalys_by_wealth_and_label = summarize(
+            extract_results(
+                results_folder,
+                module="tlo.methods.healthburden",
+                key="dalys_by_wealth_stacked_by_age_and_time",
+                custom_generate_series=get_total_num_dalys_by_wealth_and_label,
+                do_scaling=True,
+            ),
+            collapse_columns=True,
+            only_mean=True,
+        ).unstack()
+
+        format_to_plot = total_num_dalys_by_wealth_and_label \
+            .sort_index(axis=0) \
+            .reindex(columns=CAUSE_OF_DEATH_OR_DALY_LABEL_TO_COLOR_MAP.keys(), fill_value=0.0) \
+            .sort_index(axis=1, key=order_of_cause_of_death_or_daly_label)
+
+        fig, ax = plt.subplots()
+        name_of_plot = 'DALYS by Wealth and Cause, 2015-2019'
+        (
+            format_to_plot / 1e6
+        ).plot.bar(stacked=True, ax=ax,
+                   color=[get_color_cause_of_death_or_daly_label(_label) for _label in format_to_plot.columns],
+                   )
+        ax.axhline(0.0, color='black')
+        ax.set_title(name_of_plot)
+        ax.set_ylabel('Number of DALYs Averted (/1e6)')
+        ax.set_ylim(0, 10)
+        ax.set_xlabel('Wealth Percentile')
+        ax.grid()
+        ax.spines['top'].set_visible(False)
+        ax.spines['right'].set_visible(False)
+        ax.legend(ncol=3, fontsize=8, loc='upper right')
+        ax.legend().set_visible(False)
+        fig.tight_layout()
+        fig.savefig(make_graph_file_name(name_of_plot.replace(' ', '_')))
+        plt.close(fig)
+
     # %% Make graphs for each of Deaths and DALYS for a specific period
     # make_std_graphs(what='Deaths', period='2010-2014')
     # make_std_graphs(what='DALYs', period='2010-2014')

src/scripts/calibration_analyses/analysis_scripts/analysis_compare_appt_usage_real_and_simulation.py

@@ -344,7 +344,7 @@ def apply(results_folder: Path, output_folder: Path, resourcefilepath: Path = No
 
     # Plot Simulation vs Real usage (Across all levels and At each level) (trimmed to 0.1 and 10)
     # format plot
-    def format_and_save(_fig, _ax, _name_of_plot):
+    def format_and_save(_fig, _ax, _name_of_plot, legend=True):
         _ax.set_title(_name_of_plot)
         _ax.set_yscale('log')
         _ax.set_ylim(1 / 20, 20)
@@ -356,6 +356,8 @@ def format_and_save(_fig, _ax, _name_of_plot):
         _ax.xaxis.grid(True, which='major', linestyle='--')
         _ax.yaxis.grid(True, which='both', linestyle='--')
         _ax.legend(loc='center left', bbox_to_anchor=(1, 0.5))
+        if not legend:
+            _ax.legend().set_visible(False)
         _fig.tight_layout()
         _fig.savefig(make_graph_file_name(_name_of_plot.replace(',', '').replace('\n', '_').replace(' ', '_')))
         plt.close(_fig)
@@ -383,7 +385,7 @@ def format_and_save(_fig, _ax, _name_of_plot):
         if not pd.isna(rel_diff_all_levels[idx]):
             ax.text(idx, rel_diff_all_levels[idx]*(1+0.2), round(rel_diff_all_levels[idx], 1),
                     ha='left', fontsize=8)
-    format_and_save(fig, ax, name_of_plot)
+    format_and_save(fig, ax, name_of_plot, legend=False)
 
     # plot for each level
     rel_diff_by_levels = (
@@ -467,7 +469,7 @@ def format_real_usage():
                     round(rel_diff_real.loc[idx, 'mean'], 1),
                     ha='left', fontsize=8)
     ax.axhline(1.0, color='r')
-    format_and_save(fig, ax, name_of_plot)
+    format_and_save(fig, ax, name_of_plot, legend=False)
 
     # Plot Simulation vs Real usage by appt type and show fraction of usage at each level
     # Model, Adjusted real and Unadjusted real average annual usage all normalised to 1

src/scripts/calibration_analyses/analysis_scripts/analysis_demography_calibrations.py

@@ -404,7 +404,7 @@ def get_annual_mean_usage(_df):
     spacing = (np.arange(len(mean_usage_mean)) % 5) == 0
     mean_usage_mean.loc[spacing].plot.bar(stacked=True, ax=ax, legend=False)
     plt.title('Proportion Females 15-49 Using Contraceptive Methods')
-    plt.xlabel('Date')
+    plt.xlabel('Year')
     plt.ylabel('Proportion')
 
     fig.legend(loc=7)