Merge branch 'master' into dp_kidney

docker/vm_boot_images/config/tensorflow-requirements.txt

@@ -43,5 +43,4 @@ google-cloud-storage
 umap-learn[plot]
 neurite
 voxelmorph
-pystrum
-
+pystrum

ml4h/TensorMap.py

@@ -204,8 +204,6 @@ def __init__(
         elif self.activation is None and (self.is_survival_curve() or self.is_time_to_event()):
             self.activation = 'sigmoid'
 
-
-
         if self.channel_map is None and self.is_time_to_event():
             self.channel_map = DEFAULT_TIME_TO_EVENT_CHANNELS
 

ml4h/arguments.py

@@ -385,9 +385,7 @@ def parse_args():
     )
 
     # Arguments for explorations/infer_stats_from_segmented_regions
-    parser.add_argument('--analyze_ground_truth', action='store_true', help='Filter by images with ground truth segmentations, for comparison')
-    parser.add_argument('--no_analyze_ground_truth', dest='analyze_ground_truth', action='store_false', help='Do not filter by images with ground truth segmentations, for comparison')
-    parser.set_defaults(analyze_ground_truth=True)
+    parser.add_argument('--analyze_ground_truth', default=False, action='store_true', help='Whether or not to filter by images with ground truth segmentations, for comparison')
     parser.add_argument('--structures_to_analyze', nargs='*', default=[], help='Structure names to include in the .tsv files and scatter plots')
     parser.add_argument('--erosion_radius', default=1, type=int, help='Radius of the unit disk structuring element for erosion preprocessing')
     parser.add_argument('--intensity_thresh', type=float, help='Threshold value for preprocessing')

ml4h/data_descriptions.py

@@ -5,6 +5,7 @@
 from typing import Callable, List, Union, Optional, Tuple, Dict, Any
 
 import h5py
+import datetime
 import numcodecs
 import numpy as np
 import pandas as pd
@@ -331,10 +332,9 @@ def __init__(
     ):
         """
         Gets data from a column of the provided DataFrame.
-        :param df: Must be multi-indexed with sample_id, loading_option
-        # TODO: allow multiple loading options
         :param col: The column name to get data from
         :param process_col: Function to turn the column value into Tensor
+        :param name: Optional overwrite of the df column name
         """
         self.process_col = process_col or self._default_process_call
         self.df = df

ml4h/explorations.py

@@ -38,7 +38,7 @@
 from ml4h.plots import evaluate_predictions, subplot_rocs, subplot_scatters, plot_categorical_tmap_over_time
 from ml4h.defines import JOIN_CHAR, MRI_SEGMENTED_CHANNEL_MAP, CODING_VALUES_MISSING, CODING_VALUES_LESS_THAN_ONE
 from ml4h.defines import TENSOR_EXT, IMAGE_EXT, ECG_CHAR_2_IDX, ECG_IDX_2_CHAR, PARTNERS_CHAR_2_IDX, PARTNERS_IDX_2_CHAR, PARTNERS_READ_TEXT
-from ml4h.tensorize.tensor_writer_ukbb import _unit_disk
+from ml4h.tensorize.tensor_writer_ukbb import unit_disk
 
 from sklearn.pipeline import make_pipeline
 from sklearn.preprocessing import StandardScaler
@@ -824,7 +824,7 @@ def infer_stats_from_segmented_regions(args):
 
     # Structuring element used for the erosion
     if args.erosion_radius > 0:
-        structure = _unit_disk(args.erosion_radius)[np.newaxis, ..., np.newaxis]
+        structure = unit_disk(args.erosion_radius)[np.newaxis, ..., np.newaxis]
 
     # Setup for intensity thresholding
     do_intensity_thresh = args.intensity_thresh_in_structures and args.intensity_thresh_out_structure

ml4h/models/legacy_models.py

@@ -355,7 +355,7 @@ def make_hidden_layer_model(parent_model: Model, tensor_maps_in: List[TensorMap]
     dummy_input = {tm.input_name(): np.zeros((1,) + parent_model.get_layer(tm.input_name()).input_shape[0][1:]) for tm in tensor_maps_in}
     intermediate_layer_model = Model(inputs=parent_inputs, outputs=target_layer.output)
     # If we do not predict here then the graph is disconnected, I do not know why?!
-    intermediate_layer_model.predict(dummy_input)
+    intermediate_layer_model.predict(dummy_input, verbose=0)
     return intermediate_layer_model
 
 
@@ -1349,7 +1349,7 @@ def make_paired_autoencoder_model(
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 def embed_model_predict(model, tensor_maps_in, embed_layer, test_data, batch_size):
     embed_model = make_hidden_layer_model(model, tensor_maps_in, embed_layer)
-    return embed_model.predict(test_data, batch_size=batch_size)
+    return embed_model.predict(test_data, batch_size=batch_size, verbose=0)
 
 
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

ml4h/plots.py

@@ -770,7 +770,7 @@ def plot_scatter(
 
     ax1.set_xlabel("Predictions")
     ax1.set_ylabel("Actual")
-    ax1.set_title(title)
+    ax1.set_title(f'{title} N = {len(prediction)}' )
     ax1.legend(loc="lower right")
 
     sns.distplot(prediction, label="Predicted", color="r", ax=ax2)
@@ -2253,7 +2253,7 @@ def plot_ecg_rest(
     tensor_paths: List[str],
     rows: List[int],
     out_folder: str,
-    is_blind: bool,
+    is_blind: bool
 ) -> None:
     """Plots resting ECGs including annotations and LVH criteria