Merge pull request #317 from rsagroup/fix_print_result

fixing print for crossvalidation

Author: JasperVanDenBosch

demos/demo_bootstrap.ipynb

@@ -96,8 +96,16 @@ def summary(self, test_type='t-test'):
         name_length = max([max(len(m.name) for m in self.models) + 1, 6])
         means = self.get_means()
         sems = self.get_sem()
-        p_zero = self.test_zero(test_type=test_type)
-        p_noise = self.test_noise(test_type=test_type)
+        if means is None:
+            means = np.nan * np.ones(self.n_model)
+        if sems is None:
+            sems = np.nan * np.ones(self.n_model)
+        try:
+            p_zero = self.test_zero(test_type=test_type)
+            p_noise = self.test_noise(test_type=test_type)
+        except ValueError:
+            p_zero = np.nan * np.ones(self.n_model)
+            p_noise = np.nan * np.ones(self.n_model)
         # header of the results table
         summary += 'Model' + (' ' * (name_length - 5))
         summary += '|   Eval \u00B1 SEM   |'
@@ -118,7 +126,9 @@ def summary(self, test_type='t-test'):
                 summary += f'{p_noise[i]:>14.3f}  |'
             summary += '\n'
         summary += '\n'
-        if test_type == 't-test':
+        if self.cv_method == 'crossvalidation':
+            summary += 'No p-values available as crossvalidation provides no variance estimate'
+        elif test_type == 't-test':
             summary += 'p-values are based on uncorrected t-tests'
         elif test_type == 'bootstrap':
             summary += 'p-values are based on percentiles of the bootstrap samples'
@@ -250,11 +260,11 @@ def get_errorbars(self, eb_type='sem', test_type='t-test'):
                 ci_percent = float(eb_type[2:]) / 100
             ci = self.get_ci(ci_percent, test_type)
             means = self.get_means()
-            errorbar_low = -(ci[0] - means)
-            errorbar_high = (ci[1] - means)
+            errorbar_low = means - ci[0]
+            errorbar_high = ci[1] - means
             limits = np.concatenate((errorbar_low, errorbar_high))
             if np.isnan(limits).any() or (abs(limits) == np.inf).any():
-                raise Exception(
+                raise ValueError(
                     'plot_model_comparison: Too few bootstrap samples for ' +
                     'the requested confidence interval: ' + eb_type + '.')
         return (errorbar_low, errorbar_high)

src/rsatoolbox/inference/result.py

@@ -264,9 +264,10 @@ def plot_model_comparison(result, sort=False, colors=None,
         models = [models[i] for i in idx]
         if not ('descend' in sort.lower() or
                 'ascend' in sort.lower()):
-            raise Exception('plot_model_comparison: Argument ' +
-                            'sort is incorrectly defined as '
-                            + sort + '.')
+            raise ValueError(
+                'plot_model_comparison: Argument ' +
+                'sort is incorrectly defined as ' +
+                sort + '.')
 
     # run tests
     if any([test_pair_comparisons,
@@ -290,9 +291,10 @@ def plot_model_comparison(result, sort=False, colors=None,
         elif 'nili' in test_pair_comparisons.lower():
             h_pair_tests = 0.4
         else:
-            raise Exception('plot_model_comparison: Argument ' +
-                            'test_pair_comparisons is incorrectly defined as '
-                            + test_pair_comparisons + '.')
+            raise ValueError(
+                'plot_model_comparison: Argument ' +
+                'test_pair_comparisons is incorrectly defined as ' +
+                test_pair_comparisons + '.')
         ax = plt.axes((l, b, w, h*(1-h_pair_tests)))
         axbar = plt.axes((l, b + h * (1 - h_pair_tests), w,
                           h * h_pair_tests * 0.7))
@@ -360,7 +362,7 @@ def plot_model_comparison(result, sort=False, colors=None,
                     marker=10, markersize=half_sym_size,
                     linewidth=0)
         else:
-            raise Exception(
+            raise ValueError(
                 'plot_model_comparison: Argument test_above_0' +
                 ' is incorrectly defined as ' + test_above_0 + '.')
 
@@ -397,7 +399,7 @@ def plot_model_comparison(result, sort=False, colors=None,
                     markerfacecolor=noise_ceil_col,
                     markeredgecolor='none', linewidth=0)
         else:
-            raise Exception(
+            raise ValueError(
                 'plot_model_comparison: Argument ' +
                 'test_below_noise_ceil is incorrectly defined as ' +
                 test_below_noise_ceil + '.')
@@ -429,7 +431,7 @@ def plot_model_comparison(result, sort=False, colors=None,
             significant = p_pairwise < crit
         else:
             if 'uncorrected' not in multiple_pair_testing.lower():
-                raise Exception(
+                raise ValueError(
                     'plot_model_comparison: Argument ' +
                     'multiple_pair_testing is incorrectly defined as ' +
                     multiple_pair_testing + '.')
@@ -793,7 +795,11 @@ def plot_arrows(axbar, significant):
                 k += 1
             axbar.plot((i, j), (k, k), 'k-', linewidth=2)
             occupied[k-1, i*3+2:j*3+1] = 1
-    h = occupied.sum(axis=1).nonzero()[0].max()+1
+    h = occupied.sum(axis=1)
+    if np.any(h > 0):
+        h = h.nonzero()[0].max()+1
+    else:
+        h = 1
     axbar.set_ylim((0, max(expected_n_lines, h)))
 
 
@@ -905,7 +911,7 @@ def _get_model_comp_descr(test_type, n_models, multiple_pair_testing, alpha,
                             ' model-pair comparisons)')
     else:
         if 'uncorrected' not in multiple_pair_testing.lower():
-            raise Exception(
+            raise ValueError(
                 'plot_model_comparison: Argument ' +
                 'multiple_pair_testing is incorrectly defined as ' +
                 multiple_pair_testing + '.')

src/rsatoolbox/vis/model_plot.py

@@ -37,17 +37,19 @@ def test_crossval(self):
         from rsatoolbox.inference import crossval
         rdms = self.rdms
         m = self.m
-        train_set = [(rdms.subset_pattern('type', [0, 1]), np.array([0, 1])),
-                     (rdms.subset_pattern('type', [0, 4]), np.array([0, 4])),
+        train_set = [(rdms.subset_pattern('type', [0, 1, 2]), np.array([0, 1, 2])),
+                     (rdms.subset_pattern('type', [3, 4, 5]), np.array([3, 4, 5])),
                      ]
-        test_set = [(rdms.subset_pattern('type', [2, 4]), np.array([2, 4])),
-                    (rdms.subset_pattern('type', [1, 2]), np.array([1, 2])),
+        test_set = [(rdms.subset_pattern('type', [3, 4, 5]), np.array([3, 4, 5])),
+                    (rdms.subset_pattern('type', [0, 1, 2]), np.array([0, 1, 2])),
                     ]
-        ceil_set = [(rdms.subset_pattern('type', [2, 4]), np.array([2, 4])),
-                    (rdms.subset_pattern('type', [1, 2]), np.array([1, 2])),
+        ceil_set = [(rdms.subset_pattern('type', [3, 4, 5]), np.array([3, 4, 5])),
+                    (rdms.subset_pattern('type', [0, 1, 2]), np.array([0, 1, 2])),
                     ]
-        crossval(m, rdms, train_set, test_set, ceil_set,
-                 pattern_descriptor='type')
+        res = crossval(
+            m, rdms, train_set, test_set, ceil_set,
+            pattern_descriptor='type')
+        print(res)
 
     def test_bootstrap_crossval(self):
         from rsatoolbox.inference import bootstrap_crossval

tests/test_crossval.py

@@ -292,7 +292,12 @@ def test_exercise_all(self):
             models_flex, rdms_data, train_set, test_set,
             ceil_set=ceil_set, method='corr')
         # plot results
-        rsatoolbox.vis.plot_model_comparison(results_3_cv)
+        rsatoolbox.vis.plot_model_comparison(
+            results_3_cv,
+            error_bars=False,
+            test_pair_comparisons=False,
+            test_above_0=False,
+            test_below_noise_ceil=False)
 
         results_3_full = rsatoolbox.inference.bootstrap_crossval(
             models_flex, rdms_data, k_pattern=4, k_rdm=2, method='corr', N=5)