Merge branch 'main' into small_sample

Author: JasperVanDenBosch

.readthedocs.yml

@@ -10,9 +10,13 @@ sphinx:
 # Optionally build your docs in additional formats such as PDF
 formats: all
 
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.8"
+
 # Optionally set the version of Python and requirements required to build your docs
 python:
-  version: 3.7
   install:
     - requirements: requirements.txt
     - requirements: docs/requirements.txt

demos/demo_bootstrap.ipynb

@@ -98,8 +98,16 @@ def summary(self, test_type='t-test'):
         name_length = max([max(len(m.name) for m in self.models) + 1, 6])
         means = self.get_means()
         sems = self.get_sem()
-        p_zero = self.test_zero(test_type=test_type)
-        p_noise = self.test_noise(test_type=test_type)
+        if means is None:
+            means = np.nan * np.ones(self.n_model)
+        if sems is None:
+            sems = np.nan * np.ones(self.n_model)
+        try:
+            p_zero = self.test_zero(test_type=test_type)
+            p_noise = self.test_noise(test_type=test_type)
+        except ValueError:
+            p_zero = np.nan * np.ones(self.n_model)
+            p_noise = np.nan * np.ones(self.n_model)
         # header of the results table
         summary += 'Model' + (' ' * (name_length - 5))
         summary += '|   Eval \u00B1 SEM   |'
@@ -120,7 +128,9 @@ def summary(self, test_type='t-test'):
                 summary += f'{p_noise[i]:>14.3f}  |'
             summary += '\n'
         summary += '\n'
-        if test_type == 't-test':
+        if self.cv_method == 'crossvalidation':
+            summary += 'No p-values available as crossvalidation provides no variance estimate'
+        elif test_type == 't-test':
             summary += 'p-values are based on uncorrected t-tests'
         elif test_type == 'bootstrap':
             summary += 'p-values are based on percentiles of the bootstrap samples'
@@ -252,11 +262,11 @@ def get_errorbars(self, eb_type='sem', test_type='t-test'):
                 ci_percent = float(eb_type[2:]) / 100
             ci = self.get_ci(ci_percent, test_type)
             means = self.get_means()
-            errorbar_low = -(ci[0] - means)
+            errorbar_low = means - ci[0]
             errorbar_high = ci[1] - means
             limits = np.concatenate((errorbar_low, errorbar_high))
             if np.isnan(limits).any() or (abs(limits) == np.inf).any():
-                raise RuntimeError(
+                raise ValueError(
                     'plot_model_comparison: Too few bootstrap samples for ' +
                     'the requested confidence interval: ' + eb_type + '.')
         return (errorbar_low, errorbar_high)

src/rsatoolbox/inference/result.py

@@ -264,9 +264,10 @@ def plot_model_comparison(result, sort=False, colors=None,
         models = [models[i] for i in idx]
         if not ('descend' in sort.lower() or
                 'ascend' in sort.lower()):
-            raise Exception('plot_model_comparison: Argument ' +
-                            'sort is incorrectly defined as '
-                            + sort + '.')
+            raise ValueError(
+                'plot_model_comparison: Argument ' +
+                'sort is incorrectly defined as ' +
+                sort + '.')
 
     # run tests
     if any([test_pair_comparisons,
@@ -290,9 +291,10 @@ def plot_model_comparison(result, sort=False, colors=None,
         elif 'nili' in test_pair_comparisons.lower():
             h_pair_tests = 0.4
         else:
-            raise Exception('plot_model_comparison: Argument ' +
-                            'test_pair_comparisons is incorrectly defined as '
-                            + test_pair_comparisons + '.')
+            raise ValueError(
+                'plot_model_comparison: Argument ' +
+                'test_pair_comparisons is incorrectly defined as ' +
+                test_pair_comparisons + '.')
         ax = plt.axes((l, b, w, h*(1-h_pair_tests)))
         axbar = plt.axes((l, b + h * (1 - h_pair_tests), w,
                           h * h_pair_tests * 0.7))
@@ -360,7 +362,7 @@ def plot_model_comparison(result, sort=False, colors=None,
                     marker=10, markersize=half_sym_size,
                     linewidth=0)
         else:
-            raise Exception(
+            raise ValueError(
                 'plot_model_comparison: Argument test_above_0' +
                 ' is incorrectly defined as ' + test_above_0 + '.')
 
@@ -397,7 +399,7 @@ def plot_model_comparison(result, sort=False, colors=None,
                     markerfacecolor=noise_ceil_col,
                     markeredgecolor='none', linewidth=0)
         else:
-            raise Exception(
+            raise ValueError(
                 'plot_model_comparison: Argument ' +
                 'test_below_noise_ceil is incorrectly defined as ' +
                 test_below_noise_ceil + '.')
@@ -429,7 +431,7 @@ def plot_model_comparison(result, sort=False, colors=None,
             significant = p_pairwise < crit
         else:
             if 'uncorrected' not in multiple_pair_testing.lower():
-                raise Exception(
+                raise ValueError(
                     'plot_model_comparison: Argument ' +
                     'multiple_pair_testing is incorrectly defined as ' +
                     multiple_pair_testing + '.')
@@ -793,7 +795,11 @@ def plot_arrows(axbar, significant):
                 k += 1
             axbar.plot((i, j), (k, k), 'k-', linewidth=2)
             occupied[k-1, i*3+2:j*3+1] = 1
-    h = occupied.sum(axis=1).nonzero()[0].max()+1
+    h = occupied.sum(axis=1)
+    if np.any(h > 0):
+        h = h.nonzero()[0].max()+1
+    else:
+        h = 1
     axbar.set_ylim((0, max(expected_n_lines, h)))
 
 
@@ -905,7 +911,7 @@ def _get_model_comp_descr(test_type, n_models, multiple_pair_testing, alpha,
                             ' model-pair comparisons)')
     else:
         if 'uncorrected' not in multiple_pair_testing.lower():
-            raise Exception(
+            raise ValueError(
                 'plot_model_comparison: Argument ' +
                 'multiple_pair_testing is incorrectly defined as ' +
                 multiple_pair_testing + '.')

src/rsatoolbox/vis/model_plot.py

@@ -17,8 +17,9 @@
 class TestSimilarity(unittest.TestCase):
 
     def setUp(self):
-        self.v_i = np.random.rand(21)
-        self.v_j = np.random.rand(21)
+        self.rng = np.random.default_rng(0)
+        self.v_i = self.rng.random((21))
+        self.v_j = self.rng.random((21))
         self.vec_i = np.array([0.11, 0.12, 0.22, 0.30, 0.31])
         self.vec_j = np.array([0.13, 0.14, 0.21, 0.29, 0.28])
 
@@ -61,8 +62,9 @@ def test_noise(self):
 class TestCalcOne(unittest.TestCase):
 
     def setUp(self):
-        self.dat_i = np.random.rand(2, 21)
-        self.dat_j = np.random.rand(3, 21)
+        self.rng = np.random.default_rng(0)
+        self.dat_i = self.rng.random((2, 21))
+        self.dat_j = self.rng.random((3, 21))
         self.dat = np.concatenate((self.dat_i, self.dat_j), 0)
         self.data = rsatoolbox.data.Dataset(
             self.dat, obs_descriptors={'idx': [1, 1, 2, 2, 2]})
@@ -90,14 +92,16 @@ def test_integer_input_one(self):
         from rsatoolbox.data.dataset import Dataset
         ds1 = Dataset(np.asarray([[0], [2]]).T)  # one pattern, two channels
         ds2 = Dataset(np.asarray([[0], [2]]).T)  # one pattern, two channels
-        dissim, _ = calc_one_similarity_c(ds1, ds2, np.array([0]), np.array([1]))
+        dissim, _ = calc_one_similarity_c(
+            ds1, ds2, np.array([0]), np.array([1]))
         assert_almost_equal(dissim, 2)  # standard-squared euclidean
 
 
 class TestCalc(unittest.TestCase):
 
     def setUp(self):
-        self.dat = np.random.rand(300, 100)
+        self.rng = np.random.default_rng(0)
+        self.dat = self.rng.random((300, 100))
         self.data = rsatoolbox.data.Dataset(
             self.dat,
             obs_descriptors={'obs': np.repeat(np.arange(50), 6),

tests/test_c.py

@@ -29,7 +29,8 @@
 class TestCalcRDM(unittest.TestCase):
 
     def setUp(self):
-        measurements = np.random.rand(20, 5)
+        self.rng = np.random.default_rng(0)
+        measurements = self.rng.random((20, 5))
         measurements_deterministic = np.array(
             [
                 [0.11, 0.12, 0.21, 0.22, 0.30, 0.31],
@@ -85,7 +86,7 @@ def test_parse_input(self):
         from rsatoolbox.rdm.calc import _parse_input
         data = Mock()
         data.descriptors = {'session': 0, 'subj': 0}
-        data.measurements = np.random.rand(6, 5)
+        data.measurements = self.rng.random((6, 5))
         measurements, desc = _parse_input(data, None)
         self.assertIsNone(desc)
         self.assertTrue(np.all(data.measurements == measurements))
@@ -97,7 +98,7 @@ def test_calc_euclidean_as_scipy(self, _parse_input):
         data.descriptors = {'session': 0, 'subj': 0}
         data.obs_descriptors = dict(conds=np.arange(6))
         data.channel_descriptors = dict()
-        data.measurements = np.random.rand(6, 5)
+        data.measurements = self.rng.random((6, 5))
         desc = [0, 1, 2, 3, 4, 5]
         _parse_input.return_value = (data.measurements, desc)
         rdm_expected = pdist(data.measurements) ** 2 / 5
@@ -112,7 +113,7 @@ def test_calc_correlation(self, _parse_input):
         data.descriptors = {'session': 0, 'subj': 0}
         data.obs_descriptors = dict(conds=np.arange(6))
         data.channel_descriptors = dict()
-        data.measurements = np.random.rand(6, 5)
+        data.measurements = self.rng.random((6, 5))
         desc = [0, 1, 2, 3, 4, 5]
         _parse_input.return_value = (data.measurements, desc)
         rdm_expected = 1 - np.corrcoef(data.measurements)
@@ -141,7 +142,7 @@ def test_calc_mahalanobis(self):
             method='mahalanobis'
         )
         assert rdm.n_cond == 6
-        noise = np.linalg.inv(np.cov(np.random.randn(10, 5).T))
+        noise = np.linalg.inv(np.cov(self.rng.standard_normal((10, 5)).T))
         rdm = rsr.calc_rdm(
             self.test_data,
             descriptor='conds',
@@ -202,7 +203,7 @@ def test_calc_crossnobis_no_descriptors(self):
         assert rdm.n_cond == 5
 
     def test_calc_crossnobis_noise(self):
-        noise = np.random.randn(10, 5)
+        noise = self.rng.standard_normal((10, 5))
         noise = np.matmul(noise.T, noise)
         rdm = rsr.calc_rdm_crossnobis(
             self.test_data_balanced, descriptor='conds', noise=noise
@@ -211,7 +212,7 @@ def test_calc_crossnobis_noise(self):
 
     def test_calc_crossnobis_noise_list(self):
         # generate two positive definite noise matricies
-        noise = np.random.randn(2, 10, 5)
+        noise = self.rng.standard_normal((2, 10, 5))
         noise = np.einsum('ijk,ijl->ikl', noise, noise)
         rdm = rsr.calc_rdm_crossnobis(
             self.test_data_balanced,
@@ -377,7 +378,8 @@ def test_calc_with_descriptors_as_list(self):
 class TestCalcRDMMovie(unittest.TestCase):
 
     def setUp(self):
-        measurements_time = np.random.rand(20, 5, 15)
+        self.rng = np.random.default_rng(0)
+        measurements_time = self.rng.random((20, 5, 15))
         tim_des = {'time': np.linspace(0, 200, 15)}
         des = {'session': 0, 'subj': 0}
         obs_des = {
@@ -471,7 +473,7 @@ def test_calc_rdm_movie_crossnobis_no_descriptors(self):
         assert rdm.n_cond == 5
 
     def test_calc_rdm_movie_crossnobis_noise(self):
-        noise = np.random.randn(10, 5)
+        noise = self.rng.standard_normal((10, 5))
         noise = np.matmul(noise.T, noise)
         rdm = rsr.calc_rdm_movie(
             self.test_data_time_balanced,
@@ -483,7 +485,7 @@ def test_calc_rdm_movie_crossnobis_noise(self):
         assert rdm.n_cond == 5
 
     def test_calc_rdm_movie_rdm_movie_poisson(self):
-        noise = np.random.randn(10, 5)
+        noise = self.rng.standard_normal((10, 5))
         noise = np.matmul(noise.T, noise)
         rdm = rsr.calc_rdm_movie(
             self.test_data_time_balanced,

tests/test_calc_rdm.py

@@ -29,7 +29,8 @@
 class TestCalcUnbalancedRDM(unittest.TestCase):
 
     def setUp(self):
-        measurements = np.random.rand(20, 5)
+        self.rng = np.random.default_rng(0)
+        measurements = self.rng.random((20, 5))
         measurements_deterministic = np.array([
             [0.11, 0.12, 0.21, 0.22, 0.30, 0.31],
             [0.13, 0.14, 0.24, 0.21, 0.29, 0.28],
@@ -52,13 +53,13 @@ def setUp(self):
             descriptors=des,
             obs_descriptors=obs_des,
             channel_descriptors=chn_des
-            )
+        )
         self.test_data_balanced = rsa.data.Dataset(
             measurements=measurements,
             descriptors=des,
             obs_descriptors=obs_balanced,
             channel_descriptors=chn_des
-            )
+        )
         self.test_data_deterministic = rsa.data.Dataset(
             measurements=measurements_deterministic,
             descriptors=des,
@@ -80,10 +81,10 @@ def test_calc_euclid_nconds(self):
     def test_calc_euclid_as_scipy(self, _parse_input):
         from rsatoolbox.rdm import calc_rdm_unbalanced
         data = rsa.data.Dataset(
-            np.random.rand(6, 5),
+            self.rng.random((6, 5)),
             descriptors={'session': 0, 'subj': 0},
             obs_descriptors={'conds': [0, 1, 2, 3, 4, 5]}
-            )
+        )
         rdm_expected = pdist(data.measurements) ** 2 / 5
         rdms = calc_rdm_unbalanced(
             data,
@@ -94,17 +95,17 @@ def test_calc_euclid_as_scipy(self, _parse_input):
             assert_array_almost_equal(
                 rdm_expected,
                 rdms.dissimilarities.flatten()
-                )
             )
+        )
 
     @patch('rsatoolbox.rdm.calc._parse_input')
     def test_calc_correlation(self, _parse_input):
         from rsatoolbox.rdm import calc_rdm_unbalanced
         data = rsa.data.Dataset(
-            np.random.rand(6, 5),
+            self.rng.random((6, 5)),
             descriptors={'session': 0, 'subj': 0},
             obs_descriptors={'conds': [0, 1, 2, 3, 4, 5]}
-            )
+        )
         rdm_expected = 1 - np.corrcoef(data.measurements)
         rdme = rsr.RDMs(
             dissimilarities=np.array([rdm_expected]),
@@ -143,7 +144,7 @@ def test_calc_mahalanobis(self):
         assert_array_almost_equal(
             rdm_bal.dissimilarities.flatten(),
             rdm_check.dissimilarities.flatten()
-            )
+        )
 
     def test_calc_crossnobis(self):
         rdm = rsr.calc_rdm_unbalanced(self.test_data,
@@ -164,7 +165,7 @@ def test_calc_crossnobis(self):
         assert_array_almost_equal(
             rdm_bal.dissimilarities.flatten(),
             rdm_check.dissimilarities.flatten()
-            )
+        )
 
     def test_calc_crossnobis_no_descriptor(self):
         rdm = rsr.calc_rdm_unbalanced(
@@ -174,7 +175,7 @@ def test_calc_crossnobis_no_descriptor(self):
         assert rdm.n_cond == 6
 
     def test_calc_crossnobis_noise(self):
-        noise = np.random.randn(10, 5)
+        noise = self.rng.random((10, 5))
         noise = np.matmul(noise.T, noise)
         rdm = rsr.calc_rdm_unbalanced(
             self.test_data,
@@ -195,7 +196,7 @@ def test_calc_crossnobis_noise(self):
         assert_array_almost_equal(
             rdm_bal.dissimilarities.flatten(),
             rdm_check.dissimilarities.flatten()
-            )
+        )
 
     def test_calc_poisson(self):
         """ for the poisson-KL the dissimilarities differ! This is explained
@@ -305,4 +306,4 @@ def test_calc_multi_ds_passes_on_descriptors(self, method):
         assert_array_equal(
             rdms.pattern_descriptors['conds'],
             self.test_data.obs_descriptors['conds']
-        )
+        )