fix conflicts

Author: alejoe91

.github/actions/build-test-environment/action.yml

@@ -15,14 +15,7 @@ runs:
       shell: bash
       run: |
         pip install datalad-installer
-        wget https://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-amd64.tar.gz
-        mkdir /home/runner/work/installation
-        mv git-annex-standalone-amd64.tar.gz /home/runner/work/installation/
-        workdir=$(pwd)
-        cd /home/runner/work/installation
-        tar xvzf git-annex-standalone-amd64.tar.gz
-        echo "$(pwd)/git-annex.linux" >> $GITHUB_PATH
-        cd $workdir
+        datalad-installer --sudo ok git-annex --method datalad/packages
         git config --global filter.annex.process "git-annex filter-process"  # recommended for efficiency
     - name: Force installation of latest dev from key-packages when running dev (not release)
       run: |

.github/scripts/test_kilosort4_ci.py

@@ -70,7 +70,6 @@
     "nearest_chans": 8,
     "nearest_templates": 35,
     "max_channel_distance": 5,
-    "templates_from_data": False,
     "n_templates": 10,
     "n_pcs": 3,
     "Th_single_ch": 4,
@@ -109,6 +108,14 @@
     # max_peels is not affecting the results in this short dataset
     PARAMETERS_NOT_AFFECTING_RESULTS.append("max_peels")
 
+if parse(kilosort.__version__) >= parse("4.0.33"):
+    PARAMS_TO_TEST_DICT.update({"cluster_neighbors": 11})
+    PARAMETERS_NOT_AFFECTING_RESULTS.append("cluster_neighbors")
+
+if parse(kilosort.__version__) >= parse("4.0.37"):
+    PARAMS_TO_TEST_DICT.update({"max_cluster_subset": 20})
+    PARAMETERS_NOT_AFFECTING_RESULTS.append("max_cluster_subset")
+
 
 PARAMS_TO_TEST = list(PARAMS_TO_TEST_DICT.keys())
 
@@ -178,11 +185,11 @@ def _save_ground_truth_recording(self, recording, tmp_path):
         """
         paths = {
             "session_scope_tmp_path": tmp_path,
-            "recording_path": tmp_path / "my_test_recording",
+            "recording_path": tmp_path / "my_test_recording" / "traces_cached_seg0.raw",
             "probe_path": tmp_path / "my_test_probe.prb",
         }
 
-        recording.save(folder=paths["recording_path"], overwrite=True)
+        recording.save(folder=paths["recording_path"].parent, overwrite=True)
 
         probegroup = recording.get_probegroup()
         write_prb(paths["probe_path"].as_posix(), probegroup)
@@ -214,7 +221,7 @@ def test_default_settings_all_represented(self):
         tested_keys += additional_non_tested_keys
 
         for param_key in DEFAULT_SETTINGS:
-            if param_key not in ["n_chan_bin", "fs", "tmin", "tmax"]:
+            if param_key not in ["n_chan_bin", "fs", "tmin", "tmax", "templates_from_data"]:
                 assert param_key in tested_keys, f"param: {param_key} in DEFAULT SETTINGS but not tested."
 
     def test_spikeinterface_defaults_against_kilsort(self):
@@ -234,8 +241,11 @@ def test_spikeinterface_defaults_against_kilsort(self):
 
     # Testing Arguments ###
     def test_set_files_arguments(self):
+        expected_arguments = ["settings", "filename", "probe", "probe_name", "data_dir", "results_dir", "bad_channels"]
+        if parse(kilosort.__version__) >= parse("4.0.34"):
+            expected_arguments += ["shank_idx"]
         self._check_arguments(
-            set_files, ["settings", "filename", "probe", "probe_name", "data_dir", "results_dir", "bad_channels"]
+            set_files, expected_arguments
         )
 
     def test_initialize_ops_arguments(self):
@@ -248,6 +258,8 @@ def test_initialize_ops_arguments(self):
             "device",
             "save_preprocessed_copy",
         ]
+        if parse(kilosort.__version__) >= parse("4.0.37"):
+            expected_arguments += ["gui_mode"]
 
         self._check_arguments(
             initialize_ops,
@@ -533,33 +545,60 @@ def test_kilosort4_skip_preprocessing_correction(self, tmp_path, monkeypatch, pa
         kilosort_output_dir = tmp_path / "kilosort_output_dir"
         spikeinterface_output_dir = tmp_path / "spikeinterface_output_dir"
 
-        def monkeypatch_filter_function(self, X, ops=None, ibatch=None):
-            """
-            This is a direct copy of the kilosort io.BinaryFiltered.filter
-            function, with hp_filter and whitening matrix code sections, and
-            comments removed. This is the easiest way to monkeypatch (tried a few approaches)
-            """
-            if self.chan_map is not None:
-                X = X[self.chan_map]
-
-            if self.invert_sign:
-                X = X * -1
-
-            X = X - X.mean(1).unsqueeze(1)
-            if self.do_CAR:
-                X = X - torch.median(X, 0)[0]
-
-            if self.hp_filter is not None:
-                pass
-
-            if self.artifact_threshold < np.inf:
-                if torch.any(torch.abs(X) >= self.artifact_threshold):
-                    return torch.zeros_like(X)
-
-            if self.whiten_mat is not None:
-                pass
-            return X
-
+        if parse(kilosort.__version__) >= parse("4.0.33"):
+            def monkeypatch_filter_function(self, X, ops=None, ibatch=None, skip_preproc=False):
+                """
+                This is a direct copy of the kilosort io.BinaryFiltered.filter
+                function, with hp_filter and whitening matrix code sections, and
+                comments removed. This is the easiest way to monkeypatch (tried a few approaches)
+                """
+                if self.chan_map is not None:
+                    X = X[self.chan_map]
+
+                if self.invert_sign:
+                    X = X * -1
+
+                X = X - X.mean(1).unsqueeze(1)
+                if self.do_CAR:
+                    X = X - torch.median(X, 0)[0]
+
+                if self.hp_filter is not None:
+                    pass
+
+                if self.artifact_threshold < np.inf:
+                    if torch.any(torch.abs(X) >= self.artifact_threshold):
+                        return torch.zeros_like(X)
+
+                if self.whiten_mat is not None:
+                    pass
+                return X
+        else:
+            def monkeypatch_filter_function(self, X, ops=None, ibatch=None):
+                """
+                This is a direct copy of the kilosort io.BinaryFiltered.filter
+                function, with hp_filter and whitening matrix code sections, and
+                comments removed. This is the easiest way to monkeypatch (tried a few approaches)
+                """
+                if self.chan_map is not None:
+                    X = X[self.chan_map]
+
+                if self.invert_sign:
+                    X = X * -1
+
+                X = X - X.mean(1).unsqueeze(1)
+                if self.do_CAR:
+                    X = X - torch.median(X, 0)[0]
+
+                if self.hp_filter is not None:
+                    pass
+
+                if self.artifact_threshold < np.inf:
+                    if torch.any(torch.abs(X) >= self.artifact_threshold):
+                        return torch.zeros_like(X)
+
+                if self.whiten_mat is not None:
+                    pass
+                return X
         monkeypatch.setattr("kilosort.io.BinaryFiltered.filter", monkeypatch_filter_function)
 
         ks_settings, _, ks_format_probe = self._get_kilosort_native_settings(recording, paths, param_key, param_value)
@@ -620,7 +659,7 @@ def _get_kilosort_native_settings(self, recording, paths, param_key, param_value
         are through the function, these are split here.
         """
         settings = {
-            "data_dir": paths["recording_path"],
+            "filename": paths["recording_path"],
             "n_chan_bin": recording.get_num_channels(),
             "fs": recording.get_sampling_frequency(),
         }

.github/workflows/all-tests.yml

@@ -107,14 +107,7 @@ jobs:
         run: |
           pip install datalad-installer
           if [ ${{ runner.os }} = 'Linux' ]; then
-            wget https://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-amd64.tar.gz
-            mkdir /home/runner/work/installation
-            mv git-annex-standalone-amd64.tar.gz /home/runner/work/installation/
-            workdir=$(pwd)
-            cd /home/runner/work/installation
-            tar xvzf git-annex-standalone-amd64.tar.gz
-            echo "$(pwd)/git-annex.linux" >> $GITHUB_PATH
-            cd $workdir
+            datalad-installer --sudo ok git-annex --method datalad/packages
           elif [ ${{ runner.os }} = 'macOS' ]; then
             datalad-installer --sudo ok git-annex --method brew
           elif [ ${{ runner.os }} = 'Windows' ]; then

README.md

@@ -134,4 +134,4 @@ If you find SpikeInterface useful in your research, please cite:
 ```
 
 Please also cite other relevant papers for the specific components you use.
-For a ful list of references, please check the [references](https://spikeinterface.readthedocs.io/en/latest/references.html) page.
+For a full list of references, please check the [references](https://spikeinterface.readthedocs.io/en/latest/references.html) page.

doc/api.rst

@@ -153,6 +153,7 @@ Non-NEO-based
     .. autofunction:: toy_example
     .. autofunction:: read_tridesclous
     .. autofunction:: read_waveclus
+    .. autofunction:: read_whitematter
     .. autofunction:: read_yass
 
 
@@ -336,6 +337,7 @@ spikeinterface.exporters
 .. automodule:: spikeinterface.exporters
 
     .. autofunction:: export_to_phy
+    .. autofunction:: export_to_ibl_gui
     .. autofunction:: export_report
 
 

doc/development/development.rst

@@ -397,3 +397,52 @@ After this you need to add a block in `Install Sorters <https://github.com/Spike
 to describe your sorter.
 
 Finally, make a pull request so we can review the code and incorporate into the sorters module of SpikeInterface!
+
+
+
+How to make a release
+---------------------
+
+Checklist
+^^^^^^^^^
+* pyproject.toml: check that the version is ahead of current release. Also, comment out the @ (git dependencies)
+* In the top level ``__init__`` (located at ``src/spikeinterface/__init__.py``) set ``DEV_MODE`` to ``False`` (this is used for the docker installations)
+* Create a new release note for the appropriate version on doc/releases/new_version_tag.
+
+There can be large releases like:
+
+``doc/releases/0.101.0.rst``
+
+Which contain a section called "Main Changes" and minor releases which include only bug fixes like:
+
+``doc/releases/0.101.2.rst``
+
+To collect all the PRs and bug fixes we have a script in:
+``doc/scripts/``
+called ``auto-release-notes.sh``. Run it with ``bash auto-release-notes.sh`` and it will create the release notes for the module specific changes.
+
+The first time you run the script, GitHub will guide you through an authorization process if you've not already done so.
+
+The signature of the script is:
+
+.. code-block:: bash
+
+    bash auto-release-notes.sh <start_date> <end_date>
+
+Where the start date is the date of the last release and the end date is the current date. Dates are in YYYY-MM-DD format
+
+The date of the last release can be found on `PyPI <https://pypi.org/project/spikeinterface/>`_.
+
+
+As a specific example:
+.. code-block:: bash
+
+    bash auto-release-notes.sh 2025-02-19 2025-03-24
+
+* Finish the release notes and merge
+* Locally tag the main branch with the newly merged release notes with the new version
+* Push the tag to the remote repository which will trigger the release action (.github/workflows/publish-to-pypi.yml)
+* Do an after-release `PR <https://github.com/SpikeInterface/spikeinterface/pull/3828/files>`_:
+    - Uncomment the git installs in pyproject
+    - Set ``DEV_MODE`` to ``True`` in the top level ``__init__`` (located at ``src/spikeinterface/__init__.py``)
+    - Update `pyproject.toml` version one patch ahead or one minor if it is larger one.

doc/how_to/combine_recordings.rst

@@ -4,7 +4,7 @@ Combine recordings in SpikeInterface
 In this tutorial we will walk through combining multiple recording objects. Sometimes this occurs due to hardware
 settings (e.g. Intan software has a default setting of new files every 1 minute) or the experimenter decides to
 split their recording into multiple files for different experimental conditions. If the probe has not been moved,
-however, then during sorting it would likely make sense to combine these individual reocrding objects into one
+however, then during sorting it would likely make sense to combine these individual recording objects into one
 recording object.
 
 **Why Combine?**