01_utils.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "from nbdev.test import test_eq\n",
    "from nbdev.showdoc import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Utils\n",
    "> Useful functions to reshape/arrange/reduce raw data into clean data to add to the record"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import glob\n",
    "import re\n",
    "from typing import Dict, Tuple, Sequence, Union, Callable\n",
    "import scipy.interpolate as interpolate\n",
    "from scipy.ndimage import convolve1d\n",
    "from scipy.signal import savgol_filter\n",
    "import scipy.stats\n",
    "from scipy.ndimage import gaussian_filter\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "from cmath import *\n",
    "from PIL import Image\n",
    "\n",
    "from theonerig.core import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Synchronisation utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def extend_sync_timepoints(timepoints:np.ndarray, signals:np.ndarray, \n",
    "                           up_bound, low_bound=0) -> Tuple[DataChunk, DataChunk]:\n",
    "    \"\"\"\n",
    "    Extend arrays of timepoints and signals (with identical shape) from the low_bound up to the up_bound.\n",
    "    For example, the first timepoint could be 2000, and with a low_bound of 0, it would add the\n",
    "    timepoints 0, 500, 1000, 1500 if the timepoint distance is of 500 (obtained by averaging the timepoints\n",
    "    distances).\n",
    "    \n",
    "    params:\n",
    "        - timepoints: Timepoints to extend\n",
    "        - signals: Signals to extend\n",
    "        - up_bound: Up bound to which to extend both timepoints and signals\n",
    "        - low_bound: Low bound to which to extend both timepoints and signals\n",
    "        \n",
    "    returns:\n",
    "        - timepoint: Extended timepoints\n",
    "        - signals: The datachunk array is not modified, but the idx attribute is increased by the number\n",
    "        of frames added with the low_bound.\n",
    "    \"\"\"\n",
    "    assert len(timepoints) == len(signals)\n",
    "    timepoints = np.array(timepoints)\n",
    "    signals = np.array(signals)\n",
    "    spb = np.mean(timepoints[1:]-timepoints[:-1]) #spf: sample_per_bin\n",
    "        \n",
    "    #Left and right side are just prolongation of the sample_times up \n",
    "    # from (0-sample_per_fr) to (len+sample_per_fr) so it covers all timepoints\n",
    "    left_side  = np.arange(timepoints[0]-spb , low_bound - spb, -spb)[::-1].astype(int)\n",
    "    right_side = np.arange(timepoints[-1]+spb,  up_bound + spb,  spb).astype(int)\n",
    "\n",
    "    new_timepoints = np.concatenate((left_side, \n",
    "                                     timepoints, \n",
    "                                     right_side))\n",
    "    \n",
    "    timepoint_chunk = DataChunk(data=new_timepoints, idx=0, group=\"sync\")\n",
    "    signal_chunk    = DataChunk(data=signals, idx=len(left_side), group=\"sync\")\n",
    "    return (timepoint_chunk, signal_chunk)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def align_sync_timepoints(timepoints:DataChunk, signals:DataChunk,\n",
    "                          ref_timepoints:DataChunk, ref_signals:DataChunk) -> Tuple[DataChunk, DataChunk, DataChunk]:\n",
    "    \"\"\"\n",
    "    Align the signals of a timepoints timeserie to a reference ref_timepoints with the corresponding\n",
    "    ref_signals. ref_timepoints is extended to match ref_timepoints lenght.\n",
    "    \n",
    "    params:\n",
    "        - timepoints: timepoints to align\n",
    "        - signals: signals to align\n",
    "        - ref_timepoints: reference timepoints\n",
    "        - ref_signals: reference signals\n",
    "        \n",
    "    return:\n",
    "        - Aligned timepoints (DataChunk)\n",
    "        - Aligned signals (DataChunk)\n",
    "    \"\"\"    \n",
    "    shift_left = ((np.where(ref_signals)[0][0] + ref_signals.idx) \n",
    "                  - (np.where(signals)[0][0] + signals.idx))\n",
    "    shift_right   = len(ref_timepoints) - (len(timepoints) + shift_left) \n",
    "\n",
    "    spb     = np.mean(timepoints[1:]-timepoints[:-1]) #spf: sample_per_bin\n",
    "    spb_ref = np.mean(ref_timepoints[1:]-ref_timepoints[:-1]) #spf: sample_per_bin\n",
    "    \n",
    "    left_timepoints      = np.zeros(0)\n",
    "    left_timepoints_ref  = np.zeros(0)\n",
    "    right_timepoints     = np.zeros(0)\n",
    "    right_timepoints_ref = np.zeros(0)\n",
    "    \n",
    "    if shift_left > 0: #the ref started before, need to extend the other\n",
    "        init  = timepoints[0]-spb\n",
    "        left_timepoints = np.arange(init , \n",
    "                                    init-(spb*shift_left+1), \n",
    "                                    -spb)[:shift_left][::-1].astype(int)\n",
    "    else:\n",
    "        shift_left = abs(shift_left)\n",
    "        init  = ref_timepoints[0]-spb_ref\n",
    "        left_timepoints_ref = np.arange(init , \n",
    "                                        init-(spb_ref*shift_left+1), \n",
    "                                        -spb_ref)[:shift_left][::-1].astype(int)\n",
    "        #We also need to shift the index of the ref signals since we increased the size of the ref_timepoints\n",
    "        ref_signals.idx = ref_signals.idx + len(left_timepoints_ref)\n",
    "        \n",
    "    if shift_right > 0: #the ref ended after, need to extend the other\n",
    "        init  = timepoints[-1]+spb\n",
    "        right_timepoints = np.arange(init , \n",
    "                                    init+(spb*shift_right+1), \n",
    "                                    spb)[:shift_right].astype(int)\n",
    "    else:\n",
    "        shift_right = abs(shift_right)\n",
    "        init  = ref_timepoints[-1]+spb_ref\n",
    "        right_timepoints_ref = np.arange(init , \n",
    "                                    init+(spb_ref*shift_right+1), \n",
    "                                    spb_ref)[:shift_right].astype(int)\n",
    "        \n",
    "    timepoint    = DataChunk(data=np.concatenate((left_timepoints, \n",
    "                                     timepoints, \n",
    "                                     right_timepoints)), idx=0, group=\"sync\")\n",
    "    \n",
    "    timepoint_ref = DataChunk(data=np.concatenate((left_timepoints_ref, \n",
    "                                     ref_timepoints, \n",
    "                                     right_timepoints_ref)), idx=0, group=\"sync\")\n",
    "    \n",
    "    return (timepoint, timepoint_ref, ref_signals)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def resample_to_timepoints(timepoints:np.ndarray, data:np.ndarray, \n",
    "                             ref_timepoints:DataChunk, group=\"data\") -> DataChunk:\n",
    "    \"\"\"\n",
    "    Resample the data at timepoints to new timepoints given by ref_timepoints.\n",
    "    Return a DataChunk of the resampled data belonging to a specified group.\n",
    "    \n",
    "    params:\n",
    "        - timepoints: Original timepoints of the data\n",
    "        - data: Data to resample of shape (t, ...)\n",
    "        - ref_timepoints: Target timepoints for the resampling\n",
    "        - group: Group assigned to the returned DataChunk\n",
    "        \n",
    "    return:\n",
    "        - Resampled datachunk with appropriate idx.\n",
    "    \"\"\"\n",
    "    \n",
    "    assert len(timepoints) == len(data)\n",
    "    timepoints = np.array(timepoints)\n",
    "    data = np.array(data)\n",
    "    \n",
    "    start_idx = np.argmax(ref_timepoints >= timepoints[0])\n",
    "    stop_idx  = np.argmax(ref_timepoints >= timepoints[-1])\n",
    "    if stop_idx == 0:\n",
    "        stop_idx = len(ref_timepoints)\n",
    "    \n",
    "    if len(ref_timepoints[start_idx:stop_idx]) < len(timepoints): #Downsampling\n",
    "        distance = (np.argmax(timepoints>ref_timepoints[start_idx+1]) \n",
    "                - np.argmax(timepoints>ref_timepoints[start_idx]))\n",
    "    \n",
    "        kernel = np.ones(distance)/distance\n",
    "        data = convolve1d(data, kernel, axis=0) #Smooting to avoid weird sampling\n",
    "\n",
    "    new_data = interpolate.interp1d(timepoints, data, axis=0)(ref_timepoints[start_idx:stop_idx])\n",
    "\n",
    "    idx = ref_timepoints.idx + start_idx\n",
    "    return DataChunk(data=new_data, idx = idx, group=group)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def link_sync_timepoints(frame_tp_1, frame_sig_1, frame_tp_2, frame_sig_2):\n",
    "    \"\"\"\n",
    "    Creates timepoints between two timepoints array sampled at the same rate.\n",
    "    This is usefull for the LED dome which cannot generate frames in between stimuli (due to ROM update)\n",
    "    \n",
    "    params:\n",
    "        - frame_tp_1: Timepoints of the first part\n",
    "        - frame_sig_1: Signals of the first part\n",
    "        - frame_tp_2: Timepoints of the second part\n",
    "        - frame_sig_2: Signals of the second part\n",
    "    return:\n",
    "        - (concatenated_frame_timepoints, concatenated_frame_signals)\n",
    "    \"\"\"\n",
    "    assert abs(np.diff(frame_tp_1).mean() - np.diff(frame_tp_2).mean())<10, \"The frame rates are different\"\n",
    "    assert len(frame_tp_1)==len(frame_sig_1), \"The lenght of the first signals and timepoints do not match\"\n",
    "    assert len(frame_tp_2)==len(frame_sig_2), \"The lenght of the second signals and timepoints do not match\"\n",
    "    \n",
    "    n_tp = np.diff(frame_tp_1).mean()\n",
    "    n_new_frames = int(round((frame_tp_2[0] - frame_tp_1[-1])/n_tp) - 1)\n",
    "    new_frames = np.linspace(int(frame_tp_1[-1]+n_tp), frame_tp_2[0], n_new_frames, endpoint=False).astype(int)\n",
    "    \n",
    "    concat_frame_tp  = np.concatenate((frame_tp_1, new_frames, frame_tp_2))\n",
    "    concat_frame_sig = np.concatenate((frame_sig_1, [0]*n_new_frames, frame_sig_2))\n",
    "    \n",
    "    return concat_frame_tp, concat_frame_sig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def flip_stimulus(stim_inten, ud_inv, lr_inv):\n",
    "    \"\"\"\n",
    "    Flip QDSpy stimuli arrays to match the up/down left/right orientation of the stimulus displayed to \n",
    "    the mouse. \n",
    "    \n",
    "    params:\n",
    "        - stim_inten: Stimulus matrix to flip of shape (t, color, y, x)\n",
    "        - ud_inv: Up down inversion boolean (1 to make the flip, 0 for no operation)\n",
    "        - lr_inv: Up down inversion boolean (1 to make the flip, 0 for no operation)\n",
    "        \n",
    "    return:\n",
    "        - Flipped stimulus array\n",
    "    \"\"\"\n",
    "    if lr_inv:\n",
    "        stim_inten = np.flip(stim_inten, axis=3) # Axis 0:t 1:color 2:y 3:x\n",
    "    if not ud_inv: \n",
    "        #Numpy and QDSpy orientation are different. \n",
    "        #This way reorientate the stimulus approriatly for display with matplotlib and potential\n",
    "        #eye tracking corrections\n",
    "        stim_inten = np.flip(stim_inten, axis=2)\n",
    "    return stim_inten\n",
    "\n",
    "def flip_gratings(stim_shader, ud_inv, lr_inv):\n",
    "    \"\"\"\n",
    "    Flip gratings to match the up/down left/right orientation of the stimulus displayed to \n",
    "    the mouse. A grating is encoded by an array of shape (t, 3(size, angle, speed)). \n",
    "    Therefore the angles of the grating are modified to encode the \"flipped\" grating.\n",
    "    \n",
    "    params:\n",
    "        - stim_shader: Grating matrix to flip of shape (t, 3(size, angle(degree), speed))\n",
    "        - ud_inv: Up down inversion boolean (1 to make the flip, 0 for no operation)\n",
    "        - lr_inv: Up down inversion boolean (1 to make the flip, 0 for no operation)\n",
    "        \n",
    "    return:\n",
    "        - Flipped grating array\n",
    "    \"\"\"\n",
    "    mask_epochs = ~np.all(stim_shader==0,axis=1)\n",
    "    if lr_inv:\n",
    "        stim_shader[mask_epochs,1] = (360 + (180 - stim_shader[mask_epochs,1])) % 360 \n",
    "    if ud_inv:\n",
    "        stim_shader[mask_epochs,1] = (360 - stim_shader[mask_epochs,1]) % 360\n",
    "    return stim_shader\n",
    "\n",
    "def stim_to_dataChunk(stim_inten, stim_start_idx, reference:DataChunk) -> DataChunk:\n",
    "    \"\"\"\n",
    "    Factory function for DataChunk of a stimulus, that squeeze the stim_inten matrix.\n",
    "    \n",
    "    params:\n",
    "        - stim_inten: Stimulus matrix of shape (t, ...)\n",
    "        - stim_start_idx: Starting frame index of the stimulus\n",
    "        - reference: DataChunk signal reference used to determine the starting index of the stimulus\n",
    "        \n",
    "    return:\n",
    "        - Datachunk of the stimulus\n",
    "    \"\"\"\n",
    "    return DataChunk(data=np.squeeze(stim_inten), idx = (stim_start_idx + reference.idx), group=\"stim\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def phy_results_dict(phy_dir):\n",
    "    \"\"\"\n",
    "    Open the result arrays of spike sorting after manual merging with phy.\n",
    "\n",
    "    params:\n",
    "        - phy_dir: path to the phy results\n",
    "\n",
    "    return:\n",
    "        - Dictionnary of the phy arrays (amplitudes, channel_map, channel_positions, spike_clusters,\n",
    "        spike_templates, spike_times, templates)\n",
    "    \"\"\"\n",
    "    res_dict = {}\n",
    "    res_dict[\"amplitudes\"] = np.load(phy_dir+\"/amplitudes.npy\")\n",
    "    res_dict[\"channel_map\"] = np.load(phy_dir+\"/channel_map.npy\")\n",
    "    res_dict[\"channel_positions\"] = np.load(phy_dir+\"/channel_positions.npy\")\n",
    "    res_dict[\"spike_clusters\"] = np.load(phy_dir+\"/spike_clusters.npy\")\n",
    "    res_dict[\"spike_templates\"] = np.load(phy_dir+\"/spike_templates.npy\")\n",
    "    res_dict[\"spike_times\"] = np.load(phy_dir+\"/spike_times.npy\")\n",
    "    res_dict[\"templates\"] = np.load(phy_dir+\"/templates.npy\")\n",
    "    if os.path.isfile(phy_dir+\"/channel_shanks.npy\"): #Newer version of phy/spyking-circus\n",
    "        res_dict[\"channel_shanks\"] = np.load(phy_dir+\"/channel_shanks.npy\")\n",
    "        res_dict[\"template_ind\"]   = np.load(phy_dir+\"/template_ind.npy\")\n",
    "        \n",
    "    return res_dict\n",
    "\n",
    "def spike_to_dataChunk(spike_timepoints, ref_timepoints:DataChunk) -> DataChunk:\n",
    "    \"\"\"\n",
    "    Factory function of a DataChunk for spiking count of cells from spike timepoints.\n",
    "    \n",
    "    params:\n",
    "        - spike_timepoints: Dictionnary of the cells spike timepoints (list)\n",
    "        - ref_timepoints: Reference DataChunk to align the newly created spike count Datachunk\n",
    "        \n",
    "    return:\n",
    "        - Spike count datachunk of shape (t, n_cell)\n",
    "    \"\"\"\n",
    "    type_cast = type(list(spike_timepoints.keys())[0])\n",
    "    cell_keys = sorted(map(int, \n",
    "                                    spike_timepoints.keys()))\n",
    "    cell_map = dict([ (cell_key, i) for i, cell_key in enumerate(cell_keys) ])\n",
    "    spike_bins = np.zeros((ref_timepoints.shape[0], len(cell_keys)))\n",
    "    bins = np.concatenate((ref_timepoints[:], [(ref_timepoints[-1]*2)-ref_timepoints[-2]]))\n",
    "\n",
    "    for i, cell in enumerate(cell_keys):\n",
    "        spike_bins[:, i] = np.histogram(spike_timepoints[type_cast(cell)], bins)[0]\n",
    "        \n",
    "    datachunk = DataChunk(data=spike_bins, idx = ref_timepoints.idx, group=\"cell\")\n",
    "    datachunk.attrs[\"cell_map\"] = cell_map\n",
    "    return datachunk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def get_calcium_stack_lenghts(folder):\n",
    "    \"\"\"\n",
    "    Function to extract calcium stack lenghts from imageJ macro files associated to the stacks.\n",
    "    \n",
    "    params:\n",
    "        - folder: path of the folder containing the IJ macros files\n",
    "        \n",
    "    return:\n",
    "        - list of stack lenghts\n",
    "    \"\"\"\n",
    "    record_lenghts = []\n",
    "    pattern_nFrame = r\".*number=(\\d*) .*\"\n",
    "    for fn in glob.glob(folder+\"/*.txt\"):\n",
    "        with open(fn) as f:\n",
    "            line = f.readline()\n",
    "            record_lenghts.append(int(re.findall(pattern_nFrame, line)[0]))\n",
    "    return record_lenghts\n",
    "\n",
    "def twoP_dataChunks(ref_timepoints:DataChunk, frame_timepoints, len_epochs, *args):\n",
    "    \"\"\"\n",
    "    Factory function for two photon data. \n",
    "    \n",
    "    params:\n",
    "        - ref_timepoints: Reference timepoints to create the DataChunk\n",
    "        - frame_timepoints: List of frame timepoints for each sequence of two photon frame recorded.\n",
    "        - len_epochs: Lenght of the recorded epochs (<= than the corresponding frame_timepoints). Int or list\n",
    "        - args: matrices of all frames detected by CaImAn. (give as many as you want to synchronise)\n",
    "        \n",
    "    return:\n",
    "        - tuple containing the synchronised matrices in the order it was given\n",
    "    \"\"\"\n",
    "    assert len(args)>=1, \"no matrix to be synchronised was given\"\n",
    "    res_l = [[] for i in range(len(args))]\n",
    "    cursor = 0\n",
    "    if isinstance(len_epochs, int):\n",
    "        len_epochs = [len_epochs]\n",
    "    # For every recording block (defined by len_epochs),     \n",
    "    for i, len_epoch in enumerate(len_epochs):\n",
    "        start_idx = np.argmax(ref_timepoints>frame_timepoints[i][0])\n",
    "        stop_idx  = np.argmax(ref_timepoints>frame_timepoints[i][len_epoch-1])\n",
    "        for k, matrix in enumerate(args):\n",
    "            sub_mat = matrix.T[cursor:cursor+len_epoch]\n",
    "            \n",
    "            f = interpolate.interp1d(range(len_epoch), sub_mat, axis=0)\n",
    "            res_l[k].append(DataChunk(data=f(np.linspace(0,len_epoch-1,stop_idx-start_idx)), \n",
    "                                           idx=start_idx, \n",
    "                                           group=\"cell\"))\n",
    "        cursor += len_epoch\n",
    "    \n",
    "    return tuple(res_l)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Modelling utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def img_2d_fit(shape, param_d, f):\n",
    "    \"\"\"\n",
    "    Helper function to generate the 2D image of a fit.\n",
    "    \n",
    "    params:\n",
    "        - shape: Shape of the image in (y, x).\n",
    "        - param_d: Fit dictionnary.\n",
    "        - f: Function used of the fit.\n",
    "    \"\"\"\n",
    "    y_, x_ = shape\n",
    "    xy = np.meshgrid(range(x_), range(y_))\n",
    "    return f(xy, **param_d).reshape(y_, x_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def fill_nan(A):\n",
    "    \"\"\"\n",
    "    Fill nan values with interpolation. Credits to BRYAN WOODS@StackOverflow\n",
    "    \"\"\"\n",
    "    inds = np.arange(A.shape[0])\n",
    "    good = np.where(np.isfinite(A))\n",
    "    f = interpolate.interp1d(inds[good], A[good],bounds_error=False)\n",
    "    B = np.where(np.isfinite(A),A,f(inds))\n",
    "    return B"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Processing utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def stim_inten_norm(stim_inten):\n",
    "    \"\"\"\n",
    "    Normalize a stimulus with intensity in the 8bit range (0-255) to -1 to 1 range.\n",
    "    \"\"\"\n",
    "    stim_inten = stim_inten.astype(float)\n",
    "    stim_inten -= np.min(stim_inten)\n",
    "    stim_inten -= np.max(stim_inten)/2\n",
    "    stim_inten /= np.max(np.abs(stim_inten))\n",
    "    return np.round(stim_inten, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def group_direction_response(stim_prop, spike_counts, n_repeat, n_cond=32):\n",
    "    \"\"\"\n",
    "    Group the cells responses from shuffled grating stimulus repetitions. Retrieves a dictionnary\n",
    "    with a key for each condition.\n",
    "    \n",
    "    params:\n",
    "        - stim_prop: Grating array of shape (t, 3(size, angle, speed))\n",
    "        - spike_counts: Spike counts response of the cells of shape (t, n_cell)\n",
    "        - n_repeat: Number of repeat of each condition\n",
    "        - n_cond: Total number of condition (speed/size condition * n_angle)\n",
    "        \n",
    "    return:\n",
    "        - dictionnary of the spike counts for each condition (speed/size), with shape (n_angle, n_repeat, len, n_cell)\n",
    "    \"\"\"\n",
    "    \n",
    "    n_cell = spike_counts.shape[-1]\n",
    "    condition_repeat = stim_prop.reshape(n_repeat*n_cond,-1,3)[:,10,:] #Take the condition for each repeat\n",
    "    # We take it at the 10th frame in case of frame replacement during synchronisation \n",
    "    #(the 10th should be unchanged)\n",
    "    \n",
    "    #Reshape the spike response to (n_cond, len, n_cell)\n",
    "    spike_resh       = spike_counts.reshape(n_repeat*n_cond,-1,n_cell) \n",
    "\n",
    "    angles = np.unique(condition_repeat[:,1]) \n",
    "\n",
    "    data_dict = {}\n",
    "    for cond in np.unique(condition_repeat, axis=0):\n",
    "        spat_freq, angle, speed = tuple(cond)\n",
    "        idx_cond = np.argwhere(np.all(condition_repeat==cond, axis=1))[:,0]\n",
    "\n",
    "        cond_key = str(spat_freq)+\"@\"+str(round(speed,2))\n",
    "        if cond_key not in data_dict.keys():\n",
    "            data_dict[cond_key] = np.empty((len(angles), len(idx_cond), *spike_resh[0].shape))\n",
    "\n",
    "        idx_angle = np.where(angle==angles)[0][0]\n",
    "        data_dict[cond_key][idx_angle] = np.array([spike_resh[idx] for idx in idx_cond])\n",
    "    return data_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def group_chirp_bumps(stim_inten, spike_counts, n_repeat):\n",
    "    \"\"\"\n",
    "    Find the cells response to the OFF-ON-OFF initial parts of the chirps.\n",
    "    \n",
    "    params:\n",
    "        - stim_inten: Stimulus intensity array\n",
    "        - spike_counts: Spike counts array of shape (t, n_cell)\n",
    "        - n_repeat: Number of repetitions of the chirp stimulus\n",
    "        \n",
    "    return:\n",
    "        - Dictionnary of cells response to the different ON or OFF stimuli\n",
    "    \"\"\"\n",
    "    \n",
    "    repeat = stim_inten.reshape(n_repeat,-1)[0]\n",
    "    spike_counts = spike_counts.reshape(n_repeat,-1,spike_counts.shape[-1])\n",
    "    epoch_l = [0]\n",
    "    end_l = [len(repeat)]\n",
    "    i = 1\n",
    "    curr = repeat[0]\n",
    "\n",
    "    while True:\n",
    "        while repeat[i]==curr:\n",
    "            i+=1\n",
    "        epoch_l.append(i)\n",
    "        curr = repeat[i]\n",
    "        if curr==repeat[i+1]:\n",
    "            continue\n",
    "        else:\n",
    "            break\n",
    "\n",
    "    i = len(repeat)-2\n",
    "    curr = repeat[-1]\n",
    "\n",
    "    while True:\n",
    "        while repeat[i]==curr:\n",
    "            i-=1\n",
    "        end_l.insert(0,i)\n",
    "        curr = repeat[i]\n",
    "        if curr==repeat[i-1]:\n",
    "            continue\n",
    "        else:\n",
    "            break\n",
    "    slices = [slice(epoch_l[i-1],epoch_l[i]) for i in range(1,len(epoch_l))]\n",
    "    slices.extend([slice(end_l[i-1],end_l[i]) for i in range(1,len(end_l))])\n",
    "\n",
    "    res_d = {}\n",
    "    for slc in slices:\n",
    "        key = str(stim_inten[slc.start])+\"@\"+str(slc.start)\n",
    "        res_d[key] = spike_counts[:,slc]\n",
    "\n",
    "    return res_d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def get_repeat_corrected(stim_inten, spike_counts, n_repeats=10):\n",
    "    \"\"\"\n",
    "    Apply shifts (detected during synchro) to the chirp repetition.\n",
    "    \n",
    "    params:\n",
    "        - stim_inten: Stimulus DataChunk (containing the shifts and frame replacements info)\n",
    "        - spike_counts: Spike count matrix of shape (t, n_cell)\n",
    "        - n_repeats: Number of repeats of the chirp\n",
    "        \n",
    "    return:\n",
    "        - aligned cells response to stimulus, of shape (n_repeat, t, n_cell)\n",
    "        - Number of duplicated frame per repetition.\n",
    "    \"\"\"\n",
    "    def count_repl_in_range(fr_replaced, _range):\n",
    "        return sum([repl[0] in _range for repl in fr_replaced])\n",
    "    \n",
    "    signal_shifts     = stim_inten.attrs[\"signal_shifts\"]\n",
    "    frame_replacement = stim_inten.attrs[\"frame_replacement\"]\n",
    "    \n",
    "    spike_count_corr = spike_counts.copy()\n",
    "    shift_cursor = 0\n",
    "    prev_del = np.zeros((1, spike_counts.shape[1]))\n",
    "    for shift, direction in signal_shifts:\n",
    "        if direction==\"ins\":\n",
    "            spike_count_corr[shift+1:] = spike_count_corr[shift:-1]\n",
    "            prev_del = spike_count_corr[-1:]\n",
    "        else:\n",
    "            spike_count_corr[shift-1:-1] = spike_count_corr[shift:]\n",
    "            spike_count_corr[-1:] = prev_del\n",
    "            \n",
    "    len_epoch = len(stim_inten)//n_repeats\n",
    "    spike_counts_corrected = []\n",
    "    errors_per_repeat      = []\n",
    "    for i in range(n_repeats):\n",
    "        errors_per_repeat.append(count_repl_in_range(frame_replacement, range(len_epoch*i, len_epoch*(i+1))))\n",
    "        spike_counts_corrected.append(spike_count_corr[len_epoch*i:len_epoch*(i+1)])\n",
    "    return np.array(spike_counts_corrected), np.array(errors_per_repeat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def removeSlowDrift(traces, fps=60, window=80, percentile=8):\n",
    "    \"\"\"\n",
    "    Remove slow drifts from behavioral temporal traces such as locomotion speed obtained from the treadmill signal\n",
    "    or pupil size obtained from the eye_tracking signal, by extracting a specified percentile within moving window from the signal.\n",
    "    \n",
    "    params:\n",
    "        - traces: Behavioral temporal traces obtained from reM\n",
    "        - fps: Sampling rate\n",
    "        - window: Moving temporal window in seconds\n",
    "        - percentile: Percentile to be extracted within moving window\n",
    "        \n",
    "    return: \n",
    "        - Filtered temporal traces\n",
    "    \"\"\"\n",
    "    smoothed = np.zeros(len(traces))\n",
    "    n = round(window * fps)-1\n",
    "    if n%2 == 0:\n",
    "        n = n+1\n",
    "    \n",
    "    nBefore = math.floor((n-1)/2)\n",
    "    nAfter = n - nBefore - 1\n",
    "\n",
    "    for k in range(len(traces)):\n",
    "        idx1 = max(np.array([0,k-nBefore]))\n",
    "        idx2 = min(len(traces)-1,k+nAfter)\n",
    "        tmpTraces = traces[idx1:idx2]\n",
    "        smoothed[k] = np.percentile(tmpTraces, percentile)\n",
    "    \n",
    "    smoothed = savgol_filter(smoothed, n, 3)\n",
    "\n",
    "    filteredTraces = traces - smoothed\n",
    "    return filteredTraces"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def time_shift_test_corr(spike_counts, behav_signal, n_tests = 500, seed = 1): \n",
    "    \"\"\"\n",
    "    Compute the null distribution of correlation between behavioral signal and spiking signal with a time shift test.\n",
    "    \n",
    "    params:\n",
    "        - spike_counts: Array with spike counts for a specific neuron and data chunk from the reM\n",
    "        - behav_signal: Array with behavioral signal for a specific neuron and data chunk from the reM\n",
    "        - n_tests: number of used shifted signals to compute distribution\n",
    "        - seed: seed for numpy function random.randint\n",
    "        \n",
    "    return: \n",
    "        - null_dist_corr: Null distribution of correlation values \n",
    "    \"\"\"\n",
    "    \n",
    "    np.random.seed(seed)\n",
    "    \n",
    "    null_dist_corr=[]\n",
    "    for i in range(n_tests):\n",
    "        #Generate time-shifted behavioral test signal for shifts between 0.05*len(behav_signal) and len(behav_signal)\n",
    "        test_behav_signal = np.roll(behav_signal, np.random.randint(len(behav_signal)*0.05, len(behav_signal)))\n",
    "        # Compute Pearson's correlation with behavioral time-shifted test signal and spiking signal\n",
    "        null_dist_corr.append(scipy.stats.pearsonr(test_behav_signal, spike_counts)[0])    \n",
    "    \n",
    "    return null_dist_corr "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def cross_corr_with_lag(spike_counts, behav_signal, behav, conversion_factor_treadmill=None, removeslowdrift=True, fps=60, seconds=30):\n",
    "    \"\"\"\n",
    "    Compute cross-correlation with lag between behavioral signal and spiking signal.\n",
    "    Process signals, compute null distribution of the correlation with a time shift test and values .\n",
    "    Return cross-correlation array, null-distribution array and values for plotting.\n",
    "    \n",
    "    params:\n",
    "        - spike_counts: Array with spike counts for a specific neuron and data chunk from the reM\n",
    "        - behav_signal: Array with behavioral signal for a specific neuron and data chunk from the reM\n",
    "        - behav : String with name of behavioral signal to be analysed \n",
    "        - conversion_factor : The value to convert the treadmill signal into cm/s\n",
    "        - removeslowdrift: Boolean:\n",
    "            False - doesn't remove slow drifts from the signal\n",
    "            True - removes slow drifts by extracting a specified percentile within moving window from the signal\n",
    "        - fps: Sampling rate\n",
    "        - seconds: Window in seconds of the correlation lag\n",
    "    \n",
    "    return:\n",
    "        - crosscorr: Cross-correlation with lag array between behavioral signal and spiking signal\n",
    "        - corr_peak: Cross-correlation value at peak synchrony between behavioral signal and spiking signal\n",
    "        - p_value_peak: P-value of the peak cross-correlation value\n",
    "        - offset_peak: Temporal offset of the peak synchrony between behavioral signal and spiking signal in seconds\n",
    "        - null_dist_corr: Null distribution of correlation values (output of 'utils.cross_corr_with_lag')\n",
    "    \"\"\"   \n",
    "\n",
    "    if behav == \"treadmill\":\n",
    "        #Convert treadmill signal to running speed (cm/s)\n",
    "        behav_signal = behav_signal * conversion_factor_treadmill\n",
    "        behav_signal_filtered = gaussian_filter(abs(behav_signal), sigma=60)\n",
    "    else:\n",
    "        behav_signal_filtered = gaussian_filter(behav_signal, sigma=60)\n",
    "    \n",
    "    #Convolve signals with gaussian window of 1 second/60 frame\n",
    "    spike_counts_filtered = gaussian_filter(spike_counts, sigma=60)\n",
    "    \n",
    "    if removeslowdrift:\n",
    "        #Remove slow drifts from treadmill, pupil size and spiking signal\n",
    "        spike_counts_detrend = removeSlowDrift(spike_counts_filtered, fps=60, window=100, percentile=8)\n",
    "        behav_signal_detrend = removeSlowDrift(behav_signal_filtered, fps=60, window=100, percentile=8)\n",
    "    else:\n",
    "        spike_counts_detrend = spike_counts_filtered\n",
    "        behav_signal_detrend = behav_signal_filtered\n",
    "    \n",
    "    #Get null distribution for correlation between behav_signal and spike_counts signal\n",
    "    null_dist_corr = time_shift_test_corr(spike_counts_detrend, behav_signal_detrend, n_tests = 500)\n",
    "\n",
    "    #Compute cross-correlation with lag and values to plot\n",
    "    d1 = pd.Series(behav_signal_detrend)\n",
    "    d2 = pd.Series(spike_counts_detrend)\n",
    "    crosscorr = [d1.corr(d2.shift(lag)) for lag in range(-int(seconds*fps),int(seconds*fps+1))]\n",
    "    offset_peak = np.around((np.ceil(len(crosscorr)/2)-np.argmax(abs(np.array(crosscorr))))/fps, decimals=3)\n",
    "    corr_peak = np.max(abs(np.array(crosscorr)))\n",
    "    p_value_peak = round((100-scipy.stats.percentileofscore(abs(np.array(null_dist_corr)), abs(corr_peak), kind='strict'))/100,2)\n",
    "    \n",
    "    return crosscorr, corr_peak, p_value_peak, offset_peak, null_dist_corr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def get_inception_generator(imageset_folder, len_set=25, width=500, height=281):\n",
    "    \"\"\"\n",
    "    Return a function to obtain inception loop images from their index.\n",
    "    \n",
    "    params:\n",
    "        - imageset_folder: Path to the folder of the image sets\n",
    "        - len_set: Number of images concatenated per set\n",
    "        - width: image width\n",
    "    return: \n",
    "        - Function to obtain inception loop images from their index.\n",
    "    \"\"\"\n",
    "    imageset_l = []\n",
    "    paths = glob.glob(os.path.join(imageset_folder,\"*.jpg\"))\n",
    "    paths_sorted = sorted(paths, key=lambda i: int(os.path.splitext(os.path.basename(i))[0].split(\"_\")[-1]))\n",
    "    \n",
    "    for fn in paths_sorted: #Images accepted have the dimension (375,500)\n",
    "        image = np.array(Image.open(fn))\n",
    "        imageset_l.append(image)\n",
    "                           \n",
    "    def image_yield(idx):\n",
    "        if idx==-1:\n",
    "            return np.zeros((height, width))+128\n",
    "        set_idx = idx//25\n",
    "        img_idx = idx%25\n",
    "        return imageset_l[set_idx][:,width*img_idx:width*(img_idx+1), 1] #Returns a gray image\n",
    "    \n",
    "    return image_yield"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def group_omitted_epochs(stim_inten, spike_counts, n_fr_flash=4, n_fr_interflash=4, n_fr_isi=100):\n",
    "    \"\"\"\n",
    "    Group the cells reponse to the different omitted stimulus epochs conditions (n_flashes)\n",
    "    \n",
    "    params:\n",
    "        - stim_inten: The intensities of the omitted stimulus in shape (t)\n",
    "        - spike_counts: Spikes counts of the cells in shape (t, n_cell)\n",
    "        - n_fr_flash: Duration of a flash (ON flash during OFF baseline, OFF flash during ON baseline)\n",
    "        - n_fr_interflash: Number of frames between two flashes (during an epoch)\n",
    "        - n_fr_isi: Number of frames between two epochs\n",
    "    return:\n",
    "        - response_d_ON, response_d_OFF: Dictionnaries of the cells responses for different number of flashes repetions. Each contain an array of shape (n_cell, n_repeats, len_epoch+n_fr_isi).\n",
    "    \"\"\"\n",
    "    starts_ON    = []\n",
    "    stops_ON     = []\n",
    "    n_flashes_ON = []\n",
    "\n",
    "    counter   = 1\n",
    "    i         = 0\n",
    "    starts_ON.append(i)\n",
    "    while i < len(stim_inten)-(n_fr_flash+n_fr_interflash):\n",
    "        if stim_inten[i+(n_fr_flash*2+n_fr_interflash)]:\n",
    "            break\n",
    "        if stim_inten[i+(n_fr_flash+n_fr_interflash)]:\n",
    "            counter += 1\n",
    "            i+=(n_fr_flash+n_fr_interflash)\n",
    "        else:\n",
    "            stops_ON.append(i+(n_fr_flash+n_fr_interflash))\n",
    "            n_flashes_ON.append(counter)\n",
    "            counter = 1\n",
    "            i += (n_fr_flash+n_fr_interflash+n_fr_isi)\n",
    "            starts_ON.append(i)\n",
    "\n",
    "    #Switching to the omitted OFF\n",
    "    starts_OFF    = [starts_ON.pop()]\n",
    "    stops_OFF     = []\n",
    "    n_flashes_OFF = []\n",
    "    while i < len(stim_inten)-(n_fr_flash+n_fr_interflash):\n",
    "        if stim_inten[i+(n_fr_flash*2+n_fr_interflash)]==0:\n",
    "            counter += 1\n",
    "            i+=(n_fr_flash+n_fr_interflash)\n",
    "        else:\n",
    "            stops_OFF.append(i+(n_fr_flash+n_fr_interflash))\n",
    "            n_flashes_OFF.append(counter)\n",
    "            counter = 1\n",
    "            i += (n_fr_flash+n_fr_interflash+n_fr_isi)\n",
    "            starts_OFF.append(i)\n",
    "    starts_OFF.pop()\n",
    "\n",
    "    starts_ON     = np.array(starts_ON)\n",
    "    stops_ON      = np.array(stops_ON)\n",
    "    n_flashes_ON  = np.array(n_flashes_ON)\n",
    "    starts_OFF    = np.array(starts_OFF)\n",
    "    stops_OFF     = np.array(stops_OFF)\n",
    "    n_flashes_OFF = np.array(n_flashes_OFF)\n",
    "    \n",
    "    response_d_ON, response_d_OFF = {}, {}\n",
    "    for n_repeat in set(n_flashes_ON):\n",
    "        where_cond = np.where(n_flashes_ON==n_repeat)[0]\n",
    "        tmp        = np.array([spike_counts[start:stop+n_fr_isi] for start, stop in zip(starts_ON[where_cond], \n",
    "                                                                                  stops_ON[where_cond])])\n",
    "        response_d_ON[n_repeat] = np.transpose(tmp, (2, 0, 1))\n",
    "    for n_repeat in set(n_flashes_OFF):\n",
    "        where_cond = np.where(n_flashes_OFF==n_repeat)[0]\n",
    "        tmp        = np.array([spike_counts[start:stop+n_fr_isi] for start, stop in zip(starts_OFF[where_cond], \n",
    "                                                                                  stops_OFF[where_cond])])\n",
    "        response_d_OFF[n_repeat] = np.transpose(tmp, (2, 0, 1))\n",
    "    \n",
    "    return response_d_ON, response_d_OFF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plotting utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def get_shank_channels(channel_positions, shank_dist_th=80):\n",
    "    \"\"\"\n",
    "    Group the channels of a Buzsaki32 silicone probe into their shanks \n",
    "    from the channel position.\n",
    "    \n",
    "    params:\n",
    "        - channel_positions: List of channel positions\n",
    "        - shank_dist_th: Distance between channels in X to rule if on same shank or not\n",
    "        \n",
    "    return:\n",
    "        - array of grouped channel index of shape (n_shank(4), n_channel(8))\n",
    "    \"\"\"\n",
    "    found      = np.zeros(len(channel_positions))\n",
    "    shank_pos  = []\n",
    "    chann_pos  = []\n",
    "\n",
    "    while not np.all(found):\n",
    "        next_idx   = np.argmin(found)\n",
    "        next_pos   = channel_positions[next_idx][0] #getting the X position of the electrode\n",
    "        this_shank = np.where(np.abs(channel_positions[:,0]-next_pos)<shank_dist_th)[0]\n",
    "        chann_pos.append(this_shank)\n",
    "        shank_pos.append(next_pos)\n",
    "        found[this_shank] = 1\n",
    "\n",
    "    shanks_idx = np.zeros((len(shank_pos), len(this_shank)), dtype=int) - 1 #Initialize with -1 in case of channel missing\n",
    "    for i, order in enumerate(np.argsort(shank_pos)):\n",
    "        shanks_idx[i,:len(chann_pos[order])] = chann_pos[order]\n",
    "    return shanks_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def format_pval(pval, significant_figures=2):\n",
    "    \"\"\"\n",
    "    Helper function to format pvalue into string.\n",
    "    \"\"\"\n",
    "    return '{:g}'.format(float('{:.{p}g}'.format(pval, p=significant_figures)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def stim_recap_df(reM):\n",
    "    \"\"\"\n",
    "    Extract stimuli parameters (originally from the Database) to put them into a\n",
    "    dataframe that will be displayed in the recapitulation plot.\n",
    "    \n",
    "    params:\n",
    "        - reM: RecordMaster to extract stimuli parameters from\n",
    "        \n",
    "    return:\n",
    "        - dataframe with the stimuli important informations\n",
    "    \"\"\"\n",
    "    def parse_stim(stim_dc):\n",
    "        param_d = {}\n",
    "        param_d[\"hash\"]        = stim_dc.attrs[\"md5\"][:10] #the first 10 letters are more than enough\n",
    "        param_d[\"n frames\"]    = len(stim_dc)\n",
    "        param_d[\"stimulus\"]    = stim_dc.attrs[\"name\"]\n",
    "\n",
    "        if stim_dc.attrs[\"name\"] in [\"checkerboard\", \"fullfield_flicker\", \"flickering_bars\", \"flickering_bars_pr\"]:\n",
    "            param_d[\"frequency\"] = stim_dc.attrs[\"refresh_rate\"]\n",
    "        elif stim_dc.attrs[\"name\"] in [\"chirp_am\",\"chirp_fm\",\"chirp_freq_epoch\", \"chirp_co\"]:\n",
    "            param_d[\"n ON\"]      = int(stim_dc.attrs[\"tSteadyON_s\"]*60)\n",
    "            param_d[\"n OFF\"]     = int(stim_dc.attrs[\"tSteadyOFF_s\"]*60)\n",
    "            param_d[\"n repeats\"] = int(stim_dc.attrs[\"n_repeat\"])\n",
    "            if stim_dc.attrs[\"name\"] in [\"chirp_am\",\"chirp_co\"]:\n",
    "                param_d[\"frequency\"] = stim_dc.attrs[\"contrast_frequency\"]\n",
    "            elif stim_dc.attrs[\"name\"]==\"chirp_fm\":\n",
    "                param_d[\"frequency\"] = stim_dc.attrs[\"max_frequency\"]\n",
    "            elif stim_dc.attrs[\"name\"]==\"chirp_freq_epoch\":\n",
    "                param_d[\"frequency\"] = str([round(60/nfr,2) for nfr in dc.attrs[\"n_frame_cycle\"]])\n",
    "        elif stim_dc.attrs[\"name\"] in [\"fullfield_color_mix\"]:\n",
    "            param_d[\"n ON\"]      = int(stim_dc.attrs[\"n_frame_on\"])\n",
    "            param_d[\"n OFF\"]     = int(stim_dc.attrs[\"n_frame_off\"])\n",
    "            param_d[\"n repeats\"] = int(stim_dc.attrs[\"n_repeat\"])\n",
    "        elif stim_dc.attrs[\"name\"]==\"moving_gratings\":\n",
    "            param_d[\"n repeats\"]           = stim_dc.attrs[\"n_repeat\"]\n",
    "            param_d[\"n ON\"]                = stim_dc.attrs[\"n_frame_on\"]\n",
    "            param_d[\"n OFF\"]               = stim_dc.attrs[\"n_frame_off\"]\n",
    "            param_d[\"speeds\"]              = stim_dc.attrs[\"speeds\"]\n",
    "            param_d[\"spatial frequencies\"] = stim_dc.attrs[\"spatial_frequencies\"]\n",
    "            \n",
    "        if \"frame_replacement\" in stim_dc.attrs:\n",
    "            param_d[\"total drop\"] = len(stim_dc.attrs[\"frame_replacement\"])\n",
    "        if \"signal_shifts\" in stim_dc.attrs:\n",
    "            shift = 0\n",
    "            for _, which_shift in stim_dc.attrs[\"signal_shifts\"]:\n",
    "                if which_shift==\"ins\":\n",
    "                    shift += 1\n",
    "                elif which_shift==\"del\":\n",
    "                    shift -= 1\n",
    "            param_d[\"total shift\"] = shift\n",
    "\n",
    "        return param_d\n",
    "\n",
    "    series = []\n",
    "    for seq in reM._sequences:\n",
    "        for k, dc_l in seq:\n",
    "            dc = dc_l[0]\n",
    "            if dc.group == \"stim\":\n",
    "                series.append(parse_stim(dc))\n",
    "    df = pd.DataFrame.from_records(series, columns=[\"stimulus\", \"hash\", \"n frames\", \"n repeats\",\n",
    "                               \"frequency\", \"n ON\", \"n OFF\", \"speeds\", \"spatial frequencies\",\n",
    "                              \"total shift\", \"total drop\"])\n",
    "    df = df.fillna(\"\")\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Converted 00_core.ipynb.\n",
      "Converted 01_utils.ipynb.\n",
      "Converted 02_processing.ipynb.\n",
      "Converted 03_modelling.ipynb.\n",
      "Converted 04_plotting.ipynb.\n",
      "Converted 05_database.ipynb.\n",
      "Converted 06_eyetrack.ipynb.\n",
      "Converted 10_synchro.io.ipynb.\n",
      "Converted 11_synchro.extracting.ipynb.\n",
      "Converted 12_synchro.processing.ipynb.\n",
      "Converted 13_leddome.ipynb.\n",
      "Converted 99_testdata.ipynb.\n",
      "Converted index.ipynb.\n"
     ]
    }
   ],
   "source": [
    "#hide\n",
    "from nbdev.export import *\n",
    "notebook2script()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tor",
   "language": "python",
   "name": "tor"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}