diff --git a/allensdk/brain_observatory/behavior/behavior_session.py b/allensdk/brain_observatory/behavior/behavior_session.py index 413fdabf17..01fe51f0e1 100644 --- a/allensdk/brain_observatory/behavior/behavior_session.py +++ b/allensdk/brain_observatory/behavior/behavior_session.py @@ -1484,17 +1484,25 @@ def _read_data_from_stimulus_file( else: stimuli = None - trials = cls._read_trials( + stimulus_file = stimulus_file_lookup.behavior_stimulus_file.data + if "trial_log" in stimulus_file["items"]["behavior"]: + trials = cls._read_trials( stimulus_file_lookup=stimulus_file_lookup, sync_file=sync_file, monitor_delay=monitor_delay, licks=licks, rewards=rewards, - ) + ) + else: + trials = None - task_parameters = TaskParameters.from_stimulus_file( - stimulus_file=stimulus_file_lookup.behavior_stimulus_file - ) + + if "DoC" in stimulus_file["items"]["behavior"]["config"]: + task_parameters = TaskParameters.from_stimulus_file( + stimulus_file=stimulus_file_lookup.behavior_stimulus_file + ) + else: + task_parameters = None return ( session_stimulus_timestamps.subtract_monitor_delay(), diff --git a/allensdk/brain_observatory/behavior/data_files/stimulus_file.py b/allensdk/brain_observatory/behavior/data_files/stimulus_file.py index 56241d211e..1a9366f1ed 100644 --- a/allensdk/brain_observatory/behavior/data_files/stimulus_file.py +++ b/allensdk/brain_observatory/behavior/data_files/stimulus_file.py @@ -156,8 +156,10 @@ def date_of_acquisition(self) -> datetime.datetime: raise KeyError( "No 'start_time' listed in pickle file " f"{self.filepath}") - - return copy.deepcopy(self.data['start_time']) + start_time = self.data['start_time'] + if not isinstance(start_time, datetime.datetime): + start_time = datetime.datetime.fromtimestamp(start_time) + return copy.deepcopy(start_time) @property def session_type(self) -> str: @@ -239,10 +241,12 @@ def stimuli(self) -> Dict[str, Tuple[str, Union[str, int], int, int]]: return self.data['items']['behavior']['stimuli'] def validate(self) -> "BehaviorStimulusFile": - if 'items' not in self.data or 'behavior' not in self.data['items']: - raise MalformedStimulusFileError( - f'Expected to find key "behavior" in "items" dict. ' + if 'items' not in self.data or ('behavior' not in self.data['items'] and 'foraging' not in self.data['items']): + raise MalformedStimulusFileError( + f'Expected to find keys "behavior" or "foraging" in "items" dict. ' f'Found {self.data["items"].keys()}') + elif 'behavior' not in self.data['items']: + self.data['items']['behavior'] = self.data['items']['foraging'] return self diff --git a/allensdk/brain_observatory/behavior/data_objects/rewards.py b/allensdk/brain_observatory/behavior/data_objects/rewards.py index 1dcc0a745c..414d40f138 100644 --- a/allensdk/brain_observatory/behavior/data_objects/rewards.py +++ b/allensdk/brain_observatory/behavior/data_objects/rewards.py @@ -36,19 +36,22 @@ def from_stimulus_file( raise RuntimeError(msg) data = stimulus_file.data + if "trial_log" in data["items"]["behavior"]: + trial_df = pd.DataFrame(data["items"]["behavior"]["trial_log"]) + rewards_dict = {"volume": [], "timestamps": [], "auto_rewarded": []} + for idx, trial in trial_df.iterrows(): + rewards = trial["rewards"] + # as i write this there can only ever be one reward per trial + if rewards: + rewards_dict["volume"].append(rewards[0][0]) + rewards_dict["timestamps"].append( + stimulus_timestamps.value[rewards[0][2]]) + auto_rwrd = trial["trial_params"]["auto_reward"] + rewards_dict["auto_rewarded"].append(auto_rwrd) - trial_df = pd.DataFrame(data["items"]["behavior"]["trial_log"]) - rewards_dict = {"volume": [], "timestamps": [], "auto_rewarded": []} - for idx, trial in trial_df.iterrows(): - rewards = trial["rewards"] - # as i write this there can only ever be one reward per trial - if rewards: - rewards_dict["volume"].append(rewards[0][0]) - rewards_dict["timestamps"].append( - stimulus_timestamps.value[rewards[0][2]]) - auto_rwrd = trial["trial_params"]["auto_reward"] - rewards_dict["auto_rewarded"].append(auto_rwrd) - + else: + rewards_dict = data["items"]["behavior"]["rewards"] + df = pd.DataFrame(rewards_dict) return cls(rewards=df) diff --git a/allensdk/brain_observatory/behavior/data_objects/stimuli/presentations.py b/allensdk/brain_observatory/behavior/data_objects/stimuli/presentations.py index 91cc8a99ad..fdd299f11d 100644 --- a/allensdk/brain_observatory/behavior/data_objects/stimuli/presentations.py +++ b/allensdk/brain_observatory/behavior/data_objects/stimuli/presentations.py @@ -205,85 +205,88 @@ def from_stimulus_file( # 1. Nulls in `image_name` should be "gratings_" # 2. Gratings are only present (or need to be fixed) when all # values for `image_name` are null. - if pd.isnull(raw_stim_pres_df["image_name"]).all(): - if ~pd.isnull(raw_stim_pres_df["orientation"]).all(): - raw_stim_pres_df["image_name"] = ( - raw_stim_pres_df["orientation"] - .apply(lambda x: f"gratings_{x}")) - else: - raise ValueError("All values for 'orientation' and " - "'image_name are null.") - - stimulus_metadata_df = get_stimulus_metadata(data) - - idx_name = raw_stim_pres_df.index.name - stimulus_index_df = ( - raw_stim_pres_df - .reset_index() - .merge(stimulus_metadata_df.reset_index(), - on=["image_name"]) - .set_index(idx_name)) - stimulus_index_df = ( - stimulus_index_df[["image_set", "image_index", "start_time", - "phase", "spatial_frequency"]] - .rename(columns={"start_time": "timestamps"}) - .sort_index() - .set_index("timestamps", drop=True)) - stim_pres_df = raw_stim_pres_df.merge( - stimulus_index_df, left_on="start_time", right_index=True, - how="left") - if len(raw_stim_pres_df) != len(stim_pres_df): - raise ValueError("Length of `stim_pres_df` should not change after" - f" merge; was {len(raw_stim_pres_df)}, now " - f" {len(stim_pres_df)}.") - - stim_pres_df['is_change'] = is_change_event( - stimulus_presentations=stim_pres_df) - stim_pres_df['flashes_since_change'] = get_flashes_since_change( - stimulus_presentations=stim_pres_df) - - # Sort columns then drop columns which contain only all NaN values - stim_pres_df = \ - stim_pres_df[sorted(stim_pres_df)].dropna(axis=1, how='all') - if limit_to_images is not None: + if "image_name" in raw_stim_pres_df: + if pd.isnull(raw_stim_pres_df["image_name"]).all(): + if ~pd.isnull(raw_stim_pres_df["orientation"]).all(): + raw_stim_pres_df["image_name"] = ( + raw_stim_pres_df["orientation"] + .apply(lambda x: f"gratings_{x}")) + else: + raise ValueError("All values for 'orientation' and " + "'image_name are null.") + + stimulus_metadata_df = get_stimulus_metadata(data) + + idx_name = raw_stim_pres_df.index.name + stimulus_index_df = ( + raw_stim_pres_df + .reset_index() + .merge(stimulus_metadata_df.reset_index(), + on=["image_name"]) + .set_index(idx_name)) + stimulus_index_df = ( + stimulus_index_df[["image_set", "image_index", "start_time", + "phase", "spatial_frequency"]] + .rename(columns={"start_time": "timestamps"}) + .sort_index() + .set_index("timestamps", drop=True)) + stim_pres_df = raw_stim_pres_df.merge( + stimulus_index_df, left_on="start_time", right_index=True, + how="left") + if len(raw_stim_pres_df) != len(stim_pres_df): + raise ValueError("Length of `stim_pres_df` should not change after" + f" merge; was {len(raw_stim_pres_df)}, now " + f" {len(stim_pres_df)}.") + + stim_pres_df['is_change'] = is_change_event( + stimulus_presentations=stim_pres_df) + stim_pres_df['flashes_since_change'] = get_flashes_since_change( + stimulus_presentations=stim_pres_df) + + # Sort columns then drop columns which contain only all NaN values stim_pres_df = \ - stim_pres_df[stim_pres_df['image_name'].isin(limit_to_images)] - stim_pres_df.index = pd.Int64Index( - range(stim_pres_df.shape[0]), name=stim_pres_df.index.name) - - stim_pres_df['stimulus_block'] = 0 - # Match the Ecephys VBN stimulus name convention. - try: - stim_pres_df['stimulus_name'] = Path( - stimulus_file.stimuli['images']['image_set']).\ - stem.split('.')[0] - except KeyError: - # if we can't find the images key in the stimuli, check for the - # name ``grating`` as the stimulus. If not add generic - # ``behavior``. - if 'grating' in stimulus_file.stimuli.keys(): - stim_pres_df['stimulus_name'] = 'grating' - else: - stim_pres_df['stimulus_name'] = 'behavior' - - stim_pres_df = fix_omitted_end_frame(stim_pres_df) - - cls._add_is_image_novel(stimulus_presentations=stim_pres_df, - behavior_session_id=behavior_session_id) - - has_fingerprint_stimulus = \ - 'fingerprint' in stimulus_file.data['items']['behavior']['items'] - if has_fingerprint_stimulus: - stim_pres_df = cls._add_fingerprint_stimulus( - stimulus_presentations=stim_pres_df, - stimulus_file=stimulus_file, - stimulus_timestamps=stimulus_timestamps + stim_pres_df[sorted(stim_pres_df)].dropna(axis=1, how='all') + if limit_to_images is not None: + stim_pres_df = \ + stim_pres_df[stim_pres_df['image_name'].isin(limit_to_images)] + stim_pres_df.index = pd.Int64Index( + range(stim_pres_df.shape[0]), name=stim_pres_df.index.name) + + stim_pres_df['stimulus_block'] = 0 + # Match the Ecephys VBN stimulus name convention. + try: + stim_pres_df['stimulus_name'] = Path( + stimulus_file.stimuli['images']['image_set']).\ + stem.split('.')[0] + except KeyError: + # if we can't find the images key in the stimuli, check for the + # name ``grating`` as the stimulus. If not add generic + # ``behavior``. + if 'grating' in stimulus_file.stimuli.keys(): + stim_pres_df['stimulus_name'] = 'grating' + else: + stim_pres_df['stimulus_name'] = 'behavior' + + stim_pres_df = fix_omitted_end_frame(stim_pres_df) + + cls._add_is_image_novel(stimulus_presentations=stim_pres_df, + behavior_session_id=behavior_session_id) + + has_fingerprint_stimulus = \ + 'fingerprint' in stimulus_file.data['items']['behavior']['items'] + if has_fingerprint_stimulus: + stim_pres_df = cls._add_fingerprint_stimulus( + stimulus_presentations=stim_pres_df, + stimulus_file=stimulus_file, + stimulus_timestamps=stimulus_timestamps + ) + stim_pres_df = cls._postprocess( + presentations=stim_pres_df, + fill_omitted_values=fill_omitted_values, + coerce_bool_to_boolean=True ) - stim_pres_df = cls._postprocess( - presentations=stim_pres_df, - fill_omitted_values=fill_omitted_values, - coerce_bool_to_boolean=True - ) + else: + stim_pres_df = raw_stim_pres_df return Presentations(presentations=stim_pres_df, column_list=column_list) diff --git a/allensdk/brain_observatory/behavior/stimulus_processing.py b/allensdk/brain_observatory/behavior/stimulus_processing.py index 1aedb60d39..e937733209 100644 --- a/allensdk/brain_observatory/behavior/stimulus_processing.py +++ b/allensdk/brain_observatory/behavior/stimulus_processing.py @@ -2,6 +2,7 @@ import warnings from typing import Dict, List, Tuple, Union, Optional, Set +import os import numpy as np import pandas as pd @@ -433,81 +434,180 @@ def get_visual_stimuli_df( that were displayed with their frame, end_frame, start_time, and duration """ - - stimuli = data['items']['behavior']['stimuli'] - n_frames = len(time) - visual_stimuli_data = [] - for stim_dict in stimuli.values(): - for idx, (attr_name, attr_value, _, frame) in \ - enumerate(stim_dict["set_log"]): - orientation = attr_value if attr_name.lower() == "ori" else np.nan - image_name = attr_value if attr_name.lower() == "image" else np.nan - - stimulus_epoch = _get_stimulus_epoch( - stim_dict["set_log"], - idx, - frame, - n_frames, - ) - draw_epochs = _get_draw_epochs( - stim_dict["draw_log"], - *stimulus_epoch - ) - - for epoch_start, epoch_end in draw_epochs: - - visual_stimuli_data.append({ - "orientation": orientation, - "image_name": image_name, - "frame": epoch_start, - "end_frame": epoch_end, - "time": time[epoch_start], - "duration": time[epoch_end] - time[epoch_start], - # this will always work because an epoch - # will never occur near the end of time - "omitted": False - }) - - visual_stimuli_df = pd.DataFrame(data=visual_stimuli_data) - - # Add omitted flash info: - try: - omitted_flash_frame_log = \ - data['items']['behavior']['omitted_flash_frame_log'] - except KeyError: - # For sessions for which there were no omitted flashes - omitted_flash_frame_log = dict() - - omitted_flash_list = [] - for _, omitted_flash_frames in omitted_flash_frame_log.items(): - stim_frames = visual_stimuli_df['frame'].values - omitted_flash_frames = np.array(omitted_flash_frames) - - # Test offsets of omitted flash frames - # to see if they are in the stim log - offsets = np.arange(-3, 4) - offset_arr = np.add( - np.repeat(omitted_flash_frames[:, np.newaxis], - offsets.shape[0], axis=1), - offsets) - matched_any_offset = np.any(np.isin(offset_arr, stim_frames), axis=1) - - # Remove omitted flashes that also exist in the stimulus log - was_true_omitted = np.logical_not(matched_any_offset) # bool - omitted_flash_frames_to_keep = omitted_flash_frames[was_true_omitted] - - # Have to remove frames that are double-counted in omitted log - omitted_flash_list += list(np.unique(omitted_flash_frames_to_keep)) - - omitted = np.ones_like(omitted_flash_list).astype(bool) - time = [time[fi] for fi in omitted_flash_list] - omitted_df = pd.DataFrame({'omitted': omitted, - 'frame': omitted_flash_list, - 'time': time, - 'image_name': 'omitted'}) - - df = pd.concat((visual_stimuli_df, omitted_df), - sort=False).sort_values('frame').reset_index() + if data['items']['behavior']['params']['stage'] == 'STAGE_0': + n_frames = len(time) + stimuli = data['stimuli'] + pre_blank_sec = int(data['pre_blank_sec']) + stim_table = [] + for stim_index, stimulus in enumerate(stimuli): + stim_path = data['stimuli'][stim_index]['movie_path'] + stim_path = stim_path.replace('\\','/') + movie_name = os.path.split(stim_path)[1].replace('.npy','') + + display_sequence = np.array(stimulus['display_sequence'])[0] + sweep_order = stimulus['sweep_order'][:int(np.diff(display_sequence*30))] + Starts = (np.argwhere(np.array(sweep_order)==0)/30).astype(int)[:,0] + Ends = Starts.copy() + Ends[:-1] = Starts[1:] + Ends[-1] = int(len(sweep_order)/30) + num_sweeps = len(Starts) + + for i_repeat, start_frame in enumerate(Starts): + end_frame = Ends[i_repeat] + _display_sequence = np.array([np.arange(display_sequence[0],display_sequence[1]+1)[start_frame],np.arange(display_sequence[0],display_sequence[1]+1)[end_frame]]) + + _display_sequence += pre_blank_sec + _display_sequence *= int(data['fps']) # in stimulus frames + + stim_table.append({ + "movie_name": movie_name, + "repeat": i_repeat, + "frame": _display_sequence[0], + "end_frame": _display_sequence[1], + "time": time[_display_sequence[0]], + "duration": time[_display_sequence[1]] - time[_display_sequence[0]], + # this will always work because an epoch + # will never occur near the end of time + }) + df = pd.DataFrame(data=stim_table).sort_values('frame').reset_index() + + + elif data['items']['behavior']['params']['stage'] == 'STAGE_1': + + stimuli = data['stimuli'] + for stim_index, stim_data in enumerate(stimuli): + + stim_path = stim_data['stim_path'] # extract stimulus name from its path + stim_path = stim_path.replace('\\','/') + stimulus_name = os.path.split(stim_path)[1].replace('.stim','') + + stimulus = stimuli[stim_index] + + # compute display sequence + display_sequence = np.array(stimulus['display_sequence']) + pre_blank_sec = int(data['pre_blank_sec']) + fps = data['fps'] + display_sequence += pre_blank_sec + display_sequence *= int(data['fps']) # in stimulus frames + + sweep_frames = stimulus['sweep_frames'] + timing_table = pd.DataFrame(np.array(sweep_frames).astype(int), columns=('start', 'end')) + timing_table['dif'] = timing_table['end'] - timing_table['start'] + + timing_table.start += display_sequence[0, 0] + for seg in range(len(display_sequence) - 1): + for index, row in timing_table.iterrows(): + if row.start >= display_sequence[seg, 1]: + timing_table.start[index] = timing_table.start[index] - display_sequence[seg, 1]+ display_sequence[seg + 1, 0] + timing_table.end = timing_table.start + timing_table.dif + + timing_table = timing_table[timing_table.end <= display_sequence[-1, 1]] + timing_table = timing_table[timing_table.start <= display_sequence[-1, 1]] + + stim_attributes = stimulus['dimnames'] # ['Contrast', 'TF', 'SF', 'Ori'] + stim_table = pd.DataFrame(np.column_stack((timing_table['start'],timing_table['end'],time[timing_table['start']],time[timing_table['end']]-time[timing_table['start']])), columns=('frame', 'end_frame', 'time', 'duration')) + + sweep_table = stimulus['sweep_table'] + sweep_order = stimulus['sweep_order'] + + stim_table['stimulus_name'] = [stimulus_name]*len(stim_table) + unique_conditions = np.unique(sweep_order) + num_sweeps = len(sweep_order) + for i_attribure, stim_attribute in enumerate(stim_attributes): + attribute_by_sweep = np.zeros((num_sweeps,)) + attribute_by_sweep[:] = np.NaN + for i_condition, condition in enumerate(unique_conditions): + sweeps_with_condition = np.argwhere(sweep_order == condition)[:, 0] + if condition > -1: # blank sweep is -1 + if stim_attribute.find('Size')!=-1: + attribute_by_sweep[sweeps_with_condition] = sweep_table[condition][i_attribure][0] + else: + attribute_by_sweep[sweeps_with_condition] = sweep_table[condition][i_attribure] + + stim_table[stim_attribute] = attribute_by_sweep[:len(stim_table)] + + if stim_index == 0: + df = stim_table + else: + df = pd.concat([df, stim_table], ignore_index = True,sort = False) + + df = df.rename(columns={'Ori': 'direction', 'Contrast': 'contrast'}) + df = df.sort_values('frame').reset_index() + else: + stimuli = data['items']['behavior']['stimuli'] + n_frames = len(time) + visual_stimuli_data = [] + for stim_dict in stimuli.values(): + for idx, (attr_name, attr_value, _, frame) in \ + enumerate(stim_dict["set_log"]): + orientation = attr_value if attr_name.lower() == "ori" else np.nan + image_name = attr_value if attr_name.lower() == "image" else np.nan + + stimulus_epoch = _get_stimulus_epoch( + stim_dict["set_log"], + idx, + frame, + n_frames, + ) + draw_epochs = _get_draw_epochs( + stim_dict["draw_log"], + *stimulus_epoch + ) + + for epoch_start, epoch_end in draw_epochs: + + visual_stimuli_data.append({ + "orientation": orientation, + "image_name": image_name, + "frame": epoch_start, + "end_frame": epoch_end, + "time": time[epoch_start], + "duration": time[epoch_end] - time[epoch_start], + # this will always work because an epoch + # will never occur near the end of time + "omitted": False + }) + + visual_stimuli_df = pd.DataFrame(data=visual_stimuli_data) + + # Add omitted flash info: + try: + omitted_flash_frame_log = \ + data['items']['behavior']['omitted_flash_frame_log'] + except KeyError: + # For sessions for which there were no omitted flashes + omitted_flash_frame_log = dict() + + omitted_flash_list = [] + for _, omitted_flash_frames in omitted_flash_frame_log.items(): + stim_frames = visual_stimuli_df['frame'].values + omitted_flash_frames = np.array(omitted_flash_frames) + + # Test offsets of omitted flash frames + # to see if they are in the stim log + offsets = np.arange(-3, 4) + offset_arr = np.add( + np.repeat(omitted_flash_frames[:, np.newaxis], + offsets.shape[0], axis=1), + offsets) + matched_any_offset = np.any(np.isin(offset_arr, stim_frames), axis=1) + + # Remove omitted flashes that also exist in the stimulus log + was_true_omitted = np.logical_not(matched_any_offset) # bool + omitted_flash_frames_to_keep = omitted_flash_frames[was_true_omitted] + + # Have to remove frames that are double-counted in omitted log + omitted_flash_list += list(np.unique(omitted_flash_frames_to_keep)) + + omitted = np.ones_like(omitted_flash_list).astype(bool) + time = [time[fi] for fi in omitted_flash_list] + omitted_df = pd.DataFrame({'omitted': omitted, + 'frame': omitted_flash_list, + 'time': time, + 'image_name': 'omitted'}) + + df = pd.concat((visual_stimuli_df, omitted_df), + sort=False).sort_values('frame').reset_index() return df diff --git a/allensdk/brain_observatory/gad2_ophys_processing.py b/allensdk/brain_observatory/gad2_ophys_processing.py new file mode 100644 index 0000000000..ed5f14bd60 --- /dev/null +++ b/allensdk/brain_observatory/gad2_ophys_processing.py @@ -0,0 +1,84 @@ +from scipy.ndimage import percentile_filter +from tqdm import tqdm +from multiprocessing import Pool +import numpy as np +def nanmedian_filter(x,filter_length): + """ 1D median filtering with np.nanmedian + Parameters + ---------- + x: 1D trace to be filtered + filter_length: length of the filter + + Return + ------ + filtered_trace + """ + half_length = int(filter_length/2) + # Create 'reflect' traces at the extrema + temp_trace = np.concatenate((np.flip(x[:half_length]), x, np.flip(x[-half_length:]))) + filtered_trace = np.zeros_like(x) + for i in range(len(x)): + filtered_trace[i] = np.nanmedian(temp_trace[i:i+filter_length]) + return filtered_trace + +def noise_std(x, filter_length, positive_peak_scale=1.5, + outlier_std_scale=2.5, GAUSSIAN_MAD_STD_SCALE = 1.4826): + """Robust estimate of the standard deviation of the trace noise.""" + if any(np.isnan(x)): + return np.NaN + x = x - nanmedian_filter(x, filter_length) + # first pass removing big pos peak outliers + x = x[x < positive_peak_scale*np.abs(x.min())] + + rstd = GAUSSIAN_MAD_STD_SCALE*np.median(np.abs(x - np.median(x))) + # second pass removing remaining pos and neg peak outliers + x = x[abs(x) < outlier_std_scale*rstd] + x = GAUSSIAN_MAD_STD_SCALE*np.median(np.abs(x - np.median(x))) + return x + +def compute_dff_single_trace(roi_ind,corrected_trace, + frames_per_sec, inactive_kernel_size = 30, + inactive_percentile = 10): + + long_filter_length = int(round(frames_per_sec*60*inactive_kernel_size)) + short_filter_length = int(round(frames_per_sec*60*10)) # 10 min + + + noise_sd = noise_std(corrected_trace, filter_length=int(round(frames_per_sec * 3.33))) # 3.33 s is fixed + # 10th percentile "low_baseline" + low_baseline = percentile_filter(corrected_trace, size=long_filter_length, percentile=inactive_percentile, mode='reflect') + # Create trace using inactive frames only, by replacing signals in "active frames" with NaN + active_frame = np.where(corrected_trace > (low_baseline + 3 * noise_sd))[0] # type: ignore + inactive_trace = corrected_trace.copy() + for i in active_frame: + inactive_trace[i] = np.nan + # Calculate baseline using median filter + baseline_new = nanmedian_filter(inactive_trace, short_filter_length) + # Calculate DFF + + dff_trace = ((corrected_trace - baseline_new)/ np.maximum(baseline_new, noise_sd)) + dff_trace[np.argwhere(np.isnan(dff_trace))] = 0 + return dff_trace + +def compute_dff(corrected_fluorescence_traces, ophys_timestamps): + + frame_per_sec = 1/np.nanmean((np.diff(ophys_timestamps))) + dff_traces = corrected_fluorescence_traces.copy() + corrected = np.array(dff_traces['corrected_fluorescence']) + dff = corrected.copy() + + cores = 15 # cpu cores to be used, optimized so it won't max out the memory + + for i_core in tqdm(np.arange(0,np.shape(corrected)[0],cores), desc=''): + args = [(roi_ind, corrected[roi_ind], frame_per_sec) for roi_ind in np.arange(i_core,np.min((i_core+cores,np.shape(corrected)[0])))] + + with Pool() as pool: + tmp = pool.starmap(compute_dff_single_trace, args) + + dff[i_core:np.min((i_core+cores,np.shape(corrected)[0]))] = tmp + dff_traces.pop('RMSE') + dff_traces.pop('r') + dff_traces.pop('corrected_fluorescence') + dff_traces['dff'] = dff + + return(dff_traces) \ No newline at end of file diff --git a/allensdk/brain_observatory/get_tables.py b/allensdk/brain_observatory/get_tables.py new file mode 100644 index 0000000000..ef68e8e248 --- /dev/null +++ b/allensdk/brain_observatory/get_tables.py @@ -0,0 +1,109 @@ +#%% +import os +import pandas as pd +from tqdm import tqdm +from allensdk.brain_observatory.behavior.behavior_project_cache.project_apis\ + .data_io import \ + BehaviorProjectLimsApi +from allensdk.brain_observatory.behavior.data_files import BehaviorStimulusFile +from allensdk.brain_observatory.behavior.data_objects import BehaviorSessionId +from allensdk.internal.api import db_connection_creator +from allensdk.core.auth_config import LIMS_DB_CREDENTIAL_MAP +from allensdk.brain_observatory.behavior.data_objects.metadata.subject_metadata.subject_metadata import SubjectMetadata +#%% + + +def get_all_tables(project_code: str): + lims_db = db_connection_creator(fallback_credentials=LIMS_DB_CREDENTIAL_MAP) + api = BehaviorProjectLimsApi.default(passed_only=False) + session_table = api.get_ophys_session_table().reset_index() + session_df = session_table[session_table.project_code==project_code].reset_index(drop=True) + behavior_session_ids = session_df['behavior_session_id'].values + main_paths = [] + session_types = [] + mouse_ids = [] + cre_lines = [] + full_genotypes = [] + reporter_lines = [] + desc = 'loading stimulus pkl files' + + for i_session in tqdm (range(len(behavior_session_ids)), desc=desc): + ophys_experiment_id = session_df['ophys_experiment_id'][i_session][0] + behavior_session_id = BehaviorSessionId.from_lims( + ophys_experiment_id=ophys_experiment_id, db=lims_db) + subject_metadata = SubjectMetadata.from_lims( + behavior_session_id=behavior_session_id,lims_db=lims_db) + + stimulus_file = BehaviorStimulusFile.from_lims( + db=lims_db, behavior_session_id=behavior_session_id.value) + session_types.append(stimulus_file.session_type) + main_paths.append(os.sep.join(stimulus_file.filepath.split(os.sep)[:-3])) + mouse_ids.append(int(subject_metadata.mouse_id)) + cre_lines.append(subject_metadata.cre_line) + full_genotypes.append(subject_metadata.full_genotype) + reporter_lines.append(subject_metadata.reporter_line) + + session_df['main_path'] = main_paths + session_df['session_type'] = session_types + session_df['mouse_id'] = mouse_ids + session_df['cre_line'] = cre_lines + session_df['full_genotype'] = full_genotypes + session_df['reporter_line'] = reporter_lines + + experiments_table = api.get_ophys_experiment_table().reset_index() + experiments_df = experiments_table[experiments_table.project_code==project_code].reset_index(drop=True) + experiments_df = experiments_df.merge(session_df.drop(columns=['ophys_experiment_id','ophys_container_id','session_name','date_of_acquisition','project_code','ophys_session_id']), on='behavior_session_id') + + cells_df = api.get_ophys_cells_table().reset_index() + cells_df = cells_df.merge(experiments_df, on='ophys_experiment_id') + + return (session_df, experiments_df, cells_df) + + +def get_session_table(project_code: str): + lims_db = db_connection_creator(fallback_credentials=LIMS_DB_CREDENTIAL_MAP) + api = BehaviorProjectLimsApi.default(passed_only=False) + session_table = api.get_ophys_session_table().reset_index() + session_df = session_table[session_table.project_code==project_code].reset_index(drop=True) + behavior_session_ids = session_df['behavior_session_id'].values + main_paths = [] + session_types = [] + mouse_ids = [] + desc = 'loading stimulus pkl files' + for i_session in tqdm (range(len(behavior_session_ids)), desc=desc): + behavior_session_id = behavior_session_ids[i_session] + stimulus_file = BehaviorStimulusFile.from_lims( + db=lims_db, behavior_session_id=behavior_session_id)\ + .validate() + main_paths.append(os.sep.join(stimulus_file.filepath.split(os.sep)[:-3])) + session_types.append(stimulus_file.session_type) + try: + mouse_ids.append(stimulus_file.data['items']['behavior']['params']['mouse_id']) + except: + mouse_ids.append(stimulus_file.data['mouse_id']) + + session_df['main_path'] = main_paths + session_df['session_type'] = session_types + session_df['mouse_id'] = mouse_ids + + return session_df + +def get_experiment_table(project_code: str): + session_df = get_session_table(project_code = project_code) + api = BehaviorProjectLimsApi.default(passed_only=False) + experiments_table = api.get_ophys_experiment_table().reset_index() + experiments_df = experiments_table[experiments_table.project_code==project_code].reset_index(drop=True) + experiments_df = experiments_df.merge(session_df.drop(columns=['ophys_experiment_id','ophys_container_id','session_name','date_of_acquisition','project_code','ophys_session_id']), on='behavior_session_id') + + return experiments_df + +def get_cell_table(project_code: str): + experiments_df = get_experiment_table(project_code = project_code) + api = BehaviorProjectLimsApi.default(passed_only=False) + cells_df = api.get_ophys_cells_table().reset_index() + cells_df = cells_df.merge(experiments_df, on='ophys_experiment_id') + + return cells_df + + + diff --git a/allensdk/internal/brain_observatory/time_sync.py b/allensdk/internal/brain_observatory/time_sync.py index e191525f2f..8a6ca54a39 100644 --- a/allensdk/internal/brain_observatory/time_sync.py +++ b/allensdk/internal/brain_observatory/time_sync.py @@ -37,7 +37,7 @@ def get_keys(sync_dset: Dataset) -> dict: # line labels key_dict = { "photodiode": ["stim_photodiode", "photodiode"], - "2p": ["2p_vsync"], + "2p": ["vsync_2p","2p_vsync"], "stimulus": ["stim_vsync", "vsync_stim"], "eye_camera": ["cam2_exposure", "eye_tracking", "eye_frame_received"], @@ -62,12 +62,12 @@ def get_keys(sync_dset: Dataset) -> dict: # the contents of the `remove_keys` list is printed to the console # as a user warning - if len(remove_keys) > 0: - logging.warning("Could not find valid lines for the following data " - "sources") - for key in remove_keys: - logging.warning(f"{key} (valid line label(s) = {key_dict[key]}") - key_dict.pop(key) + # if len(remove_keys) > 0: + # logging.warning("Could not find valid lines for the following data " + # "sources") + # for key in remove_keys: + # logging.warning(f"{key} (valid line label(s) = {key_dict[key]}") + # key_dict.pop(key) return key_dict