Adding support for stimuli in omFISH project (STAGE_0 and STAGE_1 session types)

allensdk/brain_observatory/behavior/behavior_session.py

@@ -1484,17 +1484,25 @@ def _read_data_from_stimulus_file(
         else:
             stimuli = None
 
-        trials = cls._read_trials(
+        stimulus_file = stimulus_file_lookup.behavior_stimulus_file.data
+        if "trial_log" in stimulus_file["items"]["behavior"]:
+            trials = cls._read_trials(
             stimulus_file_lookup=stimulus_file_lookup,
             sync_file=sync_file,
             monitor_delay=monitor_delay,
             licks=licks,
             rewards=rewards,
-        )
+            )
+        else:
+            trials = None
 
-        task_parameters = TaskParameters.from_stimulus_file(
-            stimulus_file=stimulus_file_lookup.behavior_stimulus_file
-        )
+
+        if "DoC" in stimulus_file["items"]["behavior"]["config"]: 
+            task_parameters = TaskParameters.from_stimulus_file(
+                stimulus_file=stimulus_file_lookup.behavior_stimulus_file
+            )
+        else:
+            task_parameters = None
 
         return (
             session_stimulus_timestamps.subtract_monitor_delay(),

allensdk/brain_observatory/behavior/data_files/stimulus_file.py

@@ -156,8 +156,10 @@ def date_of_acquisition(self) -> datetime.datetime:
             raise KeyError(
                 "No 'start_time' listed in pickle file "
                 f"{self.filepath}")
-
-        return copy.deepcopy(self.data['start_time'])
+        start_time = self.data['start_time']
+        if not isinstance(start_time, datetime.datetime):
+            start_time = datetime.datetime.fromtimestamp(start_time)
+        return copy.deepcopy(start_time)
 
     @property
     def session_type(self) -> str:
@@ -239,10 +241,12 @@ def stimuli(self) -> Dict[str, Tuple[str, Union[str, int], int, int]]:
         return self.data['items']['behavior']['stimuli']
 
     def validate(self) -> "BehaviorStimulusFile":
-        if 'items' not in self.data or 'behavior' not in self.data['items']:
-            raise MalformedStimulusFileError(
-                f'Expected to find key "behavior" in "items" dict. '
+        if 'items' not in self.data or ('behavior' not in self.data['items']  and 'foraging' not in self.data['items']):
+             raise MalformedStimulusFileError(
+                f'Expected to find keys "behavior" or "foraging" in "items" dict. '
                 f'Found {self.data["items"].keys()}')
+        elif 'behavior' not in self.data['items']:
+                self.data['items']['behavior'] = self.data['items']['foraging']
         return self
 
 

allensdk/brain_observatory/behavior/data_objects/rewards.py

@@ -36,19 +36,22 @@ def from_stimulus_file(
             raise RuntimeError(msg)
 
         data = stimulus_file.data
+        if "trial_log" in data["items"]["behavior"]:
+            trial_df = pd.DataFrame(data["items"]["behavior"]["trial_log"])
+            rewards_dict = {"volume": [], "timestamps": [], "auto_rewarded": []}
+            for idx, trial in trial_df.iterrows():
+                rewards = trial["rewards"]
+                # as i write this there can only ever be one reward per trial
+                if rewards:
+                    rewards_dict["volume"].append(rewards[0][0])
+                    rewards_dict["timestamps"].append(
+                        stimulus_timestamps.value[rewards[0][2]])
+                    auto_rwrd = trial["trial_params"]["auto_reward"]
+                    rewards_dict["auto_rewarded"].append(auto_rwrd)
 
-        trial_df = pd.DataFrame(data["items"]["behavior"]["trial_log"])
-        rewards_dict = {"volume": [], "timestamps": [], "auto_rewarded": []}
-        for idx, trial in trial_df.iterrows():
-            rewards = trial["rewards"]
-            # as i write this there can only ever be one reward per trial
-            if rewards:
-                rewards_dict["volume"].append(rewards[0][0])
-                rewards_dict["timestamps"].append(
-                    stimulus_timestamps.value[rewards[0][2]])
-                auto_rwrd = trial["trial_params"]["auto_reward"]
-                rewards_dict["auto_rewarded"].append(auto_rwrd)
-
+        else:
+            rewards_dict = data["items"]["behavior"]["rewards"]
+
         df = pd.DataFrame(rewards_dict)
         return cls(rewards=df)
 

allensdk/brain_observatory/behavior/data_objects/stimuli/presentations.py

@@ -205,85 +205,88 @@ def from_stimulus_file(
         #   1. Nulls in `image_name` should be "gratings_<orientation>"
         #   2. Gratings are only present (or need to be fixed) when all
         #      values for `image_name` are null.
-        if pd.isnull(raw_stim_pres_df["image_name"]).all():
-            if ~pd.isnull(raw_stim_pres_df["orientation"]).all():
-                raw_stim_pres_df["image_name"] = (
-                    raw_stim_pres_df["orientation"]
-                    .apply(lambda x: f"gratings_{x}"))
-            else:
-                raise ValueError("All values for 'orientation' and "
-                                 "'image_name are null.")
-
-        stimulus_metadata_df = get_stimulus_metadata(data)
-
-        idx_name = raw_stim_pres_df.index.name
-        stimulus_index_df = (
-            raw_stim_pres_df
-            .reset_index()
-            .merge(stimulus_metadata_df.reset_index(),
-                   on=["image_name"])
-            .set_index(idx_name))
-        stimulus_index_df = (
-            stimulus_index_df[["image_set", "image_index", "start_time",
-                               "phase", "spatial_frequency"]]
-            .rename(columns={"start_time": "timestamps"})
-            .sort_index()
-            .set_index("timestamps", drop=True))
-        stim_pres_df = raw_stim_pres_df.merge(
-            stimulus_index_df, left_on="start_time", right_index=True,
-            how="left")
-        if len(raw_stim_pres_df) != len(stim_pres_df):
-            raise ValueError("Length of `stim_pres_df` should not change after"
-                             f" merge; was {len(raw_stim_pres_df)}, now "
-                             f" {len(stim_pres_df)}.")
-
-        stim_pres_df['is_change'] = is_change_event(
-            stimulus_presentations=stim_pres_df)
-        stim_pres_df['flashes_since_change'] = get_flashes_since_change(
-            stimulus_presentations=stim_pres_df)
-
-        # Sort columns then drop columns which contain only all NaN values
-        stim_pres_df = \
-            stim_pres_df[sorted(stim_pres_df)].dropna(axis=1, how='all')
-        if limit_to_images is not None:
+        if "image_name" in raw_stim_pres_df:
+            if pd.isnull(raw_stim_pres_df["image_name"]).all():
+                if ~pd.isnull(raw_stim_pres_df["orientation"]).all():
+                    raw_stim_pres_df["image_name"] = (
+                        raw_stim_pres_df["orientation"]
+                        .apply(lambda x: f"gratings_{x}"))
+                else:
+                    raise ValueError("All values for 'orientation' and "
+                                    "'image_name are null.")
+
+            stimulus_metadata_df = get_stimulus_metadata(data)
+
+            idx_name = raw_stim_pres_df.index.name
+            stimulus_index_df = (
+                raw_stim_pres_df
+                .reset_index()
+                .merge(stimulus_metadata_df.reset_index(),
+                    on=["image_name"])
+                .set_index(idx_name))
+            stimulus_index_df = (
+                stimulus_index_df[["image_set", "image_index", "start_time",
+                                "phase", "spatial_frequency"]]
+                .rename(columns={"start_time": "timestamps"})
+                .sort_index()
+                .set_index("timestamps", drop=True))
+            stim_pres_df = raw_stim_pres_df.merge(
+                stimulus_index_df, left_on="start_time", right_index=True,
+                how="left")
+            if len(raw_stim_pres_df) != len(stim_pres_df):
+                raise ValueError("Length of `stim_pres_df` should not change after"
+                                f" merge; was {len(raw_stim_pres_df)}, now "
+                                f" {len(stim_pres_df)}.")
+
+            stim_pres_df['is_change'] = is_change_event(
+                stimulus_presentations=stim_pres_df)
+            stim_pres_df['flashes_since_change'] = get_flashes_since_change(
+                stimulus_presentations=stim_pres_df)
+
+            # Sort columns then drop columns which contain only all NaN values
             stim_pres_df = \
-                stim_pres_df[stim_pres_df['image_name'].isin(limit_to_images)]
-            stim_pres_df.index = pd.Int64Index(
-                range(stim_pres_df.shape[0]), name=stim_pres_df.index.name)
-
-        stim_pres_df['stimulus_block'] = 0
-        # Match the Ecephys VBN stimulus name convention.
-        try:
-            stim_pres_df['stimulus_name'] = Path(
-                stimulus_file.stimuli['images']['image_set']).\
-                stem.split('.')[0]
-        except KeyError:
-            # if we can't find the images key in the stimuli, check for the
-            # name ``grating`` as the stimulus. If not add generic
-            # ``behavior``.
-            if 'grating' in stimulus_file.stimuli.keys():
-                stim_pres_df['stimulus_name'] = 'grating'
-            else:
-                stim_pres_df['stimulus_name'] = 'behavior'
-
-        stim_pres_df = fix_omitted_end_frame(stim_pres_df)
-
-        cls._add_is_image_novel(stimulus_presentations=stim_pres_df,
-                                behavior_session_id=behavior_session_id)
-
-        has_fingerprint_stimulus = \
-            'fingerprint' in stimulus_file.data['items']['behavior']['items']
-        if has_fingerprint_stimulus:
-            stim_pres_df = cls._add_fingerprint_stimulus(
-                stimulus_presentations=stim_pres_df,
-                stimulus_file=stimulus_file,
-                stimulus_timestamps=stimulus_timestamps
+                stim_pres_df[sorted(stim_pres_df)].dropna(axis=1, how='all')
+            if limit_to_images is not None:
+                stim_pres_df = \
+                    stim_pres_df[stim_pres_df['image_name'].isin(limit_to_images)]
+                stim_pres_df.index = pd.Int64Index(
+                    range(stim_pres_df.shape[0]), name=stim_pres_df.index.name)
+
+            stim_pres_df['stimulus_block'] = 0
+            # Match the Ecephys VBN stimulus name convention.
+            try:
+                stim_pres_df['stimulus_name'] = Path(
+                    stimulus_file.stimuli['images']['image_set']).\
+                    stem.split('.')[0]
+            except KeyError:
+                # if we can't find the images key in the stimuli, check for the
+                # name ``grating`` as the stimulus. If not add generic
+                # ``behavior``.
+                if 'grating' in stimulus_file.stimuli.keys():
+                    stim_pres_df['stimulus_name'] = 'grating'
+                else:
+                    stim_pres_df['stimulus_name'] = 'behavior'
+
+            stim_pres_df = fix_omitted_end_frame(stim_pres_df)
+
+            cls._add_is_image_novel(stimulus_presentations=stim_pres_df,
+                                    behavior_session_id=behavior_session_id)
+
+            has_fingerprint_stimulus = \
+                'fingerprint' in stimulus_file.data['items']['behavior']['items']
+            if has_fingerprint_stimulus:
+                stim_pres_df = cls._add_fingerprint_stimulus(
+                    stimulus_presentations=stim_pres_df,
+                    stimulus_file=stimulus_file,
+                    stimulus_timestamps=stimulus_timestamps
+                )
+            stim_pres_df = cls._postprocess(
+                presentations=stim_pres_df,
+                fill_omitted_values=fill_omitted_values,
+                coerce_bool_to_boolean=True
             )
-        stim_pres_df = cls._postprocess(
-            presentations=stim_pres_df,
-            fill_omitted_values=fill_omitted_values,
-            coerce_bool_to_boolean=True
-        )
+        else:
+            stim_pres_df = raw_stim_pres_df
         return Presentations(presentations=stim_pres_df,
                              column_list=column_list)