Merge pull request #46 from BirdVox/export-faults

export log file of sensor fault

birdvoxdetect/cli.py

@@ -40,6 +40,7 @@ def run(inputs,
         output_dir=None,
         export_clips=False,
         export_confidence=False,
+        export_logfile=False,
         threshold=50.0,
         suffix="",
         clip_duration=1.0,
@@ -83,6 +84,7 @@ def run(inputs,
             output_dir=output_dir,
             export_clips=export_clips,
             export_confidence=export_confidence,
+            export_logfile=export_logfile,
             threshold=threshold,
             suffix=suffix,
             clip_duration=clip_duration,
@@ -117,6 +119,10 @@ def parse_args(args):
         help='Export the time series of model confidence values of events'
              'in HDF5 format.')
 
+    parser.add_argument(
+        '--export-logfile', '-l', action='store_true',
+        help='Export log file of sensor faults in CSV format.')
+
     parser.add_argument(
         '--threshold', '-t', type=valid_threshold, default=50,
         help='Detection threshold, between 10 and 90. '
@@ -194,6 +200,7 @@ def main():
         output_dir=args.output_dir,
         export_clips=args.export_clips,
         export_confidence=args.export_confidence,
+        export_logfile=args.export_logfile,
         threshold=args.threshold,
         suffix=args.suffix,
         clip_duration=args.clip_duration,

birdvoxdetect/core.py

@@ -45,6 +45,7 @@ def process_file(
         output_dir=None,
         export_clips=False,
         export_confidence=False,
+        export_logfile=False,
         threshold=50.0,
         suffix="",
         clip_duration=1.0,
@@ -195,6 +196,15 @@ def process_file(
         })
         df.to_csv(checklist_path, columns=df_columns, index=False)
 
+    # Initialize fault log as a Pandas DataFrame.
+    if export_logfile:
+        logfile_path = get_output_path(
+            filepath, suffix + "logfile.csv", output_dir=output_dir)
+        logfile_df_columns = [
+            "Start (hh:mm:ss)", "Stop (hh:mm:ss)", "Fault?"]
+        logfile_df = pd.DataFrame(columns=logfile_df_columns)
+        logfile_df.to_csv(logfile_path, columns=logfile_df_columns, index=False)
+
     # Create directory of output clips.
     if export_clips:
         clips_dir = get_output_path(
@@ -266,10 +276,21 @@ def process_file(
         else:
             chunk_id_start = 0
             has_sensor_fault = False
+
     else:
         chunk_id_start = 0
         has_sensor_fault = False
 
+    # Add first row to sensor fault log.
+    if export_logfile:
+        logfile_df.append({
+            "Start (hh:mm:ss)": seconds_to_hhmmss(0.0),
+            "Stop (hh:mm:ss)": seconds_to_hhmmss(
+                min(chunk_duration, full_length/sr)),
+            "Fault?": int(has_sensor_fault)},
+            ignore_index=True)
+        logfile_df.to_csv(logfile_path, columns=logfile_df_columns, index=False)
+
     # Define frame rate.
     frame_rate = pcen_settings["sr"] /\
         (pcen_settings["hop_length"] * pcen_settings["stride_length"])
@@ -396,9 +417,18 @@ def process_file(
         sensor_fault_probability =\
             sensorfault_model.predict(sensorfault_features)[0]
 
-        # If probability of sensor fault is above threshold,
-        # exclude start of recording
+        # Add row to sensor fault log.
         has_sensor_fault = (sensor_fault_probability > bva_threshold)
+        if export_logfile:
+            logfile_df.append({
+                "Start (hh:mm:ss)": seconds_to_hhmmss(chunk_id*chunk_duration),
+                "Stop (hh:mm:ss)": seconds_to_hhmmss((chunk_id+1)*chunk_duration),
+                "Fault?": int(has_sensor_fault)},
+                ignore_index=True)
+            logfile_df.to_csv(
+                logfile_path, columns=logfile_df_columns, index=False)
+
+        # If probability of sensor fault is above threshold, exclude chunk.
         if has_sensor_fault:
             logger.debug("Probability of sensor fault: {:5.2f}%".format(
                 100*sensor_fault_probability))
@@ -509,13 +539,24 @@ def process_file(
     # unstable with files shorter than 30 minutes, which is why we issue a
     # warning. Also, we do not try to detect sensor faults in files shorter than
     # 30 minutes.
+    if n_chunks==1:
+        has_sensor_fault = False
+    if export_logfile:
+        logfile_df.append({
+            "Start (hh:mm:ss)": seconds_to_hhmmss(chunk_id*chunk_duration),
+            "Stop (hh:mm:ss)": seconds_to_hhmmss(full_length/sr),
+            "Fault?": int(has_sensor_fault)},
+            ignore_index=True)
+        logfile_df.to_csv(
+            logfile_path, columns=logfile_df_columns, index=False)
+
     if (n_chunks>1) and has_sensor_fault:
         logger.debug("Probability of sensor fault: {:5.2f}%".format(
             100*sensor_fault_probability))
         ignored_start_str = str(datetime.timedelta(
             seconds=chunk_id*chunk_duration))
         ignored_stop_str = str(datetime.timedelta(
-            seconds=full_length*sr))
+            seconds=full_length/sr))
         logger.debug(
             "Ignoring segment between " +\
             ignored_start_str + " and " +\
@@ -695,6 +736,9 @@ def process_file(
     if export_confidence:
         event_str = "Event detection curve is available at: {}"
         logger.info(event_str.format(confidence_path))
+    if export_logfile:
+        event_str = "Log file of sensor faults is available at: {}"
+        logger.info(event_str.format(logfile_path))
     logger.info("Done with file: {}.".format(filepath))
 
     return df

setup.py

@@ -81,7 +81,7 @@
         'birdvoxclassify',
         'h5py>=2.7.0',
         'librosa==0.7.0',
-        'numba==0.48',
+        'numba==0.48.0',
         'numpy==1.16.4',
         'pandas==0.25.1',
         'scikit-learn==0.21.2',

tests/test_core.py

@@ -126,6 +126,21 @@ def test_process_file():
     assert confidence.shape == (199,)
     shutil.rmtree(tmpdir)
 
+    # export logfile
+    tmpdir = tempfile.mkdtemp()
+    process_file(
+        os.path.join(TEST_AUDIO_DIR, POSITIVE_MD5 + '.wav'),
+        output_dir=tmpdir,
+        export_logfile=True)
+    logfile_path = os.path.join(
+        tmpdir, POSITIVE_MD5 + '_logfile.csv')
+    assert os.path.exists(logfile_path)
+    logfile_df = pd.read_csv(logfile_path)
+    columns = logfile_df.columns
+    assert np.all(
+        columns == np.array(["Start (hh:mm:ss)", "Stop (hh:mm:ss)", "Fault?"]))
+    shutil.rmtree(tmpdir)
+
     # suffix
     tmpdir = tempfile.mkdtemp()
     process_file(