[detectors] Integrate FlashFilter with AdaptiveDetector #35

Author: Breakthrough

scenedetect/_cli/config.py

@@ -33,6 +33,11 @@
 
 VALID_PYAV_THREAD_MODES = ['NONE', 'SLICE', 'FRAME', 'AUTO']
 
+DEPRECATED_CONFIG_OPTIONS = {
+    "global": {"drop-short-scenes"},
+    "detect-adaptive": {"min-delta-hsv"},
+}
+
 
 class OptionParseFailure(Exception):
     """Raised when a value provided in a user config file fails validation."""
@@ -251,8 +256,7 @@ class FlashFilterMode(Enum):
 DEFAULT_JPG_QUALITY = 95
 DEFAULT_WEBP_QUALITY = 100
 
-# TODO(v0.6.4): Warn if [detect-adaptive] min-delta-hsv and [global] drop-short-scenes are used.
-# TODO(v0.7): Remove [detect-adaptive] min-delta-hsv and [global] drop-short-scenes
+# TODO(v0.7): Remove deprecated [detect-adaptive] min-delta-hsv and [global] drop-short-scenes
 CONFIG_MAP: ConfigDict = {
     'backend-opencv': {
         'max-decode-attempts': 5,
@@ -543,6 +547,14 @@ def _load_from_disk(self, path=None):
             for log_str in errors:
                 self._init_log.append((logging.ERROR, log_str))
             raise ConfigLoadFailure(self._init_log)
+        for command in self._config:
+            for option in self._config[command]:
+                if (command in DEPRECATED_CONFIG_OPTIONS
+                        and option in DEPRECATED_CONFIG_OPTIONS[command]):
+                    self._init_log.append(
+                        (logging.WARNING, "WARNING: Config file contains deprecated option:\n  "
+                         f"[{command}] {option} will be removed in a future version."))
+                    pass
 
     def is_default(self, command: str, option: str) -> bool:
         """True if specified config option is unset (i.e. the default), False otherwise."""

scenedetect/_cli/context.py

@@ -275,11 +275,13 @@ def handle_options(
         if drop_short_scenes:
             logger.warning(
                 "WARNING: --drop-short-scenes is deprecated, use --filter-mode=drop instead.")
-            if filter_mode is None:
-                self.filter_mode = FlashFilterMode.DROP
+        if self.config.get_value("global", "drop-short-scenes", drop_short_scenes):
+            logger.info("drop-short-scenes set, overriding filter-mode")
+            self.filter_mode = FlashFilterMode.DROP
         else:
             self.filter_mode = FlashFilterMode[self.config.get_value("global", "filter-mode",
                                                                      filter_mode).upper()]
+
         self.merge_last_scene = merge_last_scene or self.config.get_value(
             "global", "merge-last-scene")
         self.frame_skip = self.config.get_value("global", "frame-skip", frame_skip)
@@ -360,13 +362,13 @@ def get_detect_adaptive_params(
 
         # TODO(v0.7): Remove these branches when removing -d/--min-delta-hsv.
         if min_delta_hsv is not None:
-            logger.error('-d/--min-delta-hsv is deprecated, use -c/--min-content-val instead.')
+            logger.error("-d/--min-delta-hsv is deprecated, use -c/--min-content-val instead.")
             if min_content_val is None:
                 min_content_val = min_delta_hsv
         # Handle case where deprecated min-delta-hsv is set, and use it to set min-content-val.
         if not self.config.is_default("detect-adaptive", "min-delta-hsv"):
-            logger.error('[detect-adaptive] config file option `min-delta-hsv` is deprecated'
-                         ', use `min-delta-hsv` instead.')
+            logger.error("[detect-adaptive] config file option `min-delta-hsv` is deprecated"
+                         ", use `min-delta-hsv` instead.")
             if self.config.is_default("detect-adaptive", "min-content-val"):
                 self.config.config_dict["detect-adaptive"]["min-content-val"] = (
                     self.config.config_dict["detect-adaptive"]["min-deleta-hsv"])
@@ -379,21 +381,21 @@ def get_detect_adaptive_params(
                 weights = ContentDetector.Components(*weights)
             except ValueError as ex:
                 logger.debug(str(ex))
-                raise click.BadParameter(str(ex), param_hint='weights')
+                raise click.BadParameter(str(ex), param_hint="weights")
         return {
-            'adaptive_threshold':
+            "adaptive_threshold":
                 self.config.get_value("detect-adaptive", "threshold", threshold),
-            'weights':
-                self.config.get_value("detect-adaptive", "weights", weights),
-            'kernel_size':
+            "flash_filter":
+                self._init_flash_filter("detect-content", min_scene_len),
+            "kernel_size":
                 self.config.get_value("detect-adaptive", "kernel-size", kernel_size),
-            'luma_only':
+            "luma_only":
                 luma_only or self.config.get_value("detect-adaptive", "luma-only"),
-            'min_content_val':
+            "min_content_val":
                 self.config.get_value("detect-adaptive", "min-content-val", min_content_val),
-            'min_scene_len':
-                min_scene_len,
-            'window_width':
+            "weights":
+                self.config.get_value("detect-adaptive", "weights", weights),
+            "window_width":
                 self.config.get_value("detect-adaptive", "frame-window", frame_window),
         }
 

scenedetect/detectors/adaptive_detector.py

@@ -18,11 +18,12 @@
 """
 
 from logging import getLogger
-from typing import List, Optional
+import typing as ty
 
 import numpy as np
 
 from scenedetect.detectors import ContentDetector
+from scenedetect.scene_detector import FlashFilter
 
 logger = getLogger('pyscenedetect')
 
@@ -43,15 +44,19 @@ def __init__(
         min_content_val: float = 15.0,
         weights: ContentDetector.Components = ContentDetector.DEFAULT_COMPONENT_WEIGHTS,
         luma_only: bool = False,
-        kernel_size: Optional[int] = None,
+        kernel_size: ty.Optional[int] = None,
+        flash_filter: ty.Optional[FlashFilter] = None,
         video_manager=None,
-        min_delta_hsv: Optional[float] = None,
+        min_delta_hsv: ty.Optional[float] = None,
     ):
         """
         Arguments:
             adaptive_threshold: Threshold (float) that score ratio must exceed to trigger a
                 new scene (see frame metric adaptive_ratio in stats file).
-            min_scene_len: Minimum length of any scene.
+            min_scene_len: Defines the minimum length of a given scene. Sequences of consecutive
+                cuts that occur closer than this length will be merged. Equivalent to setting
+                `flash_filter = FlashFilter(length=min_scene_len)`.
+                Ignored if `flash_filter` is set.
             window_width: Size of window (number of frames) before and after each frame to
                 average together in order to detect deviations from the mean. Must be at least 1.
             min_content_val: Minimum threshold (float) that the content_val must exceed in order to
@@ -65,8 +70,10 @@ def __init__(
                 Overrides `weights` if both are set.
             kernel_size: Size of kernel to use for post edge detection filtering. If None,
                 automatically set based on video resolution.
-            video_manager: [DEPRECATED] DO NOT USE. For backwards compatibility only.
-            min_delta_hsv: [DEPRECATED] DO NOT USE. Use `min_content_val` instead.
+            flash_filter: Filter to use for scene length compliance. If None, initialized as
+                `FlashFilter(length=min_scene_len)`. If set, `min_scene_length` is ignored.
+            video_manager: [DEPRECATED] DO NOT USE.
+            min_delta_hsv: [DEPRECATED] DO NOT USE.
         """
         # TODO(v0.7): Replace with DeprecationWarning that `video_manager` and `min_delta_hsv` will
         # be removed in v0.8.
@@ -77,44 +84,35 @@ def __init__(
             min_content_val = min_delta_hsv
         if window_width < 1:
             raise ValueError('window_width must be at least 1.')
-
         super().__init__(
             threshold=255.0,
-            min_scene_len=0,
+            min_scene_len=min_scene_len,
             weights=weights,
             luma_only=luma_only,
             kernel_size=kernel_size,
+            flash_filter=flash_filter,
         )
-
-        # TODO: Turn these options into properties.
-        self.min_scene_len = min_scene_len
-        self.adaptive_threshold = adaptive_threshold
-        self.min_content_val = min_content_val
-        self.window_width = window_width
-
+        self._adaptive_threshold = adaptive_threshold
+        self._min_content_val = min_content_val
+        self._window_width = window_width
         self._adaptive_ratio_key = AdaptiveDetector.ADAPTIVE_RATIO_KEY_TEMPLATE.format(
             window_width=window_width, luma_only='' if not luma_only else '_lum')
-        self._first_frame_num = None
-
-        # NOTE: This must be different than `self._last_scene_cut` which is used by the base class.
-        self._last_cut: Optional[int] = None
-
         self._buffer = []
 
     @property
     def event_buffer_length(self) -> int:
         """Number of frames any detected cuts will be behind the current frame due to buffering."""
-        return self.window_width
+        return self._window_width
 
-    def get_metrics(self) -> List[str]:
+    def get_metrics(self) -> ty.List[str]:
         """Combines base ContentDetector metric keys with the AdaptiveDetector one."""
         return super().get_metrics() + [self._adaptive_ratio_key]
 
     def stats_manager_required(self) -> bool:
         """Not required for AdaptiveDetector."""
         return False
 
-    def process_frame(self, frame_num: int, frame_img: Optional[np.ndarray]) -> List[int]:
+    def process_frame(self, frame_num: int, frame_img: ty.Optional[np.ndarray]) -> ty.List[int]:
         """Process the next frame. `frame_num` is assumed to be sequential.
 
         Args:
@@ -126,47 +124,33 @@ def process_frame(self, frame_num: int, frame_img: Optional[np.ndarray]) -> List
             List[int]: List of frames where scene cuts have been detected. There may be 0
             or more frames in the list, and not necessarily the same as frame_num.
         """
-
-        # TODO(#283): Merge this with ContentDetector and turn it on by default.
-
-        super().process_frame(frame_num=frame_num, frame_img=frame_img)
-
-        # Initialize last scene cut point at the beginning of the frames of interest.
-        if self._last_cut is None:
-            self._last_cut = frame_num
-
-        required_frames = 1 + (2 * self.window_width)
-        self._buffer.append((frame_num, self._frame_score))
+        frame_score = self._calculate_frame_score(frame_num=frame_num, frame_img=frame_img)
+        required_frames = 1 + (2 * self._window_width)
+        self._buffer.append((frame_num, frame_score))
         if not len(self._buffer) >= required_frames:
             return []
         self._buffer = self._buffer[-required_frames:]
-        target = self._buffer[self.window_width]
+        target = self._buffer[self._window_width]
         average_window_score = (
-            sum(frame[1] for i, frame in enumerate(self._buffer) if i != self.window_width) /
-            (2.0 * self.window_width))
-
+            sum(frame[1] for i, frame in enumerate(self._buffer) if i != self._window_width) /
+            (2.0 * self._window_width))
         average_is_zero = abs(average_window_score) < 0.00001
-
         adaptive_ratio = 0.0
         if not average_is_zero:
             adaptive_ratio = min(target[1] / average_window_score, 255.0)
-        elif average_is_zero and target[1] >= self.min_content_val:
+        elif average_is_zero and target[1] >= self._min_content_val:
             # if we would have divided by zero, set adaptive_ratio to the max (255.0)
             adaptive_ratio = 255.0
         if self.stats_manager is not None:
             self.stats_manager.set_metrics(target[0], {self._adaptive_ratio_key: adaptive_ratio})
 
         # Check to see if adaptive_ratio exceeds the adaptive_threshold as well as there
         # being a large enough content_val to trigger a cut
-        threshold_met: bool = (
-            adaptive_ratio >= self.adaptive_threshold and target[1] >= self.min_content_val)
-        min_length_met: bool = (frame_num - self._last_cut) >= self.min_scene_len
-        if threshold_met and min_length_met:
-            self._last_cut = target[0]
-            return [target[0]]
-        return []
-
-    def get_content_val(self, frame_num: int) -> Optional[float]:
+        found_cut: bool = (
+            adaptive_ratio >= self._adaptive_threshold and target[1] >= self._min_content_val)
+        return self._flash_filter.apply(frame_num=target[0], found_cut=found_cut)
+
+    def get_content_val(self, frame_num: int) -> ty.Optional[float]:
         """Returns the average content change for a frame."""
         # TODO(v0.7): Add DeprecationWarning that `get_content_val` will be removed in v0.7.
         logger.error("get_content_val is deprecated and will be removed. Lookup the value"
@@ -175,6 +159,10 @@ def get_content_val(self, frame_num: int) -> Optional[float]:
             return self.stats_manager.get_metrics(frame_num, [ContentDetector.FRAME_SCORE_KEY])[0]
         return 0.0
 
-    def post_process(self, _unused_frame_num: int):
-        """Not required for AdaptiveDetector."""
-        return []
+    def post_process(self, _frame_num: int):
+        # Already processed frame at self._window_width, process the rest. This ensures we emit any
+        # cuts the filtering mode might require.
+        cuts = []
+        for (frame_num, _) in self._buffer[self._window_width + 1:]:
+            cuts += self._flash_filter.apply(frame_num=frame_num, found_cut=False)
+        return cuts

scenedetect/detectors/content_detector.py

@@ -122,12 +122,10 @@ def __init__(
             kernel_size: Size of kernel for expanding detected edges. Must be odd integer
                 greater than or equal to 3. If None, automatically set using video resolution.
             flash_filter: Filter to use for scene length compliance. If None, initialized as
-                `FlashFilter(length=min_scene_len)`.
+                `FlashFilter(length=min_scene_len)`. If set, `min_scene_length` is ignored.
         """
         super().__init__()
         self._threshold: float = threshold
-        self._min_scene_len: int = min_scene_len
-        self._last_above_threshold: ty.Optional[int] = None
         self._last_frame: ty.Optional[ContentDetector._FrameData] = None
         self._weights: ContentDetector.Components = weights
         if luma_only:
@@ -138,7 +136,6 @@ def __init__(
             if kernel_size < 3 or kernel_size % 2 == 0:
                 raise ValueError('kernel_size must be odd integer >= 3')
             self._kernel = numpy.ones((kernel_size, kernel_size), numpy.uint8)
-        self._frame_score: ty.Optional[float] = None
         self._flash_filter = flash_filter if not flash_filter is None else FlashFilter(
             length=min_scene_len)
 
@@ -202,11 +199,9 @@ def process_frame(self, frame_num: int, frame_img: numpy.ndarray) -> ty.List[int
             ty.List[int]: List of frames where scene cuts have been detected. There may be 0
             or more frames in the list, and not necessarily the same as frame_num.
         """
-        self._frame_score = self._calculate_frame_score(frame_num, frame_img)
-        if self._frame_score is None:
-            return []
-        return self._flash_filter.filter(
-            frame_num=frame_num, found_cut=self._frame_score >= self._threshold)
+        frame_score = self._calculate_frame_score(frame_num, frame_img)
+        found_cut = frame_score >= self._threshold
+        return self._flash_filter.apply(frame_num=frame_num, found_cut=found_cut)
 
     def _detect_edges(self, lum: numpy.ndarray) -> numpy.ndarray:
         """Detect edges using the luma channel of a frame.