Implement media segment support

jakobkukla · jakobkukla · commit ff1917f516d8 · 2025-01-28T20:22:49.000+01:00
Most of the proposed changes in the files listed below have been shamelessly copied from the Android TV implementation in jellyfin/jellyfin-androidtv#4052. Authorship of these changes belongs to nielsvanvelzen. app/src/main/java/org/jellyfin/mobile/player/PlayerViewModel.kt app/src/main/java/org/jellyfin/mobile/player/mediasegments/MediaSegmentAction.kt app/src/main/java/org/jellyfin/mobile/player/mediasegments/MediaSegmentRepository.kt app/src/main/java/org/jellyfin/mobile/utils/extensions/MediaSegment.kt
diff --git a/app/src/main/assets/native/MediaSegmentsPlugin.js b/app/src/main/assets/native/MediaSegmentsPlugin.js
@@ -0,0 +1,32 @@
+export class MediaSegmentsPlugin {
+    SETTING_PREFIX = 'segmentTypeAction';
+
+    constructor({ events, appSettings, dashboard }) {
+        this.appSettings = appSettings;
+        this.dashboard = dashboard;
+
+        events.on(appSettings, 'change', (_, name) => this.onSettingsChanged(name));
+    }
+
+    getSettingId(type) {
+        return `${this.SETTING_PREFIX}__${type}`;
+    }
+
+    getSettingValue(id) {
+        var userId = this.dashboard.getCurrentUserId();
+
+        return this.appSettings.get(id, userId);
+    }
+
+    // Update media segment action
+    onSettingsChanged(name) {
+        if (name.startsWith(this.SETTING_PREFIX)) {
+            var type = name.slice(this.SETTING_PREFIX.length + 2);
+            var action = this.getSettingValue(this.getSettingId(type));
+
+            if (type != null && action != null) {
+                MediaSegments.setSegmentTypeAction(type, action);
+            }
+        }
+    }
+}
diff --git a/app/src/main/assets/native/nativeshell.js b/app/src/main/assets/native/nativeshell.js
@@ -18,7 +18,8 @@ const features = [
 const plugins = [
     'NavigationPlugin',
     'ExoPlayerPlugin',
-    'ExternalPlayerPlugin'
+    'ExternalPlayerPlugin',
+    'MediaSegmentsPlugin'
 ];
 
 // Add plugin loaders
diff --git a/app/src/main/java/org/jellyfin/mobile/app/AppModule.kt b/app/src/main/java/org/jellyfin/mobile/app/AppModule.kt
@@ -27,6 +27,7 @@ import org.jellyfin.mobile.events.ActivityEventHandler
 import org.jellyfin.mobile.player.audio.car.LibraryBrowser
 import org.jellyfin.mobile.player.deviceprofile.DeviceProfileBuilder
 import org.jellyfin.mobile.player.interaction.PlayerEvent
+import org.jellyfin.mobile.player.mediasegments.MediaSegmentRepository
 import org.jellyfin.mobile.player.qualityoptions.QualityOptionsProvider
 import org.jellyfin.mobile.player.source.MediaSourceResolver
 import org.jellyfin.mobile.player.ui.PlayerFragment
@@ -82,6 +83,7 @@ val applicationModule = module {
     single { MediaSourceResolver(get()) }
     single { DeviceProfileBuilder(get()) }
     single { QualityOptionsProvider() }
+    single { MediaSegmentRepository() }
 
     // ExoPlayer factories
     single<DataSource.Factory> {
diff --git a/app/src/main/java/org/jellyfin/mobile/app/AppPreferences.kt b/app/src/main/java/org/jellyfin/mobile/app/AppPreferences.kt
@@ -5,9 +5,12 @@ import android.content.SharedPreferences
 import android.os.Environment
 import android.view.WindowManager.LayoutParams.BRIGHTNESS_OVERRIDE_NONE
 import androidx.core.content.edit
+import org.jellyfin.mobile.player.mediasegments.MediaSegmentAction
+import org.jellyfin.mobile.player.mediasegments.toMediaSegmentActionsString
 import org.jellyfin.mobile.settings.ExternalPlayerPackage
 import org.jellyfin.mobile.settings.VideoPlayerType
 import org.jellyfin.mobile.utils.Constants
+import org.jellyfin.sdk.model.api.MediaSegmentType
 import java.io.File
 
 class AppPreferences(context: Context) {
@@ -90,6 +93,19 @@ class AppPreferences(context: Context) {
             }
         }
 
+    /**
+     * The actions to take for each media segment type. Managed by the MediaSegmentRepository.
+     */
+    var mediaSegmentActions: String
+        get() = sharedPreferences.getString(
+            Constants.PREF_MEDIA_SEGMENT_ACTIONS,
+            mapOf(
+                MediaSegmentType.INTRO to MediaSegmentAction.ASK_TO_SKIP,
+                MediaSegmentType.OUTRO to MediaSegmentAction.ASK_TO_SKIP,
+            ).toMediaSegmentActionsString(),
+        )!!
+        set(value) = sharedPreferences.edit { putString(Constants.PREF_MEDIA_SEGMENT_ACTIONS, value) }
+
     val musicNotificationAlwaysDismissible: Boolean
         get() = sharedPreferences.getBoolean(Constants.PREF_MUSIC_NOTIFICATION_ALWAYS_DISMISSIBLE, false)
 
diff --git a/app/src/main/java/org/jellyfin/mobile/bridge/MediaSegments.kt b/app/src/main/java/org/jellyfin/mobile/bridge/MediaSegments.kt
@@ -0,0 +1,35 @@
+package org.jellyfin.mobile.bridge
+
+import android.content.Context
+import android.webkit.JavascriptInterface
+import org.jellyfin.mobile.player.mediasegments.MediaSegmentAction
+import org.jellyfin.mobile.player.mediasegments.MediaSegmentRepository
+import org.jellyfin.sdk.model.api.MediaSegmentType
+import org.koin.core.component.KoinComponent
+import org.koin.core.component.get
+
+@Suppress("unused")
+class MediaSegments(private val context: Context) : KoinComponent {
+    private val mediaSegmentRepository: MediaSegmentRepository = get()
+
+    @JavascriptInterface
+    fun setSegmentTypeAction(typeString: String, actionString: String) {
+        val type: MediaSegmentType = when(typeString) {
+            "Intro" -> MediaSegmentType.INTRO
+            "Outro" -> MediaSegmentType.OUTRO
+            "Preview" -> MediaSegmentType.PREVIEW
+            "Recap" -> MediaSegmentType.RECAP
+            "Commercial" -> MediaSegmentType.COMMERCIAL
+            else -> return
+        }
+
+        val action: MediaSegmentAction = when(actionString) {
+            "None" -> MediaSegmentAction.NOTHING
+            "Skip" -> MediaSegmentAction.SKIP
+            "AskToSkip" -> MediaSegmentAction.ASK_TO_SKIP
+            else -> return
+        }
+
+        mediaSegmentRepository.setDefaultSegmentTypeAction(type, action)
+    }
+}
diff --git a/app/src/main/java/org/jellyfin/mobile/player/PlayerViewModel.kt b/app/src/main/java/org/jellyfin/mobile/player/PlayerViewModel.kt
@@ -38,6 +38,8 @@ import org.jellyfin.mobile.player.interaction.PlayerEvent
 import org.jellyfin.mobile.player.interaction.PlayerLifecycleObserver
 import org.jellyfin.mobile.player.interaction.PlayerMediaSessionCallback
 import org.jellyfin.mobile.player.interaction.PlayerNotificationHelper
+import org.jellyfin.mobile.player.mediasegments.MediaSegmentAction
+import org.jellyfin.mobile.player.mediasegments.MediaSegmentRepository
 import org.jellyfin.mobile.player.queue.QueueManager
 import org.jellyfin.mobile.player.source.JellyfinMediaSource
 import org.jellyfin.mobile.player.ui.DecoderType
@@ -47,7 +49,9 @@ import org.jellyfin.mobile.utils.Constants
 import org.jellyfin.mobile.utils.Constants.SUPPORTED_VIDEO_PLAYER_PLAYBACK_ACTIONS
 import org.jellyfin.mobile.utils.applyDefaultAudioAttributes
 import org.jellyfin.mobile.utils.applyDefaultLocalAudioAttributes
+import org.jellyfin.mobile.utils.extensions.end
 import org.jellyfin.mobile.utils.extensions.scaleInRange
+import org.jellyfin.mobile.utils.extensions.start
 import org.jellyfin.mobile.utils.extensions.width
 import org.jellyfin.mobile.utils.getVolumeLevelPercent
 import org.jellyfin.mobile.utils.getVolumeRange
@@ -65,6 +69,7 @@ import org.jellyfin.sdk.api.operations.DisplayPreferencesApi
 import org.jellyfin.sdk.api.operations.HlsSegmentApi
 import org.jellyfin.sdk.api.operations.PlayStateApi
 import org.jellyfin.sdk.api.operations.UserApi
+import org.jellyfin.sdk.model.api.MediaSegmentDto
 import org.jellyfin.sdk.model.api.PlayMethod
 import org.jellyfin.sdk.model.api.PlaybackOrder
 import org.jellyfin.sdk.model.api.PlaybackProgressInfo
@@ -96,6 +101,7 @@ class PlayerViewModel(application: Application) : AndroidViewModel(application),
     val queueManager = QueueManager(this)
     val mediaSourceOrNull: JellyfinMediaSource?
         get() = queueManager.currentMediaSourceOrNull
+    private val mediaSegmentRepository = MediaSegmentRepository()
 
     // ExoPlayer
     private val _player = MutableLiveData<ExoPlayer?>()
@@ -264,6 +270,9 @@ class PlayerViewModel(application: Application) : AndroidViewModel(application),
 
         val startTime = jellyfinMediaSource.startTimeMs
         if (startTime > 0) player.seekTo(startTime)
+
+        applyMediaSegments(jellyfinMediaSource)
+
         player.playWhenReady = playWhenReady
 
         mediaSession.setMetadata(jellyfinMediaSource.toMediaMetadata())
@@ -403,6 +412,46 @@ class PlayerViewModel(application: Application) : AndroidViewModel(application),
         }
     }
 
+    private fun applyMediaSegments(jellyfinMediaSource: JellyfinMediaSource) {
+        viewModelScope.launch {
+            if (jellyfinMediaSource.item != null) {
+                val mediaSegments = runCatching {
+                    mediaSegmentRepository.getSegmentsForItem(jellyfinMediaSource.item)
+                }.getOrNull().orEmpty()
+
+                for (mediaSegment in mediaSegments) {
+                    val action = mediaSegmentRepository.getMediaSegmentAction(mediaSegment)
+
+                    when(action) {
+                        MediaSegmentAction.SKIP -> addSkipAction(mediaSegment)
+                        MediaSegmentAction.NOTHING -> Unit
+                        // Unimplemented
+                        MediaSegmentAction.ASK_TO_SKIP -> Unit
+                    }
+                }
+            }
+        }
+    }
+
+    private fun addSkipAction(mediaSegment: MediaSegmentDto) {
+        val player = playerOrNull ?: return
+
+        player.createMessage { _, _ ->
+                // We can't seek directly on the ExoPlayer instance as not all media is seekable
+                // the seek function in the PlaybackController checks this and optionally starts a transcode
+                // at the requested position
+                // TODO: The above is probably true for jellyfin-android as well.
+                // But I believe there is no such logic here.
+                viewModelScope.launch(Dispatchers.Main) {
+                    player.seekTo(mediaSegment.end.inWholeMilliseconds)
+                }
+            }
+            // Segments at position 0 will never be hit by ExoPlayer so we need to add a minimum value
+            .setPosition(mediaSegment.start.inWholeMilliseconds.coerceAtLeast(1))
+            .setDeleteAfterDelivery(false)
+            .send()
+    }
+
     // Player controls
 
     fun play() {
diff --git a/app/src/main/java/org/jellyfin/mobile/player/mediasegments/MediaSegmentAction.kt b/app/src/main/java/org/jellyfin/mobile/player/mediasegments/MediaSegmentAction.kt
@@ -0,0 +1,20 @@
+package org.jellyfin.mobile.player.mediasegments
+
+enum class MediaSegmentAction {
+    /**
+     * Don't take any action for this segment.
+     */
+    NOTHING,
+
+    /**
+     * Seek to the end of this segment (endTicks). If the duration of this segment is shorter than 1 second it should do nothing to avoid
+     * lag. The skip action will only execute when playing over the segment start, not when seeking into the segment block.
+     */
+    SKIP,
+
+    /**
+     * Ask the user if they want to skip this segment. When the user agrees this behaves like [SKIP]. Confirmation should only be asked for
+     * segments with a duration of at least 3 seconds to avoid UI flickering.
+     */
+    ASK_TO_SKIP,
+}
diff --git a/app/src/main/java/org/jellyfin/mobile/player/mediasegments/MediaSegmentRepository.kt b/app/src/main/java/org/jellyfin/mobile/player/mediasegments/MediaSegmentRepository.kt
@@ -0,0 +1,95 @@
+package org.jellyfin.mobile.player.mediasegments
+
+import org.jellyfin.mobile.app.AppPreferences
+import org.jellyfin.mobile.utils.extensions.duration
+import org.jellyfin.sdk.api.client.ApiClient
+import org.jellyfin.sdk.api.client.extensions.mediaSegmentsApi
+import org.jellyfin.sdk.api.operations.MediaSegmentsApi
+import org.jellyfin.sdk.model.api.BaseItemDto
+import org.jellyfin.sdk.model.api.MediaSegmentDto
+import org.jellyfin.sdk.model.api.MediaSegmentType
+import org.koin.core.component.KoinComponent
+import org.koin.core.component.get
+import org.koin.core.component.inject
+import kotlin.time.Duration.Companion.seconds
+
+fun Map<MediaSegmentType, MediaSegmentAction>.toMediaSegmentActionsString() =
+    map { "${it.key.serialName}=${it.value.name}" }
+        .joinToString(",")
+
+class MediaSegmentRepository : KoinComponent {
+    companion object {
+        /**
+         * All media segments currently supported by the app.
+         */
+        val SupportedTypes = listOf(
+            MediaSegmentType.INTRO,
+            MediaSegmentType.OUTRO,
+            MediaSegmentType.PREVIEW,
+            MediaSegmentType.RECAP,
+            MediaSegmentType.COMMERCIAL,
+        )
+
+        /**
+         * The minimum duration for a media segment to allow the [MediaSegmentAction.SKIP] action.
+         */
+        val SkipMinDuration = 1.seconds
+    }
+
+    private val appPreferences: AppPreferences by inject()
+    private val apiClient: ApiClient = get()
+    private val mediaSegmentsApi: MediaSegmentsApi = apiClient.mediaSegmentsApi
+
+    private val mediaTypeActions = mutableMapOf<MediaSegmentType, MediaSegmentAction>()
+
+    init {
+        restoreMediaTypeActions()
+    }
+
+    private fun restoreMediaTypeActions() {
+        val restoredMediaTypeActions = appPreferences.mediaSegmentActions
+            .split(",")
+            .mapNotNull {
+                runCatching {
+                    val (type, action) = it.split('=', limit = 2)
+                    MediaSegmentType.fromName(type) to MediaSegmentAction.valueOf(action)
+                }.getOrNull()
+            }
+
+        mediaTypeActions.clear()
+        mediaTypeActions.putAll(restoredMediaTypeActions)
+    }
+
+    private fun saveMediaTypeActions() {
+        appPreferences.mediaSegmentActions = mediaTypeActions.toMediaSegmentActionsString()
+    }
+
+    fun getDefaultSegmentTypeAction(type: MediaSegmentType): MediaSegmentAction {
+        // Always return no action for unsupported types
+        if (!SupportedTypes.contains(type)) return MediaSegmentAction.NOTHING
+
+        return mediaTypeActions.getOrDefault(type, MediaSegmentAction.NOTHING)
+    }
+
+    fun setDefaultSegmentTypeAction(type: MediaSegmentType, action: MediaSegmentAction) {
+        // Don't allow modifying actions for unsupported types
+        if (!SupportedTypes.contains(type)) return
+
+        mediaTypeActions[type] = action
+        saveMediaTypeActions()
+    }
+
+    fun getMediaSegmentAction(segment: MediaSegmentDto): MediaSegmentAction {
+        val action = getDefaultSegmentTypeAction(segment.type)
+        // Skip the skip action if timespan is too short
+        if (action == MediaSegmentAction.SKIP && segment.duration < SkipMinDuration) return MediaSegmentAction.NOTHING
+        return action
+    }
+
+    suspend fun getSegmentsForItem(item: BaseItemDto): List<MediaSegmentDto> = runCatching {
+        mediaSegmentsApi.getItemSegments(
+            itemId = item.id,
+            includeSegmentTypes = SupportedTypes,
+        ).content.items
+    }.getOrDefault(emptyList())
+}
diff --git a/app/src/main/java/org/jellyfin/mobile/utils/Constants.kt b/app/src/main/java/org/jellyfin/mobile/utils/Constants.kt
@@ -42,6 +42,7 @@ object Constants {
     const val PREF_EXTERNAL_PLAYER_APP = "pref_external_player_app"
     const val PREF_SUBTITLE_STYLE = "pref_subtitle_style"
     const val PREF_DOWNLOAD_LOCATION = "pref_download_location"
+    const val PREF_MEDIA_SEGMENT_ACTIONS = "pref_media_segment_actions"
 
     // InputManager commands
     const val PLAYBACK_MANAGER_COMMAND_PLAY = "unpause"
diff --git a/app/src/main/java/org/jellyfin/mobile/utils/extensions/MediaSegment.kt b/app/src/main/java/org/jellyfin/mobile/utils/extensions/MediaSegment.kt
@@ -0,0 +1,10 @@
+package org.jellyfin.mobile.utils.extensions
+
+import org.jellyfin.sdk.model.api.MediaSegmentDto
+import org.jellyfin.sdk.model.extensions.ticks
+import kotlin.time.Duration
+
+val MediaSegmentDto.start get() = startTicks.ticks
+val MediaSegmentDto.end get() = endTicks.ticks
+
+val MediaSegmentDto.duration get() = (endTicks - startTicks).ticks.coerceAtLeast(Duration.ZERO)
diff --git a/app/src/main/java/org/jellyfin/mobile/webapp/WebViewFragment.kt b/app/src/main/java/org/jellyfin/mobile/webapp/WebViewFragment.kt
@@ -28,6 +28,7 @@ import org.jellyfin.mobile.R
 import org.jellyfin.mobile.app.ApiClientController
 import org.jellyfin.mobile.app.AppPreferences
 import org.jellyfin.mobile.bridge.ExternalPlayer
+import org.jellyfin.mobile.bridge.MediaSegments
 import org.jellyfin.mobile.bridge.NativeInterface
 import org.jellyfin.mobile.bridge.NativePlayer
 import org.jellyfin.mobile.data.entity.ServerEntity
@@ -192,6 +193,7 @@ class WebViewFragment : Fragment(), BackPressInterceptor, JellyfinWebChromeClien
         addJavascriptInterface(NativeInterface(requireContext()), "NativeInterface")
         addJavascriptInterface(nativePlayer, "NativePlayer")
         addJavascriptInterface(externalPlayer, "ExternalPlayer")
+        addJavascriptInterface(MediaSegments(requireContext()), "MediaSegments")
 
         loadUrl(server.hostname)
         postDelayed(timeoutRunnable, Constants.INITIAL_CONNECTION_TIMEOUT)
