Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ExoPlayer: Implement media segment support #1507

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions app/src/main/assets/native/MediaSegmentsPlugin.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
export class MediaSegmentsPlugin {
SETTING_PREFIX = 'segmentTypeAction';

constructor({ events, appSettings, dashboard }) {
this.appSettings = appSettings;
this.dashboard = dashboard;

events.on(appSettings, 'change', (_, name) => this.onSettingsChanged(name));
}

getSettingId(type) {
return `${this.SETTING_PREFIX}__${type}`;
}

getSettingValue(id) {
var userId = this.dashboard.getCurrentUserId();

return this.appSettings.get(id, userId);
}

// Update media segment action
onSettingsChanged(name) {
if (name.startsWith(this.SETTING_PREFIX)) {
var type = name.slice(this.SETTING_PREFIX.length + 2);
var action = this.getSettingValue(this.getSettingId(type));

if (type != null && action != null) {
MediaSegments.setSegmentTypeAction(type, action);
}
}
}
}
3 changes: 2 additions & 1 deletion app/src/main/assets/native/nativeshell.js
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@ const features = [
const plugins = [
'NavigationPlugin',
'ExoPlayerPlugin',
'ExternalPlayerPlugin'
'ExternalPlayerPlugin',
'MediaSegmentsPlugin'
];

// Add plugin loaders
Expand Down
2 changes: 2 additions & 0 deletions app/src/main/java/org/jellyfin/mobile/app/AppModule.kt
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import org.jellyfin.mobile.events.ActivityEventHandler
import org.jellyfin.mobile.player.audio.car.LibraryBrowser
import org.jellyfin.mobile.player.deviceprofile.DeviceProfileBuilder
import org.jellyfin.mobile.player.interaction.PlayerEvent
import org.jellyfin.mobile.player.mediasegments.MediaSegmentRepository
import org.jellyfin.mobile.player.qualityoptions.QualityOptionsProvider
import org.jellyfin.mobile.player.source.MediaSourceResolver
import org.jellyfin.mobile.player.ui.PlayerFragment
Expand Down Expand Up @@ -82,6 +83,7 @@ val applicationModule = module {
single { MediaSourceResolver(get()) }
single { DeviceProfileBuilder(get()) }
single { QualityOptionsProvider() }
single { MediaSegmentRepository() }

// ExoPlayer factories
single<DataSource.Factory> {
Expand Down
16 changes: 16 additions & 0 deletions app/src/main/java/org/jellyfin/mobile/app/AppPreferences.kt
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,12 @@ import android.content.SharedPreferences
import android.os.Environment
import android.view.WindowManager.LayoutParams.BRIGHTNESS_OVERRIDE_NONE
import androidx.core.content.edit
import org.jellyfin.mobile.player.mediasegments.MediaSegmentAction
import org.jellyfin.mobile.player.mediasegments.toMediaSegmentActionsString
import org.jellyfin.mobile.settings.ExternalPlayerPackage
import org.jellyfin.mobile.settings.VideoPlayerType
import org.jellyfin.mobile.utils.Constants
import org.jellyfin.sdk.model.api.MediaSegmentType
import java.io.File

class AppPreferences(context: Context) {
Expand Down Expand Up @@ -90,6 +93,19 @@ class AppPreferences(context: Context) {
}
}

/**
* The actions to take for each media segment type. Managed by the MediaSegmentRepository.
*/
var mediaSegmentActions: String
get() = sharedPreferences.getString(
Constants.PREF_MEDIA_SEGMENT_ACTIONS,
mapOf(
MediaSegmentType.INTRO to MediaSegmentAction.ASK_TO_SKIP,
MediaSegmentType.OUTRO to MediaSegmentAction.ASK_TO_SKIP,
).toMediaSegmentActionsString(),
)!!
set(value) = sharedPreferences.edit { putString(Constants.PREF_MEDIA_SEGMENT_ACTIONS, value) }

val musicNotificationAlwaysDismissible: Boolean
get() = sharedPreferences.getBoolean(Constants.PREF_MUSIC_NOTIFICATION_ALWAYS_DISMISSIBLE, false)

Expand Down
35 changes: 35 additions & 0 deletions app/src/main/java/org/jellyfin/mobile/bridge/MediaSegments.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
package org.jellyfin.mobile.bridge

import android.content.Context
import android.webkit.JavascriptInterface
import org.jellyfin.mobile.player.mediasegments.MediaSegmentAction
import org.jellyfin.mobile.player.mediasegments.MediaSegmentRepository
import org.jellyfin.sdk.model.api.MediaSegmentType
import org.koin.core.component.KoinComponent
import org.koin.core.component.get

@Suppress("unused")
class MediaSegments(private val context: Context) : KoinComponent {
private val mediaSegmentRepository: MediaSegmentRepository = get()

@JavascriptInterface
fun setSegmentTypeAction(typeString: String, actionString: String) {
val type: MediaSegmentType = when(typeString) {

Check warning

Code scanning / detekt

Reports spaces around keywords Warning

Missing spacing after "when"
"Intro" -> MediaSegmentType.INTRO
"Outro" -> MediaSegmentType.OUTRO
"Preview" -> MediaSegmentType.PREVIEW
"Recap" -> MediaSegmentType.RECAP
"Commercial" -> MediaSegmentType.COMMERCIAL
else -> return
}

val action: MediaSegmentAction = when(actionString) {

Check warning

Code scanning / detekt

Reports spaces around keywords Warning

Missing spacing after "when"
"None" -> MediaSegmentAction.NOTHING
"Skip" -> MediaSegmentAction.SKIP
"AskToSkip" -> MediaSegmentAction.ASK_TO_SKIP
else -> return
}

mediaSegmentRepository.setDefaultSegmentTypeAction(type, action)
}
}
49 changes: 49 additions & 0 deletions app/src/main/java/org/jellyfin/mobile/player/PlayerViewModel.kt
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@
import org.jellyfin.mobile.player.interaction.PlayerLifecycleObserver
import org.jellyfin.mobile.player.interaction.PlayerMediaSessionCallback
import org.jellyfin.mobile.player.interaction.PlayerNotificationHelper
import org.jellyfin.mobile.player.mediasegments.MediaSegmentAction
import org.jellyfin.mobile.player.mediasegments.MediaSegmentRepository
import org.jellyfin.mobile.player.queue.QueueManager
import org.jellyfin.mobile.player.source.JellyfinMediaSource
import org.jellyfin.mobile.player.ui.DecoderType
Expand All @@ -47,7 +49,9 @@
import org.jellyfin.mobile.utils.Constants.SUPPORTED_VIDEO_PLAYER_PLAYBACK_ACTIONS
import org.jellyfin.mobile.utils.applyDefaultAudioAttributes
import org.jellyfin.mobile.utils.applyDefaultLocalAudioAttributes
import org.jellyfin.mobile.utils.extensions.end
import org.jellyfin.mobile.utils.extensions.scaleInRange
import org.jellyfin.mobile.utils.extensions.start
import org.jellyfin.mobile.utils.extensions.width
import org.jellyfin.mobile.utils.getVolumeLevelPercent
import org.jellyfin.mobile.utils.getVolumeRange
Expand All @@ -65,6 +69,7 @@
import org.jellyfin.sdk.api.operations.HlsSegmentApi
import org.jellyfin.sdk.api.operations.PlayStateApi
import org.jellyfin.sdk.api.operations.UserApi
import org.jellyfin.sdk.model.api.MediaSegmentDto
import org.jellyfin.sdk.model.api.PlayMethod
import org.jellyfin.sdk.model.api.PlaybackOrder
import org.jellyfin.sdk.model.api.PlaybackProgressInfo
Expand Down Expand Up @@ -96,6 +101,7 @@
val queueManager = QueueManager(this)
val mediaSourceOrNull: JellyfinMediaSource?
get() = queueManager.currentMediaSourceOrNull
private val mediaSegmentRepository = MediaSegmentRepository()

// ExoPlayer
private val _player = MutableLiveData<ExoPlayer?>()
Expand Down Expand Up @@ -264,6 +270,9 @@

val startTime = jellyfinMediaSource.startTimeMs
if (startTime > 0) player.seekTo(startTime)

applyMediaSegments(jellyfinMediaSource)

player.playWhenReady = playWhenReady

mediaSession.setMetadata(jellyfinMediaSource.toMediaMetadata())
Expand Down Expand Up @@ -403,6 +412,46 @@
}
}

private fun applyMediaSegments(jellyfinMediaSource: JellyfinMediaSource) {
viewModelScope.launch {
if (jellyfinMediaSource.item != null) {
val mediaSegments = runCatching {
mediaSegmentRepository.getSegmentsForItem(jellyfinMediaSource.item)
}.getOrNull().orEmpty()

for (mediaSegment in mediaSegments) {
val action = mediaSegmentRepository.getMediaSegmentAction(mediaSegment)

when(action) {

Check warning

Code scanning / detekt

Reports spaces around keywords Warning

Missing spacing after "when"
MediaSegmentAction.SKIP -> addSkipAction(mediaSegment)
MediaSegmentAction.NOTHING -> Unit
// Unimplemented
MediaSegmentAction.ASK_TO_SKIP -> Unit
}
}
}
}
}

private fun addSkipAction(mediaSegment: MediaSegmentDto) {
val player = playerOrNull ?: return

player.createMessage { _, _ ->
// We can't seek directly on the ExoPlayer instance as not all media is seekable

Check warning

Code scanning / detekt

Reports mis-indented code Warning

Unexpected indentation (16) (should be 12)
// the seek function in the PlaybackController checks this and optionally starts a transcode

Check warning

Code scanning / detekt

Reports mis-indented code Warning

Unexpected indentation (16) (should be 12)
// at the requested position

Check warning

Code scanning / detekt

Reports mis-indented code Warning

Unexpected indentation (16) (should be 12)
// TODO: The above is probably true for jellyfin-android as well.

Check warning

Code scanning / detekt

Reports mis-indented code Warning

Unexpected indentation (16) (should be 12)
// But I believe there is no such logic here.

Check warning

Code scanning / detekt

Reports mis-indented code Warning

Unexpected indentation (16) (should be 12)
viewModelScope.launch(Dispatchers.Main) {

Check warning

Code scanning / detekt

Reports mis-indented code Warning

Unexpected indentation (16) (should be 12)
player.seekTo(mediaSegment.end.inWholeMilliseconds)

Check warning

Code scanning / detekt

Reports mis-indented code Warning

Unexpected indentation (20) (should be 16)
}

Check warning

Code scanning / detekt

Reports mis-indented code Warning

Unexpected indentation (16) (should be 12)
}

Check warning

Code scanning / detekt

Reports mis-indented code Warning

Unexpected indentation (12) (should be 8)
// Segments at position 0 will never be hit by ExoPlayer so we need to add a minimum value
.setPosition(mediaSegment.start.inWholeMilliseconds.coerceAtLeast(1))
.setDeleteAfterDelivery(false)
.send()
}

// Player controls

fun play() {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
package org.jellyfin.mobile.player.mediasegments

enum class MediaSegmentAction {
/**
* Don't take any action for this segment.
*/
NOTHING,

/**
* Seek to the end of this segment (endTicks). If the duration of this segment is shorter than 1 second it should do nothing to avoid
* lag. The skip action will only execute when playing over the segment start, not when seeking into the segment block.
*/
SKIP,

/**
* Ask the user if they want to skip this segment. When the user agrees this behaves like [SKIP]. Confirmation should only be asked for
* segments with a duration of at least 3 seconds to avoid UI flickering.
*/
ASK_TO_SKIP,
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
package org.jellyfin.mobile.player.mediasegments

import org.jellyfin.mobile.app.AppPreferences
import org.jellyfin.mobile.utils.extensions.duration
import org.jellyfin.sdk.api.client.ApiClient
import org.jellyfin.sdk.api.client.extensions.mediaSegmentsApi
import org.jellyfin.sdk.api.operations.MediaSegmentsApi
import org.jellyfin.sdk.model.api.BaseItemDto
import org.jellyfin.sdk.model.api.MediaSegmentDto
import org.jellyfin.sdk.model.api.MediaSegmentType
import org.koin.core.component.KoinComponent
import org.koin.core.component.get
import org.koin.core.component.inject
import kotlin.time.Duration.Companion.seconds

fun Map<MediaSegmentType, MediaSegmentAction>.toMediaSegmentActionsString() =
map { "${it.key.serialName}=${it.value.name}" }
.joinToString(",")

class MediaSegmentRepository : KoinComponent {
companion object {
/**
* All media segments currently supported by the app.
*/
val SupportedTypes = listOf(
MediaSegmentType.INTRO,
MediaSegmentType.OUTRO,
MediaSegmentType.PREVIEW,
MediaSegmentType.RECAP,
MediaSegmentType.COMMERCIAL,
)

/**
* The minimum duration for a media segment to allow the [MediaSegmentAction.SKIP] action.
*/
val SkipMinDuration = 1.seconds
}

private val appPreferences: AppPreferences by inject()
private val apiClient: ApiClient = get()
private val mediaSegmentsApi: MediaSegmentsApi = apiClient.mediaSegmentsApi

private val mediaTypeActions = mutableMapOf<MediaSegmentType, MediaSegmentAction>()

init {
restoreMediaTypeActions()
}

private fun restoreMediaTypeActions() {
val restoredMediaTypeActions = appPreferences.mediaSegmentActions
.split(",")
.mapNotNull {
runCatching {
val (type, action) = it.split('=', limit = 2)
MediaSegmentType.fromName(type) to MediaSegmentAction.valueOf(action)
}.getOrNull()
}

mediaTypeActions.clear()
mediaTypeActions.putAll(restoredMediaTypeActions)
}

private fun saveMediaTypeActions() {
appPreferences.mediaSegmentActions = mediaTypeActions.toMediaSegmentActionsString()
}

fun getDefaultSegmentTypeAction(type: MediaSegmentType): MediaSegmentAction {
// Always return no action for unsupported types
if (!SupportedTypes.contains(type)) return MediaSegmentAction.NOTHING

return mediaTypeActions.getOrDefault(type, MediaSegmentAction.NOTHING)
}

fun setDefaultSegmentTypeAction(type: MediaSegmentType, action: MediaSegmentAction) {
// Don't allow modifying actions for unsupported types
if (!SupportedTypes.contains(type)) return

mediaTypeActions[type] = action
saveMediaTypeActions()
}

fun getMediaSegmentAction(segment: MediaSegmentDto): MediaSegmentAction {
val action = getDefaultSegmentTypeAction(segment.type)
// Skip the skip action if timespan is too short
if (action == MediaSegmentAction.SKIP && segment.duration < SkipMinDuration) return MediaSegmentAction.NOTHING
return action
}

suspend fun getSegmentsForItem(item: BaseItemDto): List<MediaSegmentDto> = runCatching {
mediaSegmentsApi.getItemSegments(
itemId = item.id,
includeSegmentTypes = SupportedTypes,
).content.items
}.getOrDefault(emptyList())
}
1 change: 1 addition & 0 deletions app/src/main/java/org/jellyfin/mobile/utils/Constants.kt
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ object Constants {
const val PREF_EXTERNAL_PLAYER_APP = "pref_external_player_app"
const val PREF_SUBTITLE_STYLE = "pref_subtitle_style"
const val PREF_DOWNLOAD_LOCATION = "pref_download_location"
const val PREF_MEDIA_SEGMENT_ACTIONS = "pref_media_segment_actions"

// InputManager commands
const val PLAYBACK_MANAGER_COMMAND_PLAY = "unpause"
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
package org.jellyfin.mobile.utils.extensions

import org.jellyfin.sdk.model.api.MediaSegmentDto
import org.jellyfin.sdk.model.extensions.ticks
import kotlin.time.Duration

val MediaSegmentDto.start get() = startTicks.ticks
val MediaSegmentDto.end get() = endTicks.ticks

val MediaSegmentDto.duration get() = (endTicks - startTicks).ticks.coerceAtLeast(Duration.ZERO)
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import org.jellyfin.mobile.R
import org.jellyfin.mobile.app.ApiClientController
import org.jellyfin.mobile.app.AppPreferences
import org.jellyfin.mobile.bridge.ExternalPlayer
import org.jellyfin.mobile.bridge.MediaSegments
import org.jellyfin.mobile.bridge.NativeInterface
import org.jellyfin.mobile.bridge.NativePlayer
import org.jellyfin.mobile.data.entity.ServerEntity
Expand Down Expand Up @@ -192,6 +193,7 @@ class WebViewFragment : Fragment(), BackPressInterceptor, JellyfinWebChromeClien
addJavascriptInterface(NativeInterface(requireContext()), "NativeInterface")
addJavascriptInterface(nativePlayer, "NativePlayer")
addJavascriptInterface(externalPlayer, "ExternalPlayer")
addJavascriptInterface(MediaSegments(requireContext()), "MediaSegments")

loadUrl(server.hostname)
postDelayed(timeoutRunnable, Constants.INITIAL_CONNECTION_TIMEOUT)
Expand Down
Loading