From bfecd1cc5fd1e60b6305978d053c8021c6bc7084 Mon Sep 17 00:00:00 2001 From: Niels van Velzen Date: Tue, 22 Aug 2023 16:59:16 +0200 Subject: [PATCH] Add initial playback capability testing --- .../src/main/kotlin/backend/PlayerBackend.kt | 4 + .../main/kotlin/mediastream/MediaStream.kt | 19 + .../kotlin/mediastream/MediaStreamState.kt | 15 +- .../main/kotlin/support/PlaySupportReport.kt | 5 + .../src/main/kotlin/ExoPlayerBackend.kt | 7 + .../src/main/kotlin/mapping/audio.kt | 206 +++++++++ .../src/main/kotlin/mapping/container.kt | 412 ++++++++++++++++++ .../main/kotlin/support/AdaptiveSupport.kt | 18 + .../src/main/kotlin/support/DecoderSupport.kt | 18 + .../support/ExoPlayerPlaySupportReport.kt | 59 +++ .../src/main/kotlin/support/FormatSupport.kt | 22 + .../kotlin/support/mediaStreamToFormat.kt | 20 + .../mediastream/AudioMediaStreamResolver.kt | 8 +- .../UniversalAudioMediaStreamResolver.kt | 2 + .../src/main/kotlin/mediastream/tracks.kt | 38 ++ 15 files changed, 848 insertions(+), 5 deletions(-) create mode 100644 playback/core/src/main/kotlin/support/PlaySupportReport.kt create mode 100644 playback/exoplayer/src/main/kotlin/mapping/audio.kt create mode 100644 playback/exoplayer/src/main/kotlin/mapping/container.kt create mode 100644 playback/exoplayer/src/main/kotlin/support/AdaptiveSupport.kt create mode 100644 playback/exoplayer/src/main/kotlin/support/DecoderSupport.kt create mode 100644 playback/exoplayer/src/main/kotlin/support/ExoPlayerPlaySupportReport.kt create mode 100644 playback/exoplayer/src/main/kotlin/support/FormatSupport.kt create mode 100644 playback/exoplayer/src/main/kotlin/support/mediaStreamToFormat.kt create mode 100644 playback/jellyfin/src/main/kotlin/mediastream/tracks.kt diff --git a/playback/core/src/main/kotlin/backend/PlayerBackend.kt b/playback/core/src/main/kotlin/backend/PlayerBackend.kt index b6ba9bb294..57d8a0e7ee 100644 --- a/playback/core/src/main/kotlin/backend/PlayerBackend.kt +++ b/playback/core/src/main/kotlin/backend/PlayerBackend.kt @@ -2,6 +2,7 @@ package org.jellyfin.playback.core.backend import org.jellyfin.playback.core.mediastream.MediaStream import org.jellyfin.playback.core.model.PositionInfo +import org.jellyfin.playback.core.support.PlaySupportReport import kotlin.time.Duration /** @@ -9,6 +10,9 @@ import kotlin.time.Duration * preload items. */ interface PlayerBackend { + // Testing + fun supportsStream(stream: MediaStream): PlaySupportReport + // Data retrieval fun setListener(eventListener: PlayerBackendEventListener?) diff --git a/playback/core/src/main/kotlin/mediastream/MediaStream.kt b/playback/core/src/main/kotlin/mediastream/MediaStream.kt index 870ce2796c..fdd49ca1d0 100644 --- a/playback/core/src/main/kotlin/mediastream/MediaStream.kt +++ b/playback/core/src/main/kotlin/mediastream/MediaStream.kt @@ -7,4 +7,23 @@ data class MediaStream( val queueEntry: QueueEntry, val conversionMethod: MediaConversionMethod, val url: String, + val container: MediaStreamContainer, + val tracks: Collection, ) + +data class MediaStreamContainer( + val format: String, +) + +sealed interface MediaStreamTrack { + val codec: String +} + +data class MediaStreamAudioTrack( + override val codec: String, + val bitrate: Int, + val channels: Int, + val sampleRate: Int, +) : MediaStreamTrack + +// TODO: Add Video/Subtitle tracks diff --git a/playback/core/src/main/kotlin/mediastream/MediaStreamState.kt b/playback/core/src/main/kotlin/mediastream/MediaStreamState.kt index 46023cc140..4cd6348ad5 100644 --- a/playback/core/src/main/kotlin/mediastream/MediaStreamState.kt +++ b/playback/core/src/main/kotlin/mediastream/MediaStreamState.kt @@ -39,10 +39,17 @@ class DefaultMediaStreamState( val streamResult = runCatching { mediaStreamResolvers.firstNotNullOfOrNull { resolver -> resolver.getStream(entry) } } + val stream = streamResult.getOrNull() when { streamResult.isFailure -> Timber.e(streamResult.exceptionOrNull(), "Media stream resolver failed for $entry") - streamResult.getOrNull() == null -> Timber.e("Unable to resolve stream for entry $entry") - else -> setCurrent(streamResult.getOrThrow()) + stream == null -> Timber.e("Unable to resolve stream for entry $entry") + else -> { + if (!canPlayStream(stream)) { + Timber.w("Playback of the received media stream for $entry is not supported") + } + + setCurrent(stream) + } } } }.launchIn(coroutineScope) @@ -50,6 +57,10 @@ class DefaultMediaStreamState( // TODO Register some kind of event when $current item is at -30 seconds to setNext() } + private suspend fun canPlayStream(stream: MediaStream) = withContext(Dispatchers.Main) { + backendService.backend?.supportsStream(stream)?.canPlay == true + } + private suspend fun setCurrent(stream: MediaStream?) { Timber.d("Current stream changed to $stream") val backend = requireNotNull(backendService.backend) diff --git a/playback/core/src/main/kotlin/support/PlaySupportReport.kt b/playback/core/src/main/kotlin/support/PlaySupportReport.kt new file mode 100644 index 0000000000..b7eb6d827d --- /dev/null +++ b/playback/core/src/main/kotlin/support/PlaySupportReport.kt @@ -0,0 +1,5 @@ +package org.jellyfin.playback.core.support + +interface PlaySupportReport { + val canPlay: Boolean +} diff --git a/playback/exoplayer/src/main/kotlin/ExoPlayerBackend.kt b/playback/exoplayer/src/main/kotlin/ExoPlayerBackend.kt index d73b9efea1..f9a0fb439b 100644 --- a/playback/exoplayer/src/main/kotlin/ExoPlayerBackend.kt +++ b/playback/exoplayer/src/main/kotlin/ExoPlayerBackend.kt @@ -12,6 +12,9 @@ import org.jellyfin.playback.core.backend.BasePlayerBackend import org.jellyfin.playback.core.mediastream.MediaStream import org.jellyfin.playback.core.model.PlayState import org.jellyfin.playback.core.model.PositionInfo +import org.jellyfin.playback.core.support.PlaySupportReport +import org.jellyfin.playback.exoplayer.support.getPlaySupportReport +import org.jellyfin.playback.exoplayer.support.toFormat import timber.log.Timber import kotlin.time.Duration import kotlin.time.Duration.Companion.ZERO @@ -67,6 +70,10 @@ class ExoPlayerBackend( } } + override fun supportsStream( + stream: MediaStream + ): PlaySupportReport = exoPlayer.getPlaySupportReport(stream.toFormat()) + override fun prepareStream(stream: MediaStream) { val mediaItem = MediaItem.Builder().apply { setTag(stream) diff --git a/playback/exoplayer/src/main/kotlin/mapping/audio.kt b/playback/exoplayer/src/main/kotlin/mapping/audio.kt new file mode 100644 index 0000000000..83e28af192 --- /dev/null +++ b/playback/exoplayer/src/main/kotlin/mapping/audio.kt @@ -0,0 +1,206 @@ +package org.jellyfin.playback.exoplayer.mapping + +import com.google.android.exoplayer2.util.MimeTypes + +fun getFfmpegAudioMimeType(codec: String): String { + return ffmpegAudioMimeTypes.getOrDefault(codec, codec) +} + +val ffmpegAudioMimeTypes = mapOf( + "aac" to MimeTypes.AUDIO_AAC, + "ac3" to MimeTypes.AUDIO_AC3, + "alac" to MimeTypes.AUDIO_ALAC, + "amrnb" to MimeTypes.AUDIO_AMR_NB, + "amrwb" to MimeTypes.AUDIO_AMR_WB, + "dca" to MimeTypes.AUDIO_DTS, + "eac3" to MimeTypes.AUDIO_E_AC3, + "flac" to MimeTypes.AUDIO_FLAC, + "mp1" to MimeTypes.AUDIO_MPEG_L1, + "mp2" to MimeTypes.AUDIO_MPEG_L2, + "mp3" to MimeTypes.AUDIO_MPEG, + "opus" to MimeTypes.AUDIO_OPUS, + "pcm_alaw" to MimeTypes.AUDIO_ALAW, + "pcm_mulaw" to MimeTypes.AUDIO_MLAW, + "truehd" to MimeTypes.AUDIO_TRUEHD, + "vorbis" to MimeTypes.AUDIO_VORBIS, +// TODO: Find mime types for all these codecs... +// "4gv" to MimeTypes.AUDIO_4GV, +// "8svx_exp" to MimeTypes.AUDIO_8SVX_EXP, +// "8svx_fib" to MimeTypes.AUDIO_8SVX_FIB, +// "aac_latm" to MimeTypes.AUDIO_AAC_LATM, +// "acelp" to MimeTypes.AUDIO_ACELP, +// "adpcm_4xm" to MimeTypes.AUDIO_ADPCM_4_XM, +// "adpcm_adx" to MimeTypes.AUDIO_ADPCM_ADX, +// "adpcm_afc" to MimeTypes.AUDIO_ADPCM_AFC, +// "adpcm_agm" to MimeTypes.AUDIO_ADPCM_AGM, +// "adpcm_aica" to MimeTypes.AUDIO_ADPCM_AICA, +// "adpcm_argo" to MimeTypes.AUDIO_ADPCM_ARGO, +// "adpcm_ct" to MimeTypes.AUDIO_ADPCM_CT, +// "adpcm_dtk" to MimeTypes.AUDIO_ADPCM_DTK, +// "adpcm_ea" to MimeTypes.AUDIO_ADPCM_EA, +// "adpcm_ea_maxis_xa" to MimeTypes.AUDIO_ADPCM_EA_MAXIS_XA, +// "adpcm_ea_r1" to MimeTypes.AUDIO_ADPCM_EA_R_1, +// "adpcm_ea_r2" to MimeTypes.AUDIO_ADPCM_EA_R_2, +// "adpcm_ea_r3" to MimeTypes.AUDIO_ADPCM_EA_R_3, +// "adpcm_ea_xas" to MimeTypes.AUDIO_ADPCM_EA_XAS, +// "adpcm_g722" to MimeTypes.AUDIO_ADPCM_G_722, +// "adpcm_g726" to MimeTypes.AUDIO_ADPCM_G_726, +// "adpcm_g726le" to MimeTypes.AUDIO_ADPCM_G_726_LE, +// "adpcm_ima_acorn" to MimeTypes.AUDIO_ADPCM_IMA_ACORN, +// "adpcm_ima_alp" to MimeTypes.AUDIO_ADPCM_IMA_ALP, +// "adpcm_ima_amv" to MimeTypes.AUDIO_ADPCM_IMA_AMV, +// "adpcm_ima_apc" to MimeTypes.AUDIO_ADPCM_IMA_APC, +// "adpcm_ima_apm" to MimeTypes.AUDIO_ADPCM_IMA_APM, +// "adpcm_ima_cunning" to MimeTypes.AUDIO_ADPCM_IMA_CUNNING, +// "adpcm_ima_dat4" to MimeTypes.AUDIO_ADPCM_IMA_DAT_4, +// "adpcm_ima_dk3" to MimeTypes.AUDIO_ADPCM_IMA_DK_3, +// "adpcm_ima_dk4" to MimeTypes.AUDIO_ADPCM_IMA_DK_4, +// "adpcm_ima_ea_eacs" to MimeTypes.AUDIO_ADPCM_IMA_EA_EACS, +// "adpcm_ima_ea_sead" to MimeTypes.AUDIO_ADPCM_IMA_EA_SEAD, +// "adpcm_ima_iss" to MimeTypes.AUDIO_ADPCM_IMA_ISS, +// "adpcm_ima_moflex" to MimeTypes.AUDIO_ADPCM_IMA_MOFLEX, +// "adpcm_ima_mtf" to MimeTypes.AUDIO_ADPCM_IMA_MTF, +// "adpcm_ima_oki" to MimeTypes.AUDIO_ADPCM_IMA_OKI, +// "adpcm_ima_qt" to MimeTypes.AUDIO_ADPCM_IMA_QT, +// "adpcm_ima_rad" to MimeTypes.AUDIO_ADPCM_IMA_RAD, +// "adpcm_ima_smjpeg" to MimeTypes.AUDIO_ADPCM_IMA_SMJPEG, +// "adpcm_ima_ssi" to MimeTypes.AUDIO_ADPCM_IMA_SSI, +// "adpcm_ima_wav" to MimeTypes.AUDIO_ADPCM_IMA_WAV, +// "adpcm_ima_ws" to MimeTypes.AUDIO_ADPCM_IMA_WS, +// "adpcm_ms" to MimeTypes.AUDIO_ADPCM_MS, +// "adpcm_mtaf" to MimeTypes.AUDIO_ADPCM_MTAF, +// "adpcm_psx" to MimeTypes.AUDIO_ADPCM_PSX, +// "adpcm_sbpro_2" to MimeTypes.AUDIO_ADPCM_SBPRO_2, +// "adpcm_sbpro_3" to MimeTypes.AUDIO_ADPCM_SBPRO_3, +// "adpcm_sbpro_4" to MimeTypes.AUDIO_ADPCM_SBPRO_4, +// "adpcm_swf" to MimeTypes.AUDIO_ADPCM_SWF, +// "adpcm_thp" to MimeTypes.AUDIO_ADPCM_THP, +// "adpcm_thp_le" to MimeTypes.AUDIO_ADPCM_THP_LE, +// "adpcm_vima" to MimeTypes.AUDIO_ADPCM_VIMA, +// "adpcm_xa" to MimeTypes.AUDIO_ADPCM_XA, +// "adpcm_yamaha" to MimeTypes.AUDIO_ADPCM_YAMAHA, +// "adpcm_zork" to MimeTypes.AUDIO_ADPCM_ZORK, +// "ape" to MimeTypes.AUDIO_APE, +// "aptx" to MimeTypes.AUDIO_APTX, +// "aptx_hd" to MimeTypes.AUDIO_APTX_HD, +// "atrac1" to MimeTypes.AUDIO_ATRAC_1, +// "atrac3" to MimeTypes.AUDIO_ATRAC_3, +// "atrac3al" to MimeTypes.AUDIO_ATRAC_3_AL, +// "atrac3p" to MimeTypes.AUDIO_ATRAC_3_P, +// "atrac3pal" to MimeTypes.AUDIO_ATRAC_3_PAL, +// "atrac9" to MimeTypes.AUDIO_ATRAC_9, +// "avc" to MimeTypes.AUDIO_AVC, +// "binkaudio_dct" to MimeTypes.AUDIO_BINKAUDIO_DCT, +// "binkaudio_rdft" to MimeTypes.AUDIO_BINKAUDIO_RDFT, +// "bmv_audio" to MimeTypes.AUDIO_BMV_AUDIO, +// "celt" to MimeTypes.AUDIO_CELT, +// "codec2" to MimeTypes.AUDIO_CODEC_2, +// "comfortnoise" to MimeTypes.AUDIO_COMFORTNOISE, +// "cook" to MimeTypes.AUDIO_COOK, +// "derf_dpcm" to MimeTypes.AUDIO_DERF_DPCM, +// "dfpwm" to MimeTypes.AUDIO_DFPWM, +// "dolby_e" to MimeTypes.AUDIO_DOLBY_E, +// "dsd_lsbf" to MimeTypes.AUDIO_DSD_LSBF, +// "dsd_lsbf_planar" to MimeTypes.AUDIO_DSD_LSBF_PLANAR, +// "dsd_msbf" to MimeTypes.AUDIO_DSD_MSBF, +// "dsd_msbf_planar" to MimeTypes.AUDIO_DSD_MSBF_PLANAR, +// "dsicinaudio" to MimeTypes.AUDIO_DSICINAUDIO, +// "dss_sp" to MimeTypes.AUDIO_DSS_SP, +// "dst" to MimeTypes.AUDIO_DST, +// "dvaudio" to MimeTypes.AUDIO_DVAUDIO, +// "evrc" to MimeTypes.AUDIO_EVRC, +// "fastaudio" to MimeTypes.AUDIO_FASTAUDIO, +// "g723_1" to MimeTypes.AUDIO_G_723_1, +// "g729" to MimeTypes.AUDIO_G_729, +// "gremlin_dpcm" to MimeTypes.AUDIO_GREMLIN_DPCM, +// "gsm" to MimeTypes.AUDIO_GSM, +// "gsm_ms" to MimeTypes.AUDIO_GSM_MS, +// "hca" to MimeTypes.AUDIO_HCA, +// "hcom" to MimeTypes.AUDIO_HCOM, +// "iac" to MimeTypes.AUDIO_IAC, +// "ilbc" to MimeTypes.AUDIO_ILBC, +// "imc" to MimeTypes.AUDIO_IMC, +// "interplay_dpcm" to MimeTypes.AUDIO_INTERPLAY_DPCM, +// "interplayacm" to MimeTypes.AUDIO_INTERPLAYACM, +// "mace3" to MimeTypes.AUDIO_MACE_3, +// "mace6" to MimeTypes.AUDIO_MACE_6, +// "metasound" to MimeTypes.AUDIO_METASOUND, +// "mlp" to MimeTypes.AUDIO_MLP, +// "mp3adu" to MimeTypes.AUDIO_MP_3_ADU, +// "mp3on4" to MimeTypes.AUDIO_MP_3_ON_4, +// "mp4als" to MimeTypes.AUDIO_MP_4_ALS, +// "mpegh_3d_audio" to MimeTypes.AUDIO_MPEGH_3_D_AUDIO, +// "msnsiren" to MimeTypes.AUDIO_MSNSIREN, +// "musepack7" to MimeTypes.AUDIO_MUSEPACK_7, +// "musepack8" to MimeTypes.AUDIO_MUSEPACK_8, +// "nellymoser" to MimeTypes.AUDIO_NELLYMOSER, +// "paf_audio" to MimeTypes.AUDIO_PAF_AUDIO, +// "pcm_bluray" to MimeTypes.AUDIO_PCM_BLURAY, +// "pcm_dvd" to MimeTypes.AUDIO_PCM_DVD, +// "pcm_f16le" to MimeTypes.AUDIO_PCM_F_16_LE, +// "pcm_f24le" to MimeTypes.AUDIO_PCM_F_24_LE, +// "pcm_f32be" to MimeTypes.AUDIO_PCM_F_32_BE, +// "pcm_f32le" to MimeTypes.AUDIO_PCM_F_32_LE, +// "pcm_f64be" to MimeTypes.AUDIO_PCM_F_64_BE, +// "pcm_f64le" to MimeTypes.AUDIO_PCM_F_64_LE, +// "pcm_lxf" to MimeTypes.AUDIO_PCM_LXF, +// "pcm_s8" to MimeTypes.AUDIO_PCM_S_8, +// "pcm_s8_planar" to MimeTypes.AUDIO_PCM_S_8_PLANAR, +// "pcm_s16be" to MimeTypes.AUDIO_PCM_S_16_BE, +// "pcm_s16be_planar" to MimeTypes.AUDIO_PCM_S_16_BE_PLANAR, +// "pcm_s16le" to MimeTypes.AUDIO_PCM_S_16_LE, +// "pcm_s16le_planar" to MimeTypes.AUDIO_PCM_S_16_LE_PLANAR, +// "pcm_s24be" to MimeTypes.AUDIO_PCM_S_24_BE, +// "pcm_s24daud" to MimeTypes.AUDIO_PCM_S_24_DAUD, +// "pcm_s24le" to MimeTypes.AUDIO_PCM_S_24_LE, +// "pcm_s24le_planar" to MimeTypes.AUDIO_PCM_S_24_LE_PLANAR, +// "pcm_s32be" to MimeTypes.AUDIO_PCM_S_32_BE, +// "pcm_s32le" to MimeTypes.AUDIO_PCM_S_32_LE, +// "pcm_s32le_planar" to MimeTypes.AUDIO_PCM_S_32_LE_PLANAR, +// "pcm_s64be" to MimeTypes.AUDIO_PCM_S_64_BE, +// "pcm_s64le" to MimeTypes.AUDIO_PCM_S_64_LE, +// "pcm_sga" to MimeTypes.AUDIO_PCM_SGA, +// "pcm_u8" to MimeTypes.AUDIO_PCM_U_8, +// "pcm_u16be" to MimeTypes.AUDIO_PCM_U_16_BE, +// "pcm_u16le" to MimeTypes.AUDIO_PCM_U_16_LE, +// "pcm_u24be" to MimeTypes.AUDIO_PCM_U_24_BE, +// "pcm_u24le" to MimeTypes.AUDIO_PCM_U_24_LE, +// "pcm_u32be" to MimeTypes.AUDIO_PCM_U_32_BE, +// "pcm_u32le" to MimeTypes.AUDIO_PCM_U_32_LE, +// "pcm_vidc" to MimeTypes.AUDIO_PCM_VIDC, +// "qcelp" to MimeTypes.AUDIO_QCELP, +// "qdm2" to MimeTypes.AUDIO_QDM_2, +// "qdmc" to MimeTypes.AUDIO_QDMC, +// "ra_144" to MimeTypes.AUDIO_RA_144, +// "ra_288" to MimeTypes.AUDIO_RA_288, +// "ralf" to MimeTypes.AUDIO_RALF, +// "roq_dpcm" to MimeTypes.AUDIO_ROQ_DPCM, +// "s302m" to MimeTypes.AUDIO_S_302_M, +// "sbc" to MimeTypes.AUDIO_SBC, +// "sdx2_dpcm" to MimeTypes.AUDIO_SDX_2_DPCM, +// "shorten" to MimeTypes.AUDIO_SHORTEN, +// "sipr" to MimeTypes.AUDIO_SIPR, +// "siren" to MimeTypes.AUDIO_SIREN, +// "smackaudio" to MimeTypes.AUDIO_SMACKAUDIO, +// "smv" to MimeTypes.AUDIO_SMV, +// "sol_dpcm" to MimeTypes.AUDIO_SOL_DPCM, +// "sonic" to MimeTypes.AUDIO_SONIC, +// "sonicls" to MimeTypes.AUDIO_SONICLS, +// "speex" to MimeTypes.AUDIO_SPEEX, +// "tak" to MimeTypes.AUDIO_TAK, +// "truespeech" to MimeTypes.AUDIO_TRUESPEECH, +// "tta" to MimeTypes.AUDIO_TTA, +// "twinvq" to MimeTypes.AUDIO_TWINVQ, +// "vmdaudio" to MimeTypes.AUDIO_VMDAUDIO, +// "wavesynth" to MimeTypes.AUDIO_WAVESYNTH, +// "wavpack" to MimeTypes.AUDIO_WAVPACK, +// "westwood_snd1" to MimeTypes.AUDIO_WESTWOOD_SND_1, +// "wmalossless" to MimeTypes.AUDIO_WMALOSSLESS, +// "wmapro" to MimeTypes.AUDIO_WMAPRO, +// "wmav1" to MimeTypes.AUDIO_WMAV_1, +// "wmav2" to MimeTypes.AUDIO_WMAV_2, +// "wmavoice" to MimeTypes.AUDIO_WMAVOICE, +// "xan_dpcm" to MimeTypes.AUDIO_XAN_DPCM, +// "xma1" to MimeTypes.AUDIO_XMA_1, +// "xma2" to MimeTypes.AUDIO_XMA_2, +) diff --git a/playback/exoplayer/src/main/kotlin/mapping/container.kt b/playback/exoplayer/src/main/kotlin/mapping/container.kt new file mode 100644 index 0000000000..1bf02bc8ad --- /dev/null +++ b/playback/exoplayer/src/main/kotlin/mapping/container.kt @@ -0,0 +1,412 @@ +package org.jellyfin.playback.exoplayer.mapping + +import com.google.android.exoplayer2.util.MimeTypes + +fun getFfmpegContainerMimeType(codec: String): String { + // Find in container mime type list + return ffmpegContainerMimeTypes.getOrElse(codec) { + // Find in audio mime type list + ffmpegAudioMimeTypes.getOrElse(codec) { + // Return input + codec + } + } +} + +val ffmpegContainerMimeTypes = mapOf( + "aac" to MimeTypes.AUDIO_AAC, + "alaw" to MimeTypes.AUDIO_ALAW, + "amr" to MimeTypes.AUDIO_AMR, + "avi" to MimeTypes.VIDEO_AVI, + "dts" to MimeTypes.AUDIO_DTS, + "flac" to MimeTypes.AUDIO_FLAC, + "flv" to MimeTypes.VIDEO_FLV, + "matroska" to MimeTypes.APPLICATION_MATROSKA, + "mjpeg" to MimeTypes.VIDEO_MJPEG, + "mpeg" to MimeTypes.AUDIO_MPEG, + "ogg" to MimeTypes.AUDIO_OGG, + "opus" to MimeTypes.AUDIO_OPUS, + "rtsp" to MimeTypes.APPLICATION_RTSP, + "truehd" to MimeTypes.AUDIO_TRUEHD, + "ttml" to MimeTypes.APPLICATION_TTML, + "vobsub" to MimeTypes.APPLICATION_VOBSUB, + "wav" to MimeTypes.AUDIO_WAV, + "webm" to MimeTypes.APPLICATION_WEBM, +// TODO: Find mime types for all these formats... +// "3dostr" to null, +// "3g2" to null, +// "3gp" to null, +// "4xm" to null, +// "a64" to null, +// "aa" to null, +// "aax" to null, +// "ac3" to null, +// "ace" to null, +// "acm" to null, +// "act" to null, +// "adf" to null, +// "adp" to null, +// "ads" to null, +// "adts" to null, +// "adx" to null, +// "aea" to null, +// "afc" to null, +// "aiff" to null, +// "aix" to null, +// "alias_pix" to null, +// "alp" to null, +// "amrnb" to null, +// "amrwb" to null, +// "amv" to null, +// "anm" to null, +// "apc" to null, +// "ape" to null, +// "apm" to null, +// "apng" to null, +// "aptx" to null, +// "aptx_hd" to null, +// "aqtitle" to null, +// "argo_asf" to null, +// "argo_brp" to null, +// "argo_cvg" to null, +// "asf" to null, +// "asf_o" to null, +// "asf_stream" to null, +// "ass" to null, +// "ast" to null, +// "au" to null, +// "av1" to null, +// "avif" to null, +// "avm2" to null, +// "avr" to null, +// "avs" to null, +// "avs2" to null, +// "avs3" to null, +// "bethsoftvid" to null, +// "bfi" to null, +// "bfstm" to null, +// "bin" to null, +// "bink" to null, +// "binka" to null, +// "bit" to null, +// "bitpacked" to null, +// "bmp_pipe" to null, +// "bmv" to null, +// "boa" to null, +// "brender_pix" to null, +// "brstm" to null, +// "c93" to null, +// "caf" to null, +// "cavsvideo" to null, +// "cdg" to null, +// "cdxl" to null, +// "chromaprint" to null, +// "cine" to null, +// "codec2" to null, +// "codec2raw" to null, +// "concat" to null, +// "crc" to null, +// "cri_pipe" to null, +// "dash" to null, +// "data" to null, +// "daud" to null, +// "dcstr" to null, +// "dds_pipe" to null, +// "derf" to null, +// "dfa" to null, +// "dfpwm" to null, +// "dhav" to null, +// "dirac" to null, +// "dnxhd" to null, +// "dpx_pipe" to null, +// "dsf" to null, +// "dshow" to null, +// "dsicin" to null, +// "dss" to null, +// "dtshd" to null, +// "dv" to null, +// "dvbsub" to null, +// "dvbtxt" to null, +// "dvd" to null, +// "dxa" to null, +// "ea" to null, +// "ea_cdata" to null, +// "eac3" to null, +// "epaf" to null, +// "exr_pipe" to null, +// "f4v" to null, +// "f32be" to null, +// "f32le" to null, +// "f64be" to null, +// "f64le" to null, +// "ffmetadata" to null, +// "fifo" to null, +// "fifo_test" to null, +// "film_cpk" to null, +// "filmstrip" to null, +// "fits" to null, +// "flic" to null, +// "framecrc" to null, +// "framehash" to null, +// "framemd5" to null, +// "frm" to null, +// "fsb" to null, +// "fwse" to null, +// "g722" to null, +// "g723_1" to null, +// "g726" to null, +// "g726le" to null, +// "g729" to null, +// "gdigrab" to null, +// "gdv" to null, +// "gem_pipe" to null, +// "genh" to null, +// "gif" to null, +// "gif_pipe" to null, +// "gsm" to null, +// "gxf" to null, +// "h261" to null, +// "h263" to null, +// "h264" to null, +// "hash" to null, +// "hca" to null, +// "hcom" to null, +// "hds" to null, +// "hevc" to null, +// "hls" to null, +// "hnm" to null, +// "ico" to null, +// "idcin" to null, +// "idf" to null, +// "iff" to null, +// "ifv" to null, +// "ilbc" to null, +// "image2" to null, +// "image2pipe" to null, +// "imf" to null, +// "ingenient" to null, +// "ipmovie" to null, +// "ipod" to null, +// "ipu" to null, +// "ircam" to null, +// "ismv" to null, +// "iss" to null, +// "iv8" to null, +// "ivf" to null, +// "ivr" to null, +// "j2k_pipe" to null, +// "jacosub" to null, +// "jpeg_pipe" to null, +// "jpegls_pipe" to null, +// "jpegxl_pipe" to null, +// "jv" to null, +// "kux" to null, +// "kvag" to null, +// "latm" to null, +// "lavfi" to null, +// "libopenmpt" to null, +// "live_flv" to null, +// "lmlm4" to null, +// "loas" to null, +// "lrc" to null, +// "luodat" to null, +// "lvf" to null, +// "lxf" to null, +// "m4a" to null, +// "m4v" to null, +// "mca" to null, +// "mcc" to null, +// "md5" to null, +// "mgsts" to null, +// "microdvd" to null, +// "mj2" to null, +// "mjpeg_2000" to null, +// "mkvtimestamp_v2" to null, +// "mlp" to null, +// "mlv" to null, +// "mm" to null, +// "mmf" to null, +// "mods" to null, +// "moflex" to null, +// "mov" to null, +// "mp2" to null, +// "mp3" to null, +// "mp4" to null, +// "mpc" to null, +// "mpc8" to null, +// "mpeg1video" to null, +// "mpeg2video" to null, +// "mpegts" to null, +// "mpegtsraw" to null, +// "mpegvideo" to null, +// "mpjpeg" to null, +// "mpl2" to null, +// "mpsub" to null, +// "msf" to null, +// "msnwctcp" to null, +// "msp" to null, +// "mtaf" to null, +// "mtv" to null, +// "mulaw" to null, +// "musx" to null, +// "mv" to null, +// "mvi" to null, +// "mxf" to null, +// "mxf_d10" to null, +// "mxf_opatom" to null, +// "mxg" to null, +// "nc" to null, +// "nistsphere" to null, +// "nsp" to null, +// "nsv" to null, +// "null" to null, +// "nut" to null, +// "nuv" to null, +// "obu" to null, +// "oga" to null, +// "ogv" to null, +// "oma" to null, +// "paf" to null, +// "pam_pipe" to null, +// "pbm_pipe" to null, +// "pcx_pipe" to null, +// "pfm_pipe" to null, +// "pgm_pipe" to null, +// "pgmyuv_pipe" to null, +// "pgx_pipe" to null, +// "phm_pipe" to null, +// "photocd_pipe" to null, +// "pictor_pipe" to null, +// "pjs" to null, +// "pmp" to null, +// "png_pipe" to null, +// "pp_bnk" to null, +// "ppm_pipe" to null, +// "psd_pipe" to null, +// "psp" to null, +// "psxstr" to null, +// "pva" to null, +// "pvf" to null, +// "qcp" to null, +// "qdraw_pipe" to null, +// "qoi_pipe" to null, +// "r3d" to null, +// "rawvideo" to null, +// "realtext" to null, +// "redspark" to null, +// "rl2" to null, +// "rm" to null, +// "roq" to null, +// "rpl" to null, +// "rsd" to null, +// "rso" to null, +// "rtp" to null, +// "rtp_mpegts" to null, +// "s8" to null, +// "s16be" to null, +// "s16le" to null, +// "s24be" to null, +// "s24le" to null, +// "s32be" to null, +// "s32le" to null, +// "s337m" to null, +// "sami" to null, +// "sap" to null, +// "sbc" to null, +// "sbg" to null, +// "scc" to null, +// "scd" to null, +// "sdp" to null, +// "sdr2" to null, +// "sds" to null, +// "sdx" to null, +// "segment" to null, +// "ser" to null, +// "sga" to null, +// "sgi_pipe" to null, +// "shn" to null, +// "siff" to null, +// "simbiosis_imx" to null, +// "sln" to null, +// "smjpeg" to null, +// "smk" to null, +// "smoothstreaming" to null, +// "smush" to null, +// "sol" to null, +// "sox" to null, +// "spdif" to null, +// "spx" to null, +// "srt" to null, +// "ssegment" to null, +// "stl" to null, +// "stream_segment" to null, +// "streamhash" to null, +// "subviewer" to null, +// "subviewer1" to null, +// "sunrast_pipe" to null, +// "sup" to null, +// "svag" to null, +// "svcd" to null, +// "svg_pipe" to null, +// "svs" to null, +// "swf" to null, +// "tak" to null, +// "tedcaptions" to null, +// "tee" to null, +// "thp" to null, +// "tiertexseq" to null, +// "tiff_pipe" to null, +// "tmv" to null, +// "tta" to null, +// "tty" to null, +// "txd" to null, +// "ty" to null, +// "u8" to null, +// "u16be" to null, +// "u16le" to null, +// "u24be" to null, +// "u24le" to null, +// "u32be" to null, +// "u32le" to null, +// "uncodedframecrc" to null, +// "v210" to null, +// "v210x" to null, +// "vag" to null, +// "vbn_pipe" to null, +// "vc1" to null, +// "vc1test" to null, +// "vcd" to null, +// "vfwcap" to null, +// "vidc" to null, +// "vividas" to null, +// "vivo" to null, +// "vmd" to null, +// "vob" to null, +// "voc" to null, +// "vpk" to null, +// "vplayer" to null, +// "vqf" to null, +// "w64" to null, +// "wc3movie" to null, +// "webm_chunk" to null, +// "webm_dash_manifest" to null, +// "webp" to null, +// "webp_pipe" to null, +// "webvtt" to null, +// "wsaud" to null, +// "wsd" to null, +// "wsvqa" to null, +// "wtv" to null, +// "wv" to null, +// "wve" to null, +// "xa" to null, +// "xbin" to null, +// "xbm_pipe" to null, +// "xmv" to null, +// "xpm_pipe" to null, +// "xvag" to null, +// "xwd_pipe" to null, +// "xwma" to null, +// "yop" to null, +// "yuv4mpegpipe" to null, +) diff --git a/playback/exoplayer/src/main/kotlin/support/AdaptiveSupport.kt b/playback/exoplayer/src/main/kotlin/support/AdaptiveSupport.kt new file mode 100644 index 0000000000..e1a55bd353 --- /dev/null +++ b/playback/exoplayer/src/main/kotlin/support/AdaptiveSupport.kt @@ -0,0 +1,18 @@ +package org.jellyfin.playback.exoplayer.support + +import com.google.android.exoplayer2.RendererCapabilities + +enum class AdaptiveSupport { + SEAMLESS, + NOT_SEAMLESS, + NOT_SUPPORTED; + + companion object { + fun fromFlags(flags: Int) = when (RendererCapabilities.getAdaptiveSupport(flags)) { + RendererCapabilities.ADAPTIVE_SEAMLESS -> SEAMLESS + RendererCapabilities.ADAPTIVE_NOT_SEAMLESS -> NOT_SEAMLESS + RendererCapabilities.ADAPTIVE_NOT_SUPPORTED -> NOT_SUPPORTED + else -> null + } + } +} diff --git a/playback/exoplayer/src/main/kotlin/support/DecoderSupport.kt b/playback/exoplayer/src/main/kotlin/support/DecoderSupport.kt new file mode 100644 index 0000000000..06051e08bc --- /dev/null +++ b/playback/exoplayer/src/main/kotlin/support/DecoderSupport.kt @@ -0,0 +1,18 @@ +package org.jellyfin.playback.exoplayer.support + +import com.google.android.exoplayer2.RendererCapabilities + +enum class DecoderSupport { + PRIMARY, + FALLBACK_MIMETYPE, + FALLBACK; + + companion object { + fun fromFlags(flags: Int) = when (RendererCapabilities.getDecoderSupport(flags)) { + RendererCapabilities.DECODER_SUPPORT_PRIMARY -> PRIMARY + RendererCapabilities.DECODER_SUPPORT_FALLBACK_MIMETYPE -> FALLBACK_MIMETYPE + RendererCapabilities.DECODER_SUPPORT_FALLBACK -> FALLBACK + else -> null + } + } +} diff --git a/playback/exoplayer/src/main/kotlin/support/ExoPlayerPlaySupportReport.kt b/playback/exoplayer/src/main/kotlin/support/ExoPlayerPlaySupportReport.kt new file mode 100644 index 0000000000..ffbad3c549 --- /dev/null +++ b/playback/exoplayer/src/main/kotlin/support/ExoPlayerPlaySupportReport.kt @@ -0,0 +1,59 @@ +package org.jellyfin.playback.exoplayer.support + +import com.google.android.exoplayer2.BaseRenderer +import com.google.android.exoplayer2.ExoPlayer +import com.google.android.exoplayer2.Format +import com.google.android.exoplayer2.RendererCapabilities +import org.jellyfin.playback.core.support.PlaySupportReport + +data class ExoPlayerPlaySupportReport( + val format: FormatSupport?, + val adaptive: AdaptiveSupport?, + val tunneling: Boolean?, + val hardwareAcceleration: Boolean?, + val decoder: DecoderSupport?, +) : PlaySupportReport { + override val canPlay = format == FormatSupport.HANDLED || tunneling == true || hardwareAcceleration == true + + companion object { + fun fromFlags(flags: Int): ExoPlayerPlaySupportReport = ExoPlayerPlaySupportReport( + format = FormatSupport.fromFlags(RendererCapabilities.getFormatSupport(flags)), + adaptive = AdaptiveSupport.fromFlags(RendererCapabilities.getAdaptiveSupport(flags)), + tunneling = tunnelingFromFlags(RendererCapabilities.getTunnelingSupport(flags)), + hardwareAcceleration = hardwareAccelerationFromFlags(RendererCapabilities.getHardwareAccelerationSupport(flags)), + decoder = DecoderSupport.fromFlags(RendererCapabilities.getDecoderSupport(flags)), + ) + + private fun tunnelingFromFlags(flags: Int) = when (RendererCapabilities.getTunnelingSupport(flags)) { + RendererCapabilities.TUNNELING_SUPPORTED -> true + RendererCapabilities.TUNNELING_NOT_SUPPORTED -> false + else -> null + } + + private fun hardwareAccelerationFromFlags(flags: Int) = when (RendererCapabilities.getHardwareAccelerationSupport(flags)) { + RendererCapabilities.HARDWARE_ACCELERATION_SUPPORTED -> true + RendererCapabilities.HARDWARE_ACCELERATION_NOT_SUPPORTED -> false + else -> null + } + } +} + +fun ExoPlayer.getPlaySupportReport(format: Format): ExoPlayerPlaySupportReport = + ExoPlayerPlaySupportReport.fromFlags(supportsFormat(format)) + +fun ExoPlayer.supportsFormat(format: Format): Int { + var capabilities = 0 + + repeat(rendererCount) { rendererIndex -> + val renderer = getRenderer(rendererIndex) + + val rendererCapabilities = when (renderer) { + is BaseRenderer -> renderer.supportsFormat(format) + else -> renderer.capabilities.supportsFormat(format) + } + + capabilities = capabilities or rendererCapabilities + } + + return capabilities +} diff --git a/playback/exoplayer/src/main/kotlin/support/FormatSupport.kt b/playback/exoplayer/src/main/kotlin/support/FormatSupport.kt new file mode 100644 index 0000000000..a3798f9e13 --- /dev/null +++ b/playback/exoplayer/src/main/kotlin/support/FormatSupport.kt @@ -0,0 +1,22 @@ +package org.jellyfin.playback.exoplayer.support + +import com.google.android.exoplayer2.RendererCapabilities + +enum class FormatSupport { + HANDLED, + EXCEEDS_CAPABILITIES, + UNSUPPORTED_DRM, + UNSUPPORTED_SUBTYPE, + UNSUPPORTED_TYPE; + + companion object { + fun fromFlags(flags: Int) = when (RendererCapabilities.getFormatSupport(flags)) { + RendererCapabilities.FORMAT_HANDLED -> HANDLED + RendererCapabilities.FORMAT_EXCEEDS_CAPABILITIES -> EXCEEDS_CAPABILITIES + RendererCapabilities.FORMAT_UNSUPPORTED_DRM -> UNSUPPORTED_DRM + RendererCapabilities.FORMAT_UNSUPPORTED_SUBTYPE -> UNSUPPORTED_SUBTYPE + RendererCapabilities.FORMAT_UNSUPPORTED_TYPE -> UNSUPPORTED_TYPE + else -> null + } + } +} diff --git a/playback/exoplayer/src/main/kotlin/support/mediaStreamToFormat.kt b/playback/exoplayer/src/main/kotlin/support/mediaStreamToFormat.kt new file mode 100644 index 0000000000..704cabf076 --- /dev/null +++ b/playback/exoplayer/src/main/kotlin/support/mediaStreamToFormat.kt @@ -0,0 +1,20 @@ +package org.jellyfin.playback.exoplayer.support + +import com.google.android.exoplayer2.Format +import org.jellyfin.playback.core.mediastream.MediaStream +import org.jellyfin.playback.core.mediastream.MediaStreamAudioTrack +import org.jellyfin.playback.exoplayer.mapping.getFfmpegAudioMimeType +import org.jellyfin.playback.exoplayer.mapping.getFfmpegContainerMimeType + +fun MediaStream.toFormat() = Format.Builder().also { f -> + f.setId(identifier) + f.setContainerMimeType(getFfmpegContainerMimeType(container.format)) + + val audioTrack = tracks.filterIsInstance().firstOrNull() + if (audioTrack != null) { + f.setSampleMimeType(getFfmpegAudioMimeType(audioTrack.codec)) + f.setChannelCount(audioTrack.channels) + f.setAverageBitrate(audioTrack.bitrate) + f.setSampleRate(audioTrack.sampleRate) + } +}.build() diff --git a/playback/jellyfin/src/main/kotlin/mediastream/AudioMediaStreamResolver.kt b/playback/jellyfin/src/main/kotlin/mediastream/AudioMediaStreamResolver.kt index 44c5734055..f1f982e452 100644 --- a/playback/jellyfin/src/main/kotlin/mediastream/AudioMediaStreamResolver.kt +++ b/playback/jellyfin/src/main/kotlin/mediastream/AudioMediaStreamResolver.kt @@ -26,15 +26,15 @@ class AudioMediaStreamResolver( val conversionMethod = when { // Direct play - mediaInfo.mediaSource.supportsDirectPlay || forceDirectPlay -> MediaConversionMethod.None + mediaInfo.mediaSource.supportsDirectPlay || forceDirectPlay -> MediaConversionMethod.None // Remux (Direct stream) - mediaInfo.mediaSource.supportsDirectStream -> MediaConversionMethod.Remux + mediaInfo.mediaSource.supportsDirectStream -> MediaConversionMethod.Remux // Transcode mediaInfo.mediaSource.supportsTranscoding -> MediaConversionMethod.Transcode else -> error("Unable to find a suitable playback method for media") } - val url = when(conversionMethod) { + val url = when (conversionMethod) { // Direct play is MediaConversionMethod.None -> { api.audioApi.getAudioStreamUrl( @@ -73,6 +73,8 @@ class AudioMediaStreamResolver( queueEntry = queueEntry, conversionMethod = conversionMethod, url = url, + container = mediaInfo.getMediaStreamContainer(), + tracks = mediaInfo.getTracks() ) } } diff --git a/playback/jellyfin/src/main/kotlin/mediastream/UniversalAudioMediaStreamResolver.kt b/playback/jellyfin/src/main/kotlin/mediastream/UniversalAudioMediaStreamResolver.kt index bb47ae2e38..c53e08f669 100644 --- a/playback/jellyfin/src/main/kotlin/mediastream/UniversalAudioMediaStreamResolver.kt +++ b/playback/jellyfin/src/main/kotlin/mediastream/UniversalAudioMediaStreamResolver.kt @@ -40,6 +40,8 @@ class UniversalAudioMediaStreamResolver( queueEntry = queueEntry, conversionMethod = MediaConversionMethod.None, url = url, + container = mediaInfo.getMediaStreamContainer(), + tracks = mediaInfo.getTracks() ) } } diff --git a/playback/jellyfin/src/main/kotlin/mediastream/tracks.kt b/playback/jellyfin/src/main/kotlin/mediastream/tracks.kt new file mode 100644 index 0000000000..9200eb1faf --- /dev/null +++ b/playback/jellyfin/src/main/kotlin/mediastream/tracks.kt @@ -0,0 +1,38 @@ +package org.jellyfin.playback.jellyfin.mediastream + +import org.jellyfin.playback.core.mediastream.MediaStreamAudioTrack +import org.jellyfin.playback.core.mediastream.MediaStreamContainer +import org.jellyfin.sdk.model.api.MediaStream +import org.jellyfin.sdk.model.api.MediaStreamType + +fun JellyfinStreamResolver.MediaInfo.getMediaStreamContainer() = MediaStreamContainer( + format = requireNotNull(mediaSource.container) +) + +fun JellyfinStreamResolver.MediaInfo.getTracks() = + mediaSource.mediaStreams + .orEmpty() + .mapNotNull(MediaStream::getMediaStreamTrack) + +fun MediaStream.getMediaStreamTrack() = when (type) { + MediaStreamType.AUDIO -> getAudioTrack(this) + MediaStreamType.VIDEO -> getVideooTrack(this) + MediaStreamType.SUBTITLE -> getSubtitleTrack(this) + + // Ignore other track types + MediaStreamType.EMBEDDED_IMAGE, + MediaStreamType.DATA -> null +} + +private fun getAudioTrack(stream: MediaStream) = MediaStreamAudioTrack( + codec = requireNotNull(stream.codec), + bitrate = stream.bitRate ?: 0, + channels = stream.channels ?: 1, + sampleRate = stream.sampleRate ?: 0, +) + +// TODO Implement Video track type +private fun getVideooTrack(stream: MediaStream) = null + +// TODO Implement Subtitle track type +private fun getSubtitleTrack(stream: MediaStream) = null