From a5eee9f4c9e3bf453be0acce1365839521e7512a Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Tue, 22 Feb 2022 12:18:02 +0100 Subject: [PATCH] Add support for playout-delay RTP extension (see #2895) --- conf/janus.plugin.streaming.jcfg.sample.in | 6 + ice.c | 44 +++++-- ice.h | 2 + janus.c | 12 +- plugins/janus_echotest.c | 39 +++++- plugins/janus_streaming.c | 96 ++++++++++++++- plugins/janus_videoroom.c | 137 ++++++++++++++++++++- plugins/plugin.c | 2 + plugins/plugin.h | 4 +- 9 files changed, 325 insertions(+), 17 deletions(-) diff --git a/conf/janus.plugin.streaming.jcfg.sample.in b/conf/janus.plugin.streaming.jcfg.sample.in index 1a3fff8bb1..5659143c2a 100644 --- a/conf/janus.plugin.streaming.jcfg.sample.in +++ b/conf/janus.plugin.streaming.jcfg.sample.in @@ -74,6 +74,12 @@ # DO NOT SET THIS PROPERTY IF YOU DON'T KNOW WHAT YOU'RE DOING! # e2ee = true # +# To allow mountpoints to negotiate the playout-delay RTP extension, +# you can set the 'playoutdelay_ext' property to true: this way, any +# subscriber can customize the playout delay of incoming video streams, +# assuming the browser supports the RTP extension in the first place. +# playoutdelay_ext = true +# # The following options are only valid for the 'rtsp' type: # url = RTSP stream URL (only for restreaming RTSP) # rtsp_user = RTSP authorization username (only if type=rtsp) diff --git a/ice.c b/ice.c index ffd3faba91..a6ea3d5dc3 100644 --- a/ice.c +++ b/ice.c @@ -2762,7 +2762,7 @@ static void janus_ice_cb_nice_recv(NiceAgent *agent, guint stream_id, guint comp janus_plugin_rtp rtp = { .video = video, .buffer = buf, .length = buflen }; janus_plugin_rtp_extensions_reset(&rtp.extensions); /* Parse RTP extensions before involving the plugin */ - if(stream->audiolevel_ext_id != -1) { + if(!video && stream->audiolevel_ext_id != -1) { gboolean vad = FALSE; int level = -1; if(janus_rtp_header_extension_parse_audio_level(buf, buflen, @@ -2771,7 +2771,7 @@ static void janus_ice_cb_nice_recv(NiceAgent *agent, guint stream_id, guint comp rtp.extensions.audio_level_vad = vad; } } - if(stream->videoorientation_ext_id != -1) { + if(video && stream->videoorientation_ext_id != -1) { gboolean c = FALSE, f = FALSE, r1 = FALSE, r0 = FALSE; if(janus_rtp_header_extension_parse_video_orientation(buf, buflen, stream->videoorientation_ext_id, &c, &f, &r1, &r0) == 0) { @@ -2786,7 +2786,15 @@ static void janus_ice_cb_nice_recv(NiceAgent *agent, guint stream_id, guint comp rtp.extensions.video_flipped = f; } } - if(stream->dependencydesc_ext_id != -1) { + if(video && stream->playoutdelay_ext_id != -1) { + uint16_t min = 0, max = 0; + if(janus_rtp_header_extension_parse_playout_delay(buf, buflen, + stream->playoutdelay_ext_id, &min, &max) == 0) { + rtp.extensions.min_delay = min; + rtp.extensions.max_delay = max; + } + } + if(video && stream->dependencydesc_ext_id != -1) { uint8_t dd[256]; int len = sizeof(dd); if(janus_rtp_header_extension_parse_dependency_desc(buf, buflen, @@ -3988,8 +3996,9 @@ static void janus_ice_rtp_extension_update(janus_ice_handle *handle, janus_ice_q gboolean video = (packet->type == JANUS_ICE_PACKET_VIDEO); if(handle->stream->mid_ext_id > 0 || (video && handle->stream->abs_send_time_ext_id > 0) || (video && handle->stream->transport_wide_cc_ext_id > 0) || - (!video && packet->extensions.audio_level != -1 && handle->stream->audiolevel_ext_id > 0) || - (video && packet->extensions.video_rotation != -1 && handle->stream->videoorientation_ext_id > 0) || + (!video && packet->extensions.audio_level > -1 && handle->stream->audiolevel_ext_id > 0) || + (video && packet->extensions.video_rotation > -1 && handle->stream->videoorientation_ext_id > 0) || + (video && (packet->extensions.min_delay > -1 || packet->extensions.max_delay > -1) && handle->stream->playoutdelay_ext_id > 0) || (video && packet->extensions.dd_len > 0 && handle->stream->dependencydesc_ext_id > 0)) { /* Do we need 2-byte extemsions, or are 1-byte extensions fine? */ gboolean use_2byte = (video && packet->extensions.dd_len > 16 && handle->stream->dependencydesc_ext_id > 0); @@ -4042,7 +4051,7 @@ static void janus_ice_rtp_extension_update(janus_ice_handle *handle, janus_ice_q } } /* Check if the plugin (or source) included other extensions */ - if(!video && packet->extensions.audio_level != -1 && handle->stream->audiolevel_ext_id > 0) { + if(!video && packet->extensions.audio_level > -1 && handle->stream->audiolevel_ext_id > 0) { /* Add audio-level extension */ if(!use_2byte) { *index = (handle->stream->audiolevel_ext_id << 4); @@ -4059,7 +4068,7 @@ static void janus_ice_rtp_extension_update(janus_ice_handle *handle, janus_ice_q extbufsize -= 3; } } - if(video && packet->extensions.video_rotation != -1 && handle->stream->videoorientation_ext_id > 0) { + if(video && packet->extensions.video_rotation > -1 && handle->stream->videoorientation_ext_id > 0) { /* Add video-orientation extension */ gboolean c = (packet->extensions.video_back_camera == TRUE), f = (packet->extensions.video_flipped == TRUE), r1 = FALSE, r0 = FALSE; @@ -4097,6 +4106,27 @@ static void janus_ice_rtp_extension_update(janus_ice_handle *handle, janus_ice_q extbufsize -= 3; } } + if(video && (packet->extensions.min_delay > -1 || packet->extensions.max_delay > -1) && handle->stream->playoutdelay_ext_id > 0) { + /* Add playout-delay extension */ + uint32_t min_delay = (uint32_t)packet->extensions.min_delay; + uint32_t max_delay = (uint32_t)packet->extensions.max_delay; + uint32_t pd = ((min_delay << 12) & 0x00FFF000) + (max_delay & 0x00000FFF); + uint32_t pd24 = htonl(pd) >> 8; + if(!use_2byte) { + *index = (handle->stream->playoutdelay_ext_id << 4) + 2; + memcpy(index+1, &pd24, 3); + index += 4; + extlen += 4; + extbufsize -= 4; + } else { + *index = handle->stream->playoutdelay_ext_id; + *(index+1) = 3; + memcpy(index+2, &pd24, 3); + index += 5; + extlen += 5; + extbufsize -= 5; + } + } /* Check if we need to add the mid extension */ if(handle->stream->mid_ext_id > 0) { char *mid = video ? handle->video_mid : handle->audio_mid; diff --git a/ice.h b/ice.h index 13ae6e130e..3a6605823d 100644 --- a/ice.h +++ b/ice.h @@ -488,6 +488,8 @@ struct janus_ice_stream { gint audiolevel_ext_id; /*! \brief Video orientation extension ID */ gint videoorientation_ext_id; + /*! \brief Playout delay extension ID */ + gint playoutdelay_ext_id; /*! \brief Dependency descriptor extension ID */ gint dependencydesc_ext_id; /*! \brief Absolute Send Time ext ID */ diff --git a/janus.c b/janus.c index a5c14f4c13..604ae8ad4e 100644 --- a/janus.c +++ b/janus.c @@ -1561,6 +1561,8 @@ int janus_process_incoming_request(janus_request *request) { handle->stream->audiolevel_ext_id = janus_rtp_header_extension_get_id(jsep_sdp, JANUS_RTP_EXTMAP_AUDIO_LEVEL); /* Check if the video orientation ID extension is being negotiated */ handle->stream->videoorientation_ext_id = janus_rtp_header_extension_get_id(jsep_sdp, JANUS_RTP_EXTMAP_VIDEO_ORIENTATION); + /* Check if the playout delay ID extension is being negotiated */ + handle->stream->playoutdelay_ext_id = janus_rtp_header_extension_get_id(jsep_sdp, JANUS_RTP_EXTMAP_PLAYOUT_DELAY); /* Check if the abs-send-time ID extension is being negotiated */ handle->stream->abs_send_time_ext_id = janus_rtp_header_extension_get_id(jsep_sdp, JANUS_RTP_EXTMAP_ABS_SEND_TIME); /* Check if transport wide CC is supported */ @@ -1626,6 +1628,8 @@ int janus_process_incoming_request(janus_request *request) { handle->stream->audiolevel_ext_id = janus_rtp_header_extension_get_id(jsep_sdp, JANUS_RTP_EXTMAP_AUDIO_LEVEL); /* Check if the video orientation ID extension is being negotiated */ handle->stream->videoorientation_ext_id = janus_rtp_header_extension_get_id(jsep_sdp, JANUS_RTP_EXTMAP_VIDEO_ORIENTATION); + /* Check if the playout delay ID extension is being negotiated */ + handle->stream->playoutdelay_ext_id = janus_rtp_header_extension_get_id(jsep_sdp, JANUS_RTP_EXTMAP_PLAYOUT_DELAY); /* Check if the abs-send-time ID extension is being negotiated */ handle->stream->abs_send_time_ext_id = janus_rtp_header_extension_get_id(jsep_sdp, JANUS_RTP_EXTMAP_ABS_SEND_TIME); /* Check if transport wide CC is supported */ @@ -3138,6 +3142,8 @@ json_t *janus_admin_stream_summary(janus_ice_stream *stream) { json_object_set_new(se, JANUS_RTP_EXTMAP_AUDIO_LEVEL, json_integer(stream->audiolevel_ext_id)); if(stream->videoorientation_ext_id > 0) json_object_set_new(se, JANUS_RTP_EXTMAP_VIDEO_ORIENTATION, json_integer(stream->videoorientation_ext_id)); + if(stream->playoutdelay_ext_id > 0) + json_object_set_new(se, JANUS_RTP_EXTMAP_PLAYOUT_DELAY, json_integer(stream->playoutdelay_ext_id)); if(stream->dependencydesc_ext_id > 0) json_object_set_new(se, JANUS_RTP_EXTMAP_DEPENDENCY_DESC, json_integer(stream->dependencydesc_ext_id)); json_object_set_new(s, "extensions", se); @@ -3738,7 +3744,7 @@ json_t *janus_plugin_handle_sdp(janus_plugin_session *plugin_session, janus_plug } /* Make sure we don't send the rid/repaired-rid attributes when offering ourselves */ int mid_ext_id = 0, transport_wide_cc_ext_id = 0, abs_send_time_ext_id = 0, - audiolevel_ext_id = 0, videoorientation_ext_id = 0, dependencydesc_ext_id = 0; + audiolevel_ext_id = 0, videoorientation_ext_id = 0, playoutdelay_ext_id = 0, dependencydesc_ext_id = 0; int opusred_pt = 0; GList *temp = parsed_sdp->m_lines; while(temp) { @@ -3758,6 +3764,8 @@ json_t *janus_plugin_handle_sdp(janus_plugin_session *plugin_session, janus_plug audiolevel_ext_id = atoi(a->value); else if(strstr(a->value, JANUS_RTP_EXTMAP_VIDEO_ORIENTATION)) videoorientation_ext_id = atoi(a->value); + else if(strstr(a->value, JANUS_RTP_EXTMAP_PLAYOUT_DELAY)) + playoutdelay_ext_id = atoi(a->value); else if(strstr(a->value, JANUS_RTP_EXTMAP_DEPENDENCY_DESC)) dependencydesc_ext_id = atoi(a->value); else if(strstr(a->value, JANUS_RTP_EXTMAP_RID) || @@ -3791,6 +3799,8 @@ json_t *janus_plugin_handle_sdp(janus_plugin_session *plugin_session, janus_plug ice_handle->stream->audiolevel_ext_id = audiolevel_ext_id; if(ice_handle->stream && ice_handle->stream->videoorientation_ext_id != videoorientation_ext_id) ice_handle->stream->videoorientation_ext_id = videoorientation_ext_id; + if(ice_handle->stream && ice_handle->stream->playoutdelay_ext_id != playoutdelay_ext_id) + ice_handle->stream->playoutdelay_ext_id = playoutdelay_ext_id; if(ice_handle->stream && ice_handle->stream->dependencydesc_ext_id != dependencydesc_ext_id) ice_handle->stream->dependencydesc_ext_id = dependencydesc_ext_id; } else { diff --git a/plugins/janus_echotest.c b/plugins/janus_echotest.c index 46d73618e2..c2175dc23e 100644 --- a/plugins/janus_echotest.c +++ b/plugins/janus_echotest.c @@ -204,6 +204,8 @@ static struct janus_json_parameter request_parameters[] = { {"videocodec", JSON_STRING, 0}, {"videoprofile", JSON_STRING, 0}, {"opusred", JANUS_JSON_BOOL, 0}, + {"min_delay", JSON_INTEGER, 0}, + {"max_delay", JSON_INTEGER, 0}, }; /* Useful stuff */ @@ -246,6 +248,7 @@ typedef struct janus_echotest_session { gboolean e2ee; /* Whether media is encrypted, e.g., using Insertable Streams */ janus_mutex rec_mutex; /* Mutex to protect the recorders from race conditions */ guint16 slowlink_count; + int16_t min_delay, max_delay; volatile gint hangingup; volatile gint destroyed; janus_refcount ref; @@ -430,6 +433,8 @@ void janus_echotest_create_session(janus_plugin_session *handle, int *error) { janus_rtp_switching_context_reset(&session->context); janus_rtp_simulcasting_context_reset(&session->sim_context); janus_vp8_simulcast_context_reset(&session->vp8_context); + session->min_delay = -1; + session->max_delay = -1; session->destroyed = 0; g_atomic_int_set(&session->hangingup, 0); g_atomic_int_set(&session->destroyed, 0); @@ -581,6 +586,10 @@ void janus_echotest_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp gboolean video = packet->video; char *buf = packet->buffer; uint16_t len = packet->length; + if(session->min_delay > -1 && session->max_delay > -1) { + packet->extensions.min_delay = session->min_delay; + packet->extensions.max_delay = session->max_delay; + } if(video && session->video_active && (session->ssrc[0] != 0 || session->rid[0] != NULL)) { /* Handle simulcast: backup the header information first */ janus_rtp_header *header = (janus_rtp_header *)buf; @@ -849,6 +858,8 @@ static void janus_echotest_hangup_media_internal(janus_plugin_session *handle) { janus_rtp_switching_context_reset(&session->context); janus_rtp_simulcasting_context_reset(&session->sim_context); janus_vp8_simulcast_context_reset(&session->vp8_context); + session->min_delay = -1; + session->max_delay = -1; g_atomic_int_set(&session->hangingup, 0); } @@ -940,6 +951,8 @@ static void *janus_echotest_handler(void *data) { json_t *videocodec = json_object_get(root, "videocodec"); json_t *videoprofile = json_object_get(root, "videoprofile"); json_t *opusred = json_object_get(root, "opusred"); + json_t *min_delay = json_object_get(root, "min_delay"); + json_t *max_delay = json_object_get(root, "max_delay"); /* Enforce request */ if(audio) { session->audio_active = json_is_true(audio); @@ -1003,6 +1016,28 @@ static void *janus_echotest_handler(void *data) { gateway->send_pli(session->handle); } } + if(min_delay) { + int16_t md = json_integer_value(min_delay); + if(md < 0) { + session->min_delay = -1; + session->max_delay = -1; + } else { + session->min_delay = md; + if(session->min_delay > session->max_delay) + session->max_delay = session->min_delay; + } + } + if(max_delay) { + int16_t md = json_integer_value(max_delay); + if(md < 0) { + session->min_delay = -1; + session->max_delay = -1; + } else { + session->max_delay = md; + if(session->max_delay < session->min_delay) + session->min_delay = session->max_delay; + } + } /* Any SDP to handle? */ if(msg_sdp) { @@ -1012,7 +1047,8 @@ static void *janus_echotest_handler(void *data) { session->has_data = (strstr(msg_sdp, "DTLS/SCTP") != NULL); } - if(!audio && !video && !videocodec && !videoprofile && !opusred && !bitrate && !substream && !temporal && !fallback && !record && !msg_sdp) { + if(!audio && !video && !videocodec && !videoprofile && !opusred && !bitrate && + !substream && !temporal && !fallback && !record && !min_delay && !max_delay && !msg_sdp) { JANUS_LOG(LOG_ERR, "No supported attributes found\n"); error_code = JANUS_ECHOTEST_ERROR_INVALID_ELEMENT; g_snprintf(error_cause, 512, "Message error: no supported attributes found"); @@ -1097,6 +1133,7 @@ static void *janus_echotest_handler(void *data) { JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_REPAIRED_RID, JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_AUDIO_LEVEL, JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_VIDEO_ORIENTATION, + JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_PLAYOUT_DELAY, JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_TRANSPORT_WIDE_CC, JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_DEPENDENCY_DESC, JANUS_SDP_OA_DONE); diff --git a/plugins/janus_streaming.c b/plugins/janus_streaming.c index 7cf2817b57..da092b380e 100644 --- a/plugins/janus_streaming.c +++ b/plugins/janus_streaming.c @@ -134,6 +134,12 @@ so neither Janus nor the Streaming plugin have access to anything. DO NOT SET THIS PROPERTY IF YOU DON'T KNOW WHAT YOU'RE DOING! e2ee = true +To allow mountpoints to negotiate the playout-delay RTP extension, +you can set the 'playoutdelay_ext' property to true: this way, any +subscriber can customize the playout delay of incoming video streams, +assuming the browser supports the RTP extension in the first place. +playoutdelay_ext = true + The following options are only valid for the 'rtsp' type: url = RTSP stream URL rtsp_user = RTSP authorization username, if needed @@ -619,7 +625,9 @@ rtsp_conn_timeout = connection timeout for cURL (CURLOPT_CONNECTTIMEOUT) call ga "temporal" : , "fallback" : , "spatial_layer" : , - "temporal_layer" : + "temporal_layer" : , + "min_delay" : , + "max_delay" : } \endverbatim * @@ -849,7 +857,8 @@ static struct janus_json_parameter rtp_parameters[] = { {"threads", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"srtpsuite", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"srtpcrypto", JSON_STRING, 0}, - {"e2ee", JANUS_JSON_BOOL, 0} + {"e2ee", JANUS_JSON_BOOL, 0}, + {"playoutdelay_ext", JANUS_JSON_BOOL, 0} }; static struct janus_json_parameter live_parameters[] = { {"filename", JSON_STRING, JANUS_JSON_PARAM_REQUIRED}, @@ -950,7 +959,10 @@ static struct janus_json_parameter configure_parameters[] = { {"fallback", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, /* For VP9 SVC */ {"spatial_layer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, - {"temporal_layer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE} + {"temporal_layer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, + /* For the playout-delay RTP extension, if negotiated */ + {"min_delay", JSON_INTEGER, 0}, + {"max_delay", JSON_INTEGER, 0}, }; static struct janus_json_parameter disable_parameters[] = { {"stop_recording", JANUS_JSON_BOOL, 0} @@ -1111,6 +1123,8 @@ typedef struct janus_streaming_rtp_source { srtp_policy_t srtp_policy; /* If the media is end-to-end encrypted, we may need to know */ gboolean e2ee; + /* Whether the playout-delay extension should be negotiated or not for new subscribers */ + gboolean playoutdelay_ext; } janus_streaming_rtp_source; typedef struct janus_streaming_file_source { @@ -1192,7 +1206,7 @@ static void janus_streaming_helper_rtprtcp_packet(gpointer data, gpointer user_d /* Helper to create an RTP live source (e.g., from gstreamer/ffmpeg/vlc/etc.) */ janus_streaming_mountpoint *janus_streaming_create_rtp_source( uint64_t id, char *id_str, char *name, char *desc, char *metadata, - int srtpsuite, char *srtpcrypto, int threads, gboolean e2ee, + int srtpsuite, char *srtpcrypto, int threads, gboolean e2ee, gboolean playoutdelay_ext, gboolean doaudio, gboolean doaudiortcp, char *amcast, const janus_network_address *aiface, uint16_t aport, uint16_t artcpport, uint8_t acodec, char *artpmap, char *afmtp, gboolean doaskew, gboolean dovideo, gboolean dovideortcp, char *vmcast, const janus_network_address *viface, @@ -1239,6 +1253,10 @@ typedef struct janus_streaming_session { int spatial_layer, target_spatial_layer; gint64 last_spatial_layer[3]; int temporal_layer, target_temporal_layer; + /* Whether the playout-delay extension should be negotiated */ + gboolean playoutdelay_ext; + /* Playout delays to enforce when relaying this stream, if the extension has been negotiated */ + int16_t min_delay, max_delay; /* If the media is end-to-end encrypted, we may need to know */ gboolean e2ee; janus_mutex mutex; @@ -1770,6 +1788,7 @@ int janus_streaming_init(janus_callbacks *callback, const char *config_path) { janus_config_item *ssuite = janus_config_get(config, cat, janus_config_type_item, "srtpsuite"); janus_config_item *scrypto = janus_config_get(config, cat, janus_config_type_item, "srtpcrypto"); janus_config_item *e2ee = janus_config_get(config, cat, janus_config_type_item, "e2ee"); + janus_config_item *pd = janus_config_get(config, cat, janus_config_type_item, "playoutdelay_ext"); gboolean is_private = priv && priv->value && janus_is_true(priv->value); gboolean doaudio = audio && audio->value && janus_is_true(audio->value); gboolean doaskew = audio && askew && askew->value && janus_is_true(askew->value); @@ -1927,6 +1946,7 @@ int janus_streaming_init(janus_callbacks *callback, const char *config_path) { scrypto && scrypto->value ? (char *)scrypto->value : NULL, (threads && threads->value) ? atoi(threads->value) : 0, (e2ee && e2ee->value) ? janus_is_true(e2ee->value) : FALSE, + (pd && pd->value) ? janus_is_true(pd->value) : FALSE, doaudio, doaudiortcp, amcast ? (char *)amcast->value : NULL, doaudio && aiface && aiface->value ? &audio_iface : NULL, @@ -2405,6 +2425,12 @@ json_t *janus_streaming_query_session(janus_plugin_session *handle) { json_object_set_new(svc, "target-temporal-layer", json_integer(session->target_temporal_layer)); json_object_set_new(info, "svc", svc); } + if(session->playoutdelay_ext) { + json_t *pd = json_object(); + json_object_set_new(pd, "min-delay", json_integer(session->min_delay)); + json_object_set_new(pd, "max-delay", json_integer(session->max_delay)); + json_object_set_new(info, "playout-delay", pd); + } } janus_refcount_decrease(&mp->ref); } @@ -2777,7 +2803,8 @@ static json_t *janus_streaming_process_synchronous_request(janus_streaming_sessi json_t *ssuite = json_object_get(root, "srtpsuite"); json_t *scrypto = json_object_get(root, "srtpcrypto"); json_t *e2ee = json_object_get(root, "e2ee"); - gboolean doaudio = audio ? json_is_true(audio) : FALSE, doaudiortcp = FALSE; + json_t *pd = json_object_get(root, "playoutdelay_ext"); + gboolean doaudio = audio ? json_is_true(audio) : FALSE, doaudiortcp = FALSE; gboolean dovideo = video ? json_is_true(video) : FALSE, dovideortcp = FALSE; gboolean dodata = data ? json_is_true(data) : FALSE; gboolean doaskew = FALSE, dovskew = FALSE, dosvc = FALSE; @@ -2974,6 +3001,7 @@ static json_t *janus_streaming_process_synchronous_request(janus_streaming_sessi scrypto ? (char *)json_string_value(scrypto) : NULL, threads ? json_integer_value(threads) : 0, e2ee ? json_is_true(e2ee) : FALSE, + pd ? json_is_true(pd) : FALSE, doaudio, doaudiortcp, amcast, &audio_iface, aport, artcpport, acodec, artpmap, afmtp, doaskew, dovideo, dovideortcp, vmcast, &video_iface, vport, vrtcpport, vcodec, vrtpmap, vfmtp, bufferkf, simulcast, vport2, vport3, dosvc, dovskew, @@ -3371,6 +3399,10 @@ static json_t *janus_streaming_process_synchronous_request(janus_streaming_sessi g_snprintf(value, BUFSIZ, "%d", mp->helper_threads); janus_config_add(config, c, janus_config_item_create("threads", value)); } + if(source->e2ee) + janus_config_add(config, c, janus_config_item_create("e2ee", "yes")); + if(source->playoutdelay_ext) + janus_config_add(config, c, janus_config_item_create("playoutdelay_ext", "yes")); } else if(!strcasecmp(type_text, "live") || !strcasecmp(type_text, "ondemand")) { janus_streaming_file_source *source = mp->source; janus_config_add(config, c, janus_config_item_create("filename", source->filename)); @@ -3706,6 +3738,10 @@ static json_t *janus_streaming_process_synchronous_request(janus_streaming_sessi g_snprintf(value, BUFSIZ, "%d", mp->helper_threads); janus_config_add(config, c, janus_config_item_create("threads", value)); } + if(source->e2ee) + janus_config_add(config, c, janus_config_item_create("e2ee", "yes")); + if(source->playoutdelay_ext) + janus_config_add(config, c, janus_config_item_create("playoutdelay_ext", "yes")); } } else { janus_config_add(config, c, janus_config_item_create("type", (mp->streaming_type == janus_streaming_type_live) ? "live" : "ondemand")); @@ -4886,6 +4922,10 @@ static void *janus_streaming_handler(void *data) { /* If this mountpoint is broadcasting end-to-end encrypted media, * add the info to the JSEP offer we'll be sending them */ session->e2ee = source->e2ee; + /* Also check if we have to offer the playout-delay extension */ + session->playoutdelay_ext = source->playoutdelay_ext; + session->min_delay = -1; + session->max_delay = -1; } janus_refcount_increase(&session->ref); done: @@ -4959,6 +4999,10 @@ static void *janus_streaming_handler(void *data) { janus_strlcat(sdptemp, buffer, 2048); g_snprintf(buffer, 512, "a=extmap:%d %s\r\n", 2, JANUS_RTP_EXTMAP_ABS_SEND_TIME); janus_strlcat(sdptemp, buffer, 2048); + if(session->playoutdelay_ext) { + g_snprintf(buffer, 512, "a=extmap:%d %s\r\n", 3, JANUS_RTP_EXTMAP_PLAYOUT_DELAY); + janus_strlcat(sdptemp, buffer, 2048); + } } #ifdef HAVE_SCTP if(mp->data && session->data) { @@ -5159,6 +5203,33 @@ static void *janus_streaming_handler(void *data) { session->target_temporal_layer = temporal_layer; } } + if(session->playoutdelay_ext) { + /* Check if we need to specify a custom playout delay for this stream */ + json_t *min_delay = json_object_get(root, "min_delay"); + if(min_delay) { + int16_t md = json_integer_value(min_delay); + if(md < 0) { + session->min_delay = -1; + session->max_delay = -1; + } else { + session->min_delay = md; + if(session->min_delay > session->max_delay) + session->max_delay = session->min_delay; + } + } + json_t *max_delay = json_object_get(root, "max_delay"); + if(max_delay) { + int16_t md = json_integer_value(max_delay); + if(md < 0) { + session->min_delay = -1; + session->max_delay = -1; + } else { + session->max_delay = md; + if(session->max_delay < session->min_delay) + session->min_delay = session->max_delay; + } + } + } } /* Done */ result = json_object(); @@ -5739,7 +5810,7 @@ static void janus_streaming_file_source_free(janus_streaming_file_source *source /* Helper to create an RTP live source (e.g., from gstreamer/ffmpeg/vlc/etc.) */ janus_streaming_mountpoint *janus_streaming_create_rtp_source( uint64_t id, char *id_str, char *name, char *desc, char *metadata, - int srtpsuite, char *srtpcrypto, int threads, gboolean e2ee, + int srtpsuite, char *srtpcrypto, int threads, gboolean e2ee, gboolean playoutdelay_ext, gboolean doaudio, gboolean doaudiortcp, char *amcast, const janus_network_address *aiface, uint16_t aport, uint16_t artcpport, uint8_t acodec, char *artpmap, char *afmtp, gboolean doaskew, gboolean dovideo, gboolean dovideortcp, char *vmcast, const janus_network_address *viface, uint16_t vport, uint16_t vrtcpport, uint8_t vcodec, char *vrtpmap, char *vfmtp, gboolean bufferkf, gboolean simulcast, uint16_t vport2, uint16_t vport3, gboolean svc, gboolean dovskew, int rtp_collision, @@ -6027,6 +6098,7 @@ janus_streaming_mountpoint *janus_streaming_create_rtp_source( live_rtp_source->srtpcrypto = g_strdup(srtpcrypto); } live_rtp_source->e2ee = e2ee; + live_rtp_source->playoutdelay_ext = playoutdelay_ext; live_rtp_source->audio_mcast = doaudio ? (amcast ? inet_addr(amcast) : INADDR_ANY) : INADDR_ANY; live_rtp_source->audio_iface = doaudio && !janus_network_address_is_null(aiface) ? *aiface : nil; live_rtp_source->audio_port = doaudio ? aport : -1; @@ -8468,6 +8540,10 @@ static void janus_streaming_relay_rtp_packet(gpointer data, gpointer user_data) packet->data->type = session->video_pt; janus_plugin_rtp rtp = { .video = packet->is_video, .buffer = (char *)packet->data, .length = packet->length }; janus_plugin_rtp_extensions_reset(&rtp.extensions); + if(session->min_delay > -1 && session->max_delay > -1) { + rtp.extensions.min_delay = session->min_delay; + rtp.extensions.max_delay = session->max_delay; + } if(gateway != NULL) gateway->relay_rtp(session->handle, &rtp); if(override_mark_bit && !has_marker_bit) { @@ -8540,6 +8616,10 @@ static void janus_streaming_relay_rtp_packet(gpointer data, gpointer user_data) /* Send the packet */ janus_plugin_rtp rtp = { .video = packet->is_video, .buffer = (char *)packet->data, .length = packet->length }; janus_plugin_rtp_extensions_reset(&rtp.extensions); + if(session->min_delay > -1 && session->max_delay > -1) { + rtp.extensions.min_delay = session->min_delay; + rtp.extensions.max_delay = session->max_delay; + } if(gateway != NULL) gateway->relay_rtp(session->handle, &rtp); /* Restore the timestamp and sequence number to what the publisher set them to */ @@ -8557,6 +8637,10 @@ static void janus_streaming_relay_rtp_packet(gpointer data, gpointer user_data) packet->data->type = session->video_pt; janus_plugin_rtp rtp = { .video = packet->is_video, .buffer = (char *)packet->data, .length = packet->length }; janus_plugin_rtp_extensions_reset(&rtp.extensions); + if(session->min_delay > -1 && session->max_delay > -1) { + rtp.extensions.min_delay = session->min_delay; + rtp.extensions.max_delay = session->max_delay; + } if(gateway != NULL) gateway->relay_rtp(session->handle, &rtp); /* Restore the timestamp and sequence number to what the video source set them to */ diff --git a/plugins/janus_videoroom.c b/plugins/janus_videoroom.c index 1cfa4d59bf..0f5f153544 100644 --- a/plugins/janus_videoroom.c +++ b/plugins/janus_videoroom.c @@ -685,6 +685,8 @@ room-: { "display" : "", "audio_active_packets" : "", "audio_level_average" : "", + "min_delay" : , + "max_delay" : , } \endverbatim * @@ -1088,7 +1090,9 @@ room-: { "spatial_layer" : , "temporal_layer" : , "audio_level_average" : "", - "audio_active_packets" : "" + "audio_active_packets" : "", + "min_delay" : , + "max_delay" : } \endverbatim * @@ -1381,6 +1385,9 @@ static struct janus_json_parameter publish_parameters[] = { {"secret", JSON_STRING, 0}, {"audio_level_averge", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"audio_active_packets", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, + /* For the playout-delay RTP extension, if negotiated */ + {"min_delay", JSON_INTEGER, 0}, + {"max_delay", JSON_INTEGER, 0}, /* The following are just to force a renegotiation and/or an ICE restart */ {"update", JANUS_JSON_BOOL, 0}, {"restart", JANUS_JSON_BOOL, 0} @@ -1430,6 +1437,9 @@ static struct janus_json_parameter configure_parameters[] = { /* For VP9 SVC */ {"spatial_layer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"temporal_layer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, + /* For the playout-delay RTP extension, if negotiated */ + {"min_delay", JSON_INTEGER, 0}, + {"max_delay", JSON_INTEGER, 0}, /* The following is to handle a renegotiation */ {"update", JANUS_JSON_BOOL, 0} }; @@ -1449,6 +1459,9 @@ static struct janus_json_parameter subscriber_parameters[] = { /* For VP9 SVC */ {"spatial_layer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"temporal_layer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, + /* For the playout-delay RTP extension, if negotiated */ + {"min_delay", JSON_INTEGER, 0}, + {"max_delay", JSON_INTEGER, 0}, }; /* Static configuration instance */ @@ -1660,6 +1673,8 @@ typedef struct janus_videoroom_publisher { int user_audio_active_packets; /* Participant's audio_active_packets overwriting global room setting */ int user_audio_level_average; /* Participant's audio_level_average overwriting global room setting */ gboolean talking; /* Whether this participant is currently talking (uses audio levels extension) */ + /* Playout delays to enforce when relaying this stream, if the extension has been negotiated */ + int16_t min_delay, max_delay; gboolean data_active, data_muted; gboolean firefox; /* We send Firefox users a different kind of FIR */ uint32_t bitrate; @@ -1715,6 +1730,8 @@ typedef struct janus_videoroom_subscriber { int spatial_layer, target_spatial_layer; gint64 last_spatial_layer[3]; int temporal_layer, target_temporal_layer; + /* Playout delays to enforce when relaying this stream, if the extension has been negotiated */ + int16_t min_delay, max_delay; gboolean e2ee; /* If media for this subscriber is end-to-end encrypted */ volatile gint destroyed; janus_refcount ref; @@ -5526,6 +5543,10 @@ void janus_videoroom_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp /* Backup the actual timestamp and sequence number set by the publisher, in case switching is involved */ packet.timestamp = ntohl(packet.data->timestamp); packet.seq_number = ntohs(packet.data->seq_number); + if(participant->min_delay > -1 && participant->max_delay > -1) { + packet.extensions.min_delay = participant->min_delay; + packet.extensions.max_delay = participant->max_delay; + } /* Go: some viewers may decide to drop the packet, but that's up to them */ janus_mutex_lock_nodebug(&participant->subscribers_mutex); g_slist_foreach(participant->subscribers, janus_videoroom_relay_rtp_packet, &packet); @@ -6578,6 +6599,8 @@ static void *janus_videoroom_handler(void *data) { goto error; } json_t *sc_fallback = json_object_get(root, "fallback"); + json_t *min_delay = json_object_get(root, "min_delay"); + json_t *max_delay = json_object_get(root, "max_delay"); janus_videoroom_publisher *owner = NULL; janus_videoroom_publisher *publisher = g_hash_table_lookup(videoroom->participants, string_ids ? (gpointer)feed_id_str : (gpointer)&feed_id); @@ -6672,6 +6695,31 @@ static void *janus_videoroom_handler(void *data) { subscriber->temporal_layer = -1; subscriber->target_temporal_layer = temporal ? json_integer_value(temporal) : 2; } + /* Override the playout-delay properties */ + subscriber->min_delay = -1; + subscriber->max_delay = -1; + if(min_delay) { + int16_t md = json_integer_value(min_delay); + if(md < 0) { + subscriber->min_delay = -1; + subscriber->max_delay = -1; + } else { + subscriber->min_delay = md; + if(subscriber->min_delay > subscriber->max_delay) + subscriber->max_delay = subscriber->min_delay; + } + } + if(max_delay) { + int16_t md = json_integer_value(max_delay); + if(md < 0) { + subscriber->min_delay = -1; + subscriber->max_delay = -1; + } else { + subscriber->max_delay = md; + if(subscriber->max_delay < subscriber->min_delay) + subscriber->min_delay = subscriber->max_delay; + } + } session->participant = subscriber; janus_mutex_lock(&publisher->subscribers_mutex); publisher->subscribers = g_slist_append(publisher->subscribers, subscriber); @@ -6810,6 +6858,8 @@ static void *janus_videoroom_handler(void *data) { json_t *update = json_object_get(root, "update"); json_t *user_audio_active_packets = json_object_get(root, "audio_active_packets"); json_t *user_audio_level_average = json_object_get(root, "audio_level_average"); + json_t *min_delay = json_object_get(root, "min_delay"); + json_t *max_delay = json_object_get(root, "max_delay"); if(audio) { gboolean audio_active = json_is_true(audio); if(g_atomic_int_get(&session->started) && audio_active && !participant->audio_active && !participant->audio_muted) { @@ -6924,6 +6974,29 @@ static void *janus_videoroom_handler(void *data) { record_locked = TRUE; } } + /* Override the playout-delay properties */ + if(min_delay) { + int16_t md = json_integer_value(min_delay); + if(md < 0) { + participant->min_delay = -1; + participant->max_delay = -1; + } else { + participant->min_delay = md; + if(participant->min_delay > participant->max_delay) + participant->max_delay = participant->min_delay; + } + } + if(max_delay) { + int16_t md = json_integer_value(max_delay); + if(md < 0) { + participant->min_delay = -1; + participant->max_delay = -1; + } else { + participant->max_delay = md; + if(participant->max_delay < participant->min_delay) + participant->min_delay = participant->max_delay; + } + } janus_mutex_lock(&participant->rec_mutex); gboolean prev_recording_active = participant->recording_active; if(record && !record_locked) { @@ -7110,6 +7183,8 @@ static void *janus_videoroom_handler(void *data) { goto error; } json_t *sc_fallback = json_object_get(root, "fallback"); + json_t *min_delay = json_object_get(root, "min_delay"); + json_t *max_delay = json_object_get(root, "max_delay"); /* Update the audio/video/data flags, if set */ janus_videoroom_publisher *publisher = subscriber->feed; if(publisher) { @@ -7219,6 +7294,29 @@ static void *janus_videoroom_handler(void *data) { subscriber->target_temporal_layer = temporal_layer; } } + /* Override the playout-delay properties */ + if(min_delay) { + int16_t md = json_integer_value(min_delay); + if(md < 0) { + subscriber->min_delay = -1; + subscriber->max_delay = -1; + } else { + subscriber->min_delay = md; + if(subscriber->min_delay > subscriber->max_delay) + subscriber->max_delay = subscriber->min_delay; + } + } + if(max_delay) { + int16_t md = json_integer_value(max_delay); + if(md < 0) { + subscriber->min_delay = -1; + subscriber->max_delay = -1; + } else { + subscriber->max_delay = md; + if(subscriber->max_delay < subscriber->min_delay) + subscriber->min_delay = subscriber->max_delay; + } + } event = json_object(); json_object_set_new(event, "videoroom", json_string("event")); json_object_set_new(event, "room", string_ids ? json_string(subscriber->room_id_str) : json_integer(subscriber->room_id)); @@ -7385,6 +7483,8 @@ static void *janus_videoroom_handler(void *data) { goto error; } json_t *sc_fallback = json_object_get(root, "fallback"); + json_t *min_delay = json_object_get(root, "min_delay"); + json_t *max_delay = json_object_get(root, "max_delay"); if(!subscriber->room) { JANUS_LOG(LOG_ERR, "Room Destroyed\n"); error_code = JANUS_VIDEOROOM_ERROR_NO_SUCH_ROOM; @@ -7467,6 +7567,29 @@ static void *janus_videoroom_handler(void *data) { subscriber->temporal_layer = -1; subscriber->target_temporal_layer = temporal ? json_integer_value(temporal) : 2; } + /* Override the playout-delay properties */ + if(min_delay) { + int16_t md = json_integer_value(min_delay); + if(md < 0) { + subscriber->min_delay = -1; + subscriber->max_delay = -1; + } else { + subscriber->min_delay = md; + if(subscriber->min_delay > subscriber->max_delay) + subscriber->max_delay = subscriber->min_delay; + } + } + if(max_delay) { + int16_t md = json_integer_value(max_delay); + if(md < 0) { + subscriber->min_delay = -1; + subscriber->max_delay = -1; + } else { + subscriber->max_delay = md; + if(subscriber->max_delay < subscriber->min_delay) + subscriber->min_delay = subscriber->max_delay; + } + } janus_mutex_lock(&publisher->subscribers_mutex); publisher->subscribers = g_slist_append(publisher->subscribers, subscriber); janus_mutex_unlock(&publisher->subscribers_mutex); @@ -8145,6 +8268,10 @@ static void janus_videoroom_relay_rtp_packet(gpointer data, gpointer user_data) if(gateway != NULL) { janus_plugin_rtp rtp = { .video = packet->is_video, .buffer = (char *)packet->data, .length = packet->length, .extensions = packet->extensions }; + if(subscriber->min_delay > -1 && subscriber->max_delay > -1) { + rtp.extensions.min_delay = subscriber->min_delay; + rtp.extensions.max_delay = subscriber->max_delay; + } gateway->relay_rtp(session->handle, &rtp); } if(override_mark_bit && !has_marker_bit) { @@ -8210,6 +8337,10 @@ static void janus_videoroom_relay_rtp_packet(gpointer data, gpointer user_data) if(gateway != NULL) { janus_plugin_rtp rtp = { .video = packet->is_video, .buffer = (char *)packet->data, .length = packet->length, .extensions = packet->extensions }; + if(subscriber->min_delay > -1 && subscriber->max_delay > -1) { + rtp.extensions.min_delay = subscriber->min_delay; + rtp.extensions.max_delay = subscriber->max_delay; + } gateway->relay_rtp(session->handle, &rtp); } /* Restore the timestamp and sequence number to what the publisher set them to */ @@ -8226,6 +8357,10 @@ static void janus_videoroom_relay_rtp_packet(gpointer data, gpointer user_data) if(gateway != NULL) { janus_plugin_rtp rtp = { .video = packet->is_video, .buffer = (char *)packet->data, .length = packet->length, .extensions = packet->extensions }; + if(subscriber->min_delay > -1 && subscriber->max_delay > -1) { + rtp.extensions.min_delay = subscriber->min_delay; + rtp.extensions.max_delay = subscriber->max_delay; + } gateway->relay_rtp(session->handle, &rtp); } /* Restore the timestamp and sequence number to what the publisher set them to */ diff --git a/plugins/plugin.c b/plugins/plugin.c index 1f68508010..550d6d0d96 100644 --- a/plugins/plugin.c +++ b/plugins/plugin.c @@ -44,6 +44,8 @@ void janus_plugin_rtp_extensions_reset(janus_plugin_rtp_extensions *extensions) extensions->video_rotation = -1; extensions->video_back_camera = FALSE; extensions->video_flipped = FALSE; + extensions->min_delay = -1; + extensions->max_delay = -1; extensions->dd_len = 0; memset(extensions->dd_content, 0, sizeof(extensions->dd_content)); } diff --git a/plugins/plugin.h b/plugins/plugin.h index b21d2cca67..cbf9a3aee8 100644 --- a/plugins/plugin.h +++ b/plugins/plugin.h @@ -171,7 +171,7 @@ janus_plugin *create(void) { * Janus instance or it will crash. * */ -#define JANUS_PLUGIN_API_VERSION 16 +#define JANUS_PLUGIN_API_VERSION 17 /*! \brief Initialization of all plugin properties to NULL * @@ -561,6 +561,8 @@ struct janus_plugin_rtp_extensions { /*! \brief Whether the video orientation extension says it's flipped horizontally * @note Will be ignored if no rotation value is set */ gboolean video_flipped; + /*! \brief Min and max playout delay, if available; -1 means no extension */ + int16_t min_delay, max_delay; /*! \brief Length of Dependency Descriptor data, if available */ uint8_t dd_len; /*! \brief Dependency Descriptor content */