Add support for playout-delay RTP extension (#2895)

conf/janus.plugin.streaming.jcfg.sample.in

@@ -74,6 +74,12 @@
 # DO NOT SET THIS PROPERTY IF YOU DON'T KNOW WHAT YOU'RE DOING!
 # e2ee = true
 #
+# To allow mountpoints to negotiate the playout-delay RTP extension,
+# you can set the 'playoutdelay_ext' property to true: this way, any
+# subscriber can customize the playout delay of incoming video streams,
+# assuming the browser supports the RTP extension in the first place.
+# playoutdelay_ext = true
+#
 # The following options are only valid for the 'rtsp' type:
 # url = RTSP stream URL (only for restreaming RTSP)
 # rtsp_user = RTSP authorization username (only if type=rtsp)

src/ice.c

@@ -2715,7 +2715,7 @@ static void janus_ice_cb_nice_recv(NiceAgent *agent, guint stream_id, guint comp
 				janus_plugin_rtp rtp = { .mindex = medium->mindex, .video = video, .buffer = buf, .length = buflen };
 				janus_plugin_rtp_extensions_reset(&rtp.extensions);
 				/* Parse RTP extensions before involving the plugin */
-				if(pc->audiolevel_ext_id != -1) {
+				if(!video && pc->audiolevel_ext_id != -1) {
 					gboolean vad = FALSE;
 					int level = -1;
 					if(janus_rtp_header_extension_parse_audio_level(buf, buflen,
@@ -2724,7 +2724,7 @@ static void janus_ice_cb_nice_recv(NiceAgent *agent, guint stream_id, guint comp
 						rtp.extensions.audio_level_vad = vad;
 					}
 				}
-				if(pc->videoorientation_ext_id != -1) {
+				if(video && pc->videoorientation_ext_id != -1) {
 					gboolean c = FALSE, f = FALSE, r1 = FALSE, r0 = FALSE;
 					if(janus_rtp_header_extension_parse_video_orientation(buf, buflen,
 							pc->videoorientation_ext_id, &c, &f, &r1, &r0) == 0) {
@@ -2739,7 +2739,15 @@ static void janus_ice_cb_nice_recv(NiceAgent *agent, guint stream_id, guint comp
 						rtp.extensions.video_flipped = f;
 					}
 				}
-				if(pc->dependencydesc_ext_id != -1) {
+				if(video && pc->playoutdelay_ext_id != -1) {
+					uint16_t min = 0, max = 0;
+					if(janus_rtp_header_extension_parse_playout_delay(buf, buflen,
+							pc->playoutdelay_ext_id, &min, &max) == 0) {
+						rtp.extensions.min_delay = min;
+						rtp.extensions.max_delay = max;
+					}
+				}
+				if(video && pc->dependencydesc_ext_id != -1) {
 					uint8_t dd[256];
 					int len = sizeof(dd);
 					if(janus_rtp_header_extension_parse_dependency_desc(buf, buflen,
@@ -3756,8 +3764,9 @@ static void janus_ice_rtp_extension_update(janus_ice_handle *handle, janus_ice_p
 	gboolean video = (packet->type == JANUS_ICE_PACKET_VIDEO);
 	if(handle->pc->mid_ext_id > 0 || (video && handle->pc->abs_send_time_ext_id > 0) ||
 			(video && handle->pc->transport_wide_cc_ext_id > 0) ||
-			(!video && packet->extensions.audio_level != -1 && handle->pc->audiolevel_ext_id > 0) ||
-			(video && packet->extensions.video_rotation != -1 && handle->pc->videoorientation_ext_id > 0) ||
+			(!video && packet->extensions.audio_level > -1 && handle->pc->audiolevel_ext_id > 0) ||
+			(video && packet->extensions.video_rotation > -1 && handle->pc->videoorientation_ext_id > 0) ||
+			(video && (packet->extensions.min_delay > -1 || packet->extensions.max_delay > -1) && handle->pc->playoutdelay_ext_id > 0) ||
 			(video && packet->extensions.dd_len > 0 && handle->pc->dependencydesc_ext_id > 0)) {
 		/* Do we need 2-byte extemsions, or are 1-byte extensions fine? */
 		gboolean use_2byte = (video && packet->extensions.dd_len > 16 && handle->pc->dependencydesc_ext_id > 0);
@@ -3810,7 +3819,7 @@ static void janus_ice_rtp_extension_update(janus_ice_handle *handle, janus_ice_p
 			}
 		}
 		/* Check if the plugin (or source) included other extensions */
-		if(!video && packet->extensions.audio_level != -1 && handle->pc->audiolevel_ext_id > 0) {
+		if(!video && packet->extensions.audio_level > -1 && handle->pc->audiolevel_ext_id > 0) {
 			/* Add audio-level extension */
 			if(!use_2byte) {
 				*index = (handle->pc->audiolevel_ext_id << 4);
@@ -3827,7 +3836,7 @@ static void janus_ice_rtp_extension_update(janus_ice_handle *handle, janus_ice_p
 				extbufsize -= 3;
 			}
 		}
-		if(video && packet->extensions.video_rotation != -1 && handle->pc->videoorientation_ext_id > 0) {
+		if(video && packet->extensions.video_rotation > -1 && handle->pc->videoorientation_ext_id > 0) {
 			/* Add video-orientation extension */
 			gboolean c = (packet->extensions.video_back_camera == TRUE),
 				f = (packet->extensions.video_flipped == TRUE), r1 = FALSE, r0 = FALSE;
@@ -3865,6 +3874,27 @@ static void janus_ice_rtp_extension_update(janus_ice_handle *handle, janus_ice_p
 				extbufsize -= 3;
 			}
 		}
+		if(video && (packet->extensions.min_delay > -1 || packet->extensions.max_delay > -1) && handle->pc->playoutdelay_ext_id > 0) {
+			/* Add playout-delay extension */
+			uint32_t min_delay = (uint32_t)packet->extensions.min_delay;
+			uint32_t max_delay = (uint32_t)packet->extensions.max_delay;
+			uint32_t pd = ((min_delay << 12) & 0x00FFF000) + (max_delay & 0x00000FFF);
+			uint32_t pd24 = htonl(pd) >> 8;
+			if(!use_2byte) {
+				*index = (handle->pc->playoutdelay_ext_id << 4) + 2;
+				memcpy(index+1, &pd24, 3);
+				index += 4;
+				extlen += 4;
+				extbufsize -= 4;
+			} else {
+				*index = handle->pc->playoutdelay_ext_id;
+				*(index+1) = 3;
+				memcpy(index+2, &pd24, 3);
+				index += 5;
+				extlen += 5;
+				extbufsize -= 5;
+			}
+		}
 		/* Check if we need to add the mid extension */
 		if(handle->pc->mid_ext_id > 0) {
 			char *mid = medium->mid;

src/ice.h

@@ -444,6 +444,8 @@ struct janus_ice_peerconnection {
 	gint audiolevel_ext_id;
 	/*! \brief Video orientation extension ID */
 	gint videoorientation_ext_id;
+	/*! \brief Playout delay extension ID */
+	gint playoutdelay_ext_id;
 	/*! \brief Dependency descriptor extension ID */
 	gint dependencydesc_ext_id;
 	/*! \brief Absolute Send Time ext ID */

src/janus.c

@@ -1552,6 +1552,8 @@ int janus_process_incoming_request(janus_request *request) {
 					handle->pc->audiolevel_ext_id = janus_rtp_header_extension_get_id(jsep_sdp, JANUS_RTP_EXTMAP_AUDIO_LEVEL);
 					/* Check if the video orientation ID extension is being negotiated */
 					handle->pc->videoorientation_ext_id = janus_rtp_header_extension_get_id(jsep_sdp, JANUS_RTP_EXTMAP_VIDEO_ORIENTATION);
+					/* Check if the playout delay ID extension is being negotiated */
+					handle->pc->playoutdelay_ext_id = janus_rtp_header_extension_get_id(jsep_sdp, JANUS_RTP_EXTMAP_PLAYOUT_DELAY);
 					/* Check if the abs-send-time ID extension is being negotiated */
 					handle->pc->abs_send_time_ext_id = janus_rtp_header_extension_get_id(jsep_sdp, JANUS_RTP_EXTMAP_ABS_SEND_TIME);
 					/* Check if transport wide CC is supported */
@@ -1616,6 +1618,8 @@ int janus_process_incoming_request(janus_request *request) {
 					handle->pc->audiolevel_ext_id = janus_rtp_header_extension_get_id(jsep_sdp, JANUS_RTP_EXTMAP_AUDIO_LEVEL);
 					/* Check if the video orientation ID extension is being negotiated */
 					handle->pc->videoorientation_ext_id = janus_rtp_header_extension_get_id(jsep_sdp, JANUS_RTP_EXTMAP_VIDEO_ORIENTATION);
+					/* Check if the playout delay ID extension is being negotiated */
+					handle->pc->playoutdelay_ext_id = janus_rtp_header_extension_get_id(jsep_sdp, JANUS_RTP_EXTMAP_PLAYOUT_DELAY);
 					/* Check if the abs-send-time ID extension is being negotiated */
 					handle->pc->abs_send_time_ext_id = janus_rtp_header_extension_get_id(jsep_sdp, JANUS_RTP_EXTMAP_ABS_SEND_TIME);
 					/* Check if transport wide CC is supported */
@@ -3162,6 +3166,8 @@ json_t *janus_admin_peerconnection_summary(janus_ice_peerconnection *pc) {
 		json_object_set_new(se, JANUS_RTP_EXTMAP_AUDIO_LEVEL, json_integer(pc->audiolevel_ext_id));
 	if(pc->videoorientation_ext_id > 0)
 		json_object_set_new(se, JANUS_RTP_EXTMAP_VIDEO_ORIENTATION, json_integer(pc->videoorientation_ext_id));
+	if(pc->playoutdelay_ext_id > 0)
+		json_object_set_new(se, JANUS_RTP_EXTMAP_PLAYOUT_DELAY, json_integer(pc->playoutdelay_ext_id));
 	if(pc->dependencydesc_ext_id > 0)
 		json_object_set_new(se, JANUS_RTP_EXTMAP_DEPENDENCY_DESC, json_integer(pc->dependencydesc_ext_id));
 	json_object_set_new(w, "extensions", se);
@@ -3709,7 +3715,7 @@ json_t *janus_plugin_handle_sdp(janus_plugin_session *plugin_session, janus_plug
 		/* Make sure we don't send the rid/repaired-rid attributes when offering ourselves */
 		int mindex = 0;
 		int mid_ext_id = 0, transport_wide_cc_ext_id = 0, abs_send_time_ext_id = 0,
-			audiolevel_ext_id = 0, videoorientation_ext_id = 0, dependencydesc_ext_id = 0;
+			audiolevel_ext_id = 0, videoorientation_ext_id = 0, playoutdelay_ext_id = 0, dependencydesc_ext_id = 0;
 		GList *temp = parsed_sdp->m_lines;
 		while(temp) {
 			janus_sdp_mline *m = (janus_sdp_mline *)temp->data;
@@ -3730,6 +3736,8 @@ json_t *janus_plugin_handle_sdp(janus_plugin_session *plugin_session, janus_plug
 							audiolevel_ext_id = atoi(a->value);
 						else if(strstr(a->value, JANUS_RTP_EXTMAP_VIDEO_ORIENTATION))
 							videoorientation_ext_id = atoi(a->value);
+						else if(strstr(a->value, JANUS_RTP_EXTMAP_PLAYOUT_DELAY))
+							playoutdelay_ext_id = atoi(a->value);
 						else if(strstr(a->value, JANUS_RTP_EXTMAP_DEPENDENCY_DESC))
 							dependencydesc_ext_id = atoi(a->value);
 						else if(strstr(a->value, JANUS_RTP_EXTMAP_RID) ||
@@ -3764,6 +3772,8 @@ json_t *janus_plugin_handle_sdp(janus_plugin_session *plugin_session, janus_plug
 			ice_handle->pc->audiolevel_ext_id = audiolevel_ext_id;
 		if(ice_handle->pc && ice_handle->pc->videoorientation_ext_id != videoorientation_ext_id)
 			ice_handle->pc->videoorientation_ext_id = videoorientation_ext_id;
+		if(ice_handle->pc && ice_handle->pc->playoutdelay_ext_id != playoutdelay_ext_id)
+			ice_handle->pc->playoutdelay_ext_id = playoutdelay_ext_id;
 		if(ice_handle->pc && ice_handle->pc->dependencydesc_ext_id != dependencydesc_ext_id)
 			ice_handle->pc->dependencydesc_ext_id = dependencydesc_ext_id;
 	} else {

src/plugins/janus_echotest.c

@@ -204,6 +204,8 @@ static struct janus_json_parameter request_parameters[] = {
 	{"videocodec", JSON_STRING, 0},
 	{"videoprofile", JSON_STRING, 0},
 	{"opusred", JANUS_JSON_BOOL, 0},
+	{"min_delay", JSON_INTEGER, 0},
+	{"max_delay", JSON_INTEGER, 0},
 };
 
 /* Useful stuff */
@@ -246,6 +248,7 @@ typedef struct janus_echotest_session {
 	gboolean e2ee;			/* Whether media is encrypted, e.g., using Insertable Streams */
 	janus_mutex rec_mutex;	/* Mutex to protect the recorders from race conditions */
 	guint16 slowlink_count;
+	int16_t min_delay, max_delay;
 	volatile gint hangingup;
 	volatile gint destroyed;
 	janus_refcount ref;
@@ -430,6 +433,8 @@ void janus_echotest_create_session(janus_plugin_session *handle, int *error) {
 	janus_rtp_switching_context_reset(&session->context);
 	janus_rtp_simulcasting_context_reset(&session->sim_context);
 	janus_vp8_simulcast_context_reset(&session->vp8_context);
+	session->min_delay = -1;
+	session->max_delay = -1;
 	session->destroyed = 0;
 	g_atomic_int_set(&session->hangingup, 0);
 	g_atomic_int_set(&session->destroyed, 0);
@@ -581,6 +586,10 @@ void janus_echotest_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp
 		gboolean video = packet->video;
 		char *buf = packet->buffer;
 		uint16_t len = packet->length;
+		if(session->min_delay > -1 && session->max_delay > -1) {
+			packet->extensions.min_delay = session->min_delay;
+			packet->extensions.max_delay = session->max_delay;
+		}
 		if(video && session->video_active && (session->ssrc[0] != 0 || session->rid[0] != NULL)) {
 			/* Handle simulcast: backup the header information first */
 			janus_rtp_header *header = (janus_rtp_header *)buf;
@@ -849,6 +858,8 @@ static void janus_echotest_hangup_media_internal(janus_plugin_session *handle) {
 	janus_rtp_switching_context_reset(&session->context);
 	janus_rtp_simulcasting_context_reset(&session->sim_context);
 	janus_vp8_simulcast_context_reset(&session->vp8_context);
+	session->min_delay = -1;
+	session->max_delay = -1;
 	g_atomic_int_set(&session->hangingup, 0);
 }
 
@@ -947,6 +958,8 @@ static void *janus_echotest_handler(void *data) {
 		json_t *videocodec = json_object_get(root, "videocodec");
 		json_t *videoprofile = json_object_get(root, "videoprofile");
 		json_t *opusred = json_object_get(root, "opusred");
+		json_t *min_delay = json_object_get(root, "min_delay");
+		json_t *max_delay = json_object_get(root, "max_delay");
 		/* Enforce request */
 		if(audio) {
 			session->audio_active = json_is_true(audio);
@@ -1010,6 +1023,28 @@ static void *janus_echotest_handler(void *data) {
 				gateway->send_pli(session->handle);
 			}
 		}
+		if(min_delay) {
+			int16_t md = json_integer_value(min_delay);
+			if(md < 0) {
+				session->min_delay = -1;
+				session->max_delay = -1;
+			} else {
+				session->min_delay = md;
+				if(session->min_delay > session->max_delay)
+					session->max_delay = session->min_delay;
+			}
+		}
+		if(max_delay) {
+			int16_t md = json_integer_value(max_delay);
+			if(md < 0) {
+				session->min_delay = -1;
+				session->max_delay = -1;
+			} else {
+				session->max_delay = md;
+				if(session->max_delay < session->min_delay)
+					session->min_delay = session->max_delay;
+			}
+		}
 
 		/* Any SDP to handle? */
 		if(msg_sdp) {
@@ -1019,7 +1054,8 @@ static void *janus_echotest_handler(void *data) {
 			session->has_data = (strstr(msg_sdp, "DTLS/SCTP") != NULL);
 		}
 
-		if(!audio && !video && !videocodec && !videoprofile && !opusred && !bitrate && !substream && !temporal && !fallback && !record && !msg_sdp) {
+		if(!audio && !video && !videocodec && !videoprofile && !opusred && !bitrate &&
+				!substream && !temporal && !fallback && !record && !min_delay && !max_delay && !msg_sdp) {
 			JANUS_LOG(LOG_ERR, "No supported attributes found\n");
 			error_code = JANUS_ECHOTEST_ERROR_INVALID_ELEMENT;
 			g_snprintf(error_cause, 512, "Message error: no supported attributes found");
@@ -1109,6 +1145,7 @@ static void *janus_echotest_handler(void *data) {
 					JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_REPAIRED_RID,
 					JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_AUDIO_LEVEL,
 					JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_VIDEO_ORIENTATION,
+					JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_PLAYOUT_DELAY,
 					JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_TRANSPORT_WIDE_CC,
 					JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_DEPENDENCY_DESC,
 					JANUS_SDP_OA_DONE);