Initial support for VP9 and AV1 simulcast (and fix for broken AV1/SVC) (

#3218)

html/echotest.js

@@ -203,7 +203,7 @@ $(document).ready(function() {
 									if((substream !== null && substream !== undefined) || (temporal !== null && temporal !== undefined)) {
 										if(!simulcastStarted) {
 											simulcastStarted = true;
-											addSimulcastSvcButtons(msg["videocodec"] === "vp8");
+											addSimulcastSvcButtons(msg["videocodec"] === "vp8" || msg["videocodec"] === "vp9" || msg["videocodec"] === "av1");
 										}
 										// We just received notice that there's been a switch, update the buttons
 										updateSimulcastSvcButtons(substream, temporal);

html/janus.js

@@ -2497,9 +2497,9 @@ function Janus(gatewayCallbacks) {
 								direction: 'sendrecv',
 								streams: [config.myStream],
 								sendEncodings: track.sendEncodings || [
-									{ rid: 'h', active: true, maxBitrate: maxBitrates.high },
-									{ rid: 'm', active: true, maxBitrate: maxBitrates.medium, scaleResolutionDownBy: 2 },
-									{ rid: 'l', active: true, maxBitrate: maxBitrates.low, scaleResolutionDownBy: 4 }
+									{ rid: 'h', active: true, scalabilityMode: 'L1T2', maxBitrate: maxBitrates.high },
+									{ rid: 'm', active: true, scalabilityMode: 'L1T2', maxBitrate: maxBitrates.medium, scaleResolutionDownBy: 2 },
+									{ rid: 'l', active: true, scalabilityMode: 'L1T2', maxBitrate: maxBitrates.low, scaleResolutionDownBy: 4 }
 								]
 							});
 						} else {

src/plugins/duktape/janus-sdp.js

@@ -399,6 +399,8 @@ JANUSSDP.generateAnswer = function(offer, options) {
 						answer.push({ type: "a", name: a.name, value: value });
 					} else if(options.enableAudioLevel !== false && a.value.indexOf("urn:ietf:params:rtp-hdrext:ssrc-audio-level") !== -1) {
 						answer.push({ type: "a", name: a.name, value: value });
+					} else if(options.enableAudioLevel !== false && a.value.indexOf("dependency-descriptor-rtp-header-extension") !== -1) {
+						answer.push({ type: "a", name: a.name, value: value });
 					}
 				}
 			} else {

src/plugins/janus_duktape.c

@@ -619,10 +619,6 @@ static duk_ret_t janus_duktape_method_pushevent(duk_context *ctx) {
 			janus_sdp_find_first_codec(parsed_sdp, JANUS_SDP_VIDEO, -1, &vcodec);
 			if(vcodec)
 				session->vcodec = janus_videocodec_from_name(vcodec);
-			if(session->vcodec != JANUS_VIDEOCODEC_VP8 && session->vcodec != JANUS_VIDEOCODEC_H264) {
-				/* VP8 r H.264 were not negotiated, if simulcasting was enabled then disable it here */
-				janus_rtp_simulcasting_cleanup(&session->rid_extmap_id, session->ssrc, session->rid, &session->rid_mutex);
-			}
 		}
 		janus_sdp_destroy(parsed_sdp);
 		/* Send asynchronously */
@@ -2250,10 +2246,6 @@ struct janus_plugin_result *janus_duktape_handle_message(janus_plugin_session *h
 			janus_sdp_find_first_codec(parsed_sdp, JANUS_SDP_VIDEO, -1, &vcodec);
 			if(vcodec)
 				session->vcodec = janus_videocodec_from_name(vcodec);
-			if(session->vcodec != JANUS_VIDEOCODEC_VP8 && session->vcodec != JANUS_VIDEOCODEC_H264) {
-				/* VP8 r H.264 were not negotiated, if simulcasting was enabled then disable it here */
-				janus_rtp_simulcasting_cleanup(&session->rid_extmap_id, session->ssrc, session->rid, &session->rid_mutex);
-			}
 			janus_sdp_destroy(parsed_sdp);
 		}
 		if(json_is_true(json_object_get(jsep, "e2ee")))
@@ -2469,7 +2461,7 @@ void janus_duktape_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *
 	} else {
 		/* We're simulcasting, save the best video quality */
 		gboolean save = janus_rtp_simulcasting_context_process_rtp(&session->rec_simctx,
-			buf, len, session->ssrc, session->rid, session->vcodec, &session->rec_ctx, &session->rid_mutex);
+			buf, len, NULL, 0, session->ssrc, session->rid, session->vcodec, &session->rec_ctx, &session->rid_mutex);
 		if(save) {
 			uint32_t seq_number = ntohs(rtp->seq_number);
 			uint32_t timestamp = ntohl(rtp->timestamp);
@@ -2791,7 +2783,8 @@ static void janus_duktape_relay_rtp_packet(gpointer data, gpointer user_data) {
 			return;
 		/* Process this packet: don't relay if it's not the SSRC/layer we wanted to handle */
 		gboolean relay = janus_rtp_simulcasting_context_process_rtp(&session->sim_context,
-			(char *)packet->data, packet->length, packet->ssrc, NULL, sender->vcodec, &session->vrtpctx, NULL);
+			(char *)packet->data, packet->length, packet->extensions.dd_content, packet->extensions.dd_len,
+			packet->ssrc, NULL, sender->vcodec, &session->vrtpctx, NULL);
 		if(session->sim_context.need_pli && sender->handle) {
 			/* Send a PLI */
 			JANUS_LOG(LOG_VERB, "We need a PLI for the simulcast context\n");

src/plugins/janus_echotest.c

@@ -624,11 +624,12 @@ void janus_echotest_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp
 			if(simulcast) {
 				/* Process this simulcast packet: don't relay if it's not the SSRC/layer we wanted to handle */
 				relay = janus_rtp_simulcasting_context_process_rtp(&session->sim_context,
-					buf, len, session->ssrc, session->rid, session->vcodec, &session->context, &session->rid_mutex);
+					buf, len, packet->extensions.dd_content, packet->extensions.dd_len,
+					session->ssrc, session->rid, session->vcodec, &session->context, &session->rid_mutex);
 			} else {
 				/* Process this SVC packet: don't relay if it's not the layer we wanted to handle */
 				relay = janus_rtp_svc_context_process_rtp(&session->svc_context,
-					buf, len, session->vcodec, NULL, &session->context);
+					buf, len, packet->extensions.dd_content, packet->extensions.dd_len, session->vcodec, NULL, &session->context);
 			}
 			if(session->sim_context.need_pli || session->svc_context.need_pli) {
 				/* Send a PLI */
@@ -1088,7 +1089,7 @@ static void *janus_echotest_handler(void *data) {
 			session->sim_context.templayer_target = json_integer_value(temporal);
 			JANUS_LOG(LOG_VERB, "Setting video temporal layer to let through (simulcast): %d (was %d)\n",
 				session->sim_context.templayer_target, session->sim_context.templayer);
-			if(session->vcodec == JANUS_VIDEOCODEC_VP8 && session->sim_context.templayer_target == session->sim_context.templayer) {
+			if(session->sim_context.templayer_target == session->sim_context.templayer) {
 				/* No need to do anything, we're already getting the right temporal, so notify the user */
 				json_t *event = json_object();
 				json_object_set_new(event, "echotest", json_string("event"));
@@ -1280,10 +1281,6 @@ static void *janus_echotest_handler(void *data) {
 				session->vcodec = janus_videocodec_from_name(vcodec);
 			session->has_audio = session->acodec != JANUS_AUDIOCODEC_NONE;
 			session->has_video = session->vcodec != JANUS_VIDEOCODEC_NONE;
-			if(session->vcodec != JANUS_VIDEOCODEC_VP8 && session->vcodec != JANUS_VIDEOCODEC_H264) {
-				/* VP8 r H.264 were not negotiated, if simulcasting was enabled then disable it here */
-				janus_rtp_simulcasting_cleanup(NULL, session->ssrc, session->rid, &session->rid_mutex);
-			}
 			g_free(session->vfmtp);
 			session->vfmtp = NULL;
 			if(session->has_video) {

src/plugins/janus_lua.c

@@ -547,10 +547,6 @@ static int janus_lua_method_pushevent(lua_State *s) {
 			janus_sdp_find_first_codec(parsed_sdp, JANUS_SDP_VIDEO, -1, &vcodec);
 			if(vcodec)
 				session->vcodec = janus_videocodec_from_name(vcodec);
-			if(session->vcodec != JANUS_VIDEOCODEC_VP8 && session->vcodec != JANUS_VIDEOCODEC_H264) {
-				/* VP8 r H.264 were not negotiated, if simulcasting was enabled then disable it here */
-				janus_rtp_simulcasting_cleanup(&session->rid_extmap_id, session->ssrc, session->rid, &session->rid_mutex);
-			}
 		}
 		janus_sdp_destroy(parsed_sdp);
 		/* Send asynchronously */
@@ -1936,10 +1932,6 @@ struct janus_plugin_result *janus_lua_handle_message(janus_plugin_session *handl
 			janus_sdp_find_first_codec(parsed_sdp, JANUS_SDP_VIDEO, -1, &vcodec);
 			if(vcodec)
 				session->vcodec = janus_videocodec_from_name(vcodec);
-			if(session->vcodec != JANUS_VIDEOCODEC_VP8 && session->vcodec != JANUS_VIDEOCODEC_H264) {
-				/* VP8 r H.264 were not negotiated, if simulcasting was enabled then disable it here */
-				janus_rtp_simulcasting_cleanup(&session->rid_extmap_id, session->ssrc, session->rid, &session->rid_mutex);
-			}
 			janus_sdp_destroy(parsed_sdp);
 		}
 		if(json_is_true(json_object_get(jsep, "e2ee")))
@@ -2127,7 +2119,7 @@ void janus_lua_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *rtp_
 	} else {
 		/* We're simulcasting, save the best video quality */
 		gboolean save = janus_rtp_simulcasting_context_process_rtp(&session->rec_simctx,
-			buf, len, session->ssrc, session->rid, session->vcodec, &session->rec_ctx, &session->rid_mutex);
+			buf, len, NULL, 0, session->ssrc, session->rid, session->vcodec, &session->rec_ctx, &session->rid_mutex);
 		if(save) {
 			uint32_t seq_number = ntohs(rtp->seq_number);
 			uint32_t timestamp = ntohl(rtp->timestamp);
@@ -2421,7 +2413,8 @@ static void janus_lua_relay_rtp_packet(gpointer data, gpointer user_data) {
 			return;
 		/* Process this packet: don't relay if it's not the SSRC/layer we wanted to handle */
 		gboolean relay = janus_rtp_simulcasting_context_process_rtp(&session->sim_context,
-			(char *)packet->data, packet->length, packet->ssrc, NULL, sender->vcodec, &session->vrtpctx, NULL);
+			(char *)packet->data, packet->length, packet->extensions.dd_content, packet->extensions.dd_len,
+			packet->ssrc, NULL, sender->vcodec, &session->vrtpctx, NULL);
 		if(session->sim_context.need_pli && sender->handle) {
 			/* Send a PLI */
 			JANUS_LOG(LOG_VERB, "We need a PLI for the simulcast context\n");

src/plugins/janus_recordplay.c

@@ -1304,7 +1304,8 @@ void janus_recordplay_incoming_rtp(janus_plugin_session *handle, janus_plugin_rt
 		uint32_t ssrc = ntohl(header->ssrc);
 		/* Process this packet: don't save if it's not the SSRC/layer we wanted to handle */
 		gboolean save = janus_rtp_simulcasting_context_process_rtp(&session->sim_context,
-			buf, len, session->ssrc, session->rid, session->recording->vcodec, &session->context, &session->rid_mutex);
+			buf, len, NULL, 0, session->ssrc, session->rid, session->recording->vcodec,
+			&session->context, &session->rid_mutex);
 		if(session->sim_context.need_pli) {
 			/* Send a PLI */
 			JANUS_LOG(LOG_VERB, "We need a PLI for the simulcast context\n");
@@ -1879,10 +1880,6 @@ static void *janus_recordplay_handler(void *data) {
 					janus_mutex_unlock(&session->rid_mutex);
 					session->sim_context.substream_target = 2;	/* Let's aim for the highest quality */
 					session->sim_context.templayer_target = 2;	/* Let's aim for all temporal layers */
-					if(rec->vcodec != JANUS_VIDEOCODEC_VP8 && rec->vcodec != JANUS_VIDEOCODEC_H264) {
-						/* VP8 r H.264 were not negotiated, if simulcasting was enabled then disable it here */
-						janus_rtp_simulcasting_cleanup(NULL, session->ssrc, session->rid, &session->rid_mutex);
-					}
 					/* FIXME We're stopping at the first item, there may be more */
 					break;
 				}

src/plugins/janus_streaming.c

@@ -10030,7 +10030,8 @@ static void janus_streaming_relay_rtp_packet(gpointer data, gpointer user_data)
 					return;
 				/* Process this packet: don't relay if it's not the SSRC/layer we wanted to handle */
 				gboolean relay = janus_rtp_simulcasting_context_process_rtp(&s->sim_context,
-					(char *)packet->data, packet->length, packet->ssrc, NULL, packet->codec, &s->context, NULL);
+					(char *)packet->data, packet->length, NULL, 0,
+					packet->ssrc, NULL, packet->codec, &s->context, NULL);
 				if(!relay) {
 					/* Did a lot of time pass before we could relay a packet? */
 					gint64 now = janus_get_monotonic_time();

src/plugins/janus_videocall.c

@@ -778,7 +778,8 @@ void janus_videocall_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp
 			/* Process this packet: don't relay if it's not the SSRC/layer we wanted to handle
 			 * The caveat is that the targets in OUR simulcast context are the PEER's targets */
 			gboolean relay = janus_rtp_simulcasting_context_process_rtp(&peer->sim_context,
-				buf, len, session->ssrc, session->rid, session->vcodec, &peer->context, &session->rid_mutex);
+				buf, len, packet->extensions.dd_content, packet->extensions.dd_len,
+				session->ssrc, session->rid, session->vcodec, &peer->context, &session->rid_mutex);
 			/* Do we need to drop this? */
 			if(!relay)
 				return;
@@ -1374,7 +1375,7 @@ static void *janus_videocall_handler(void *data) {
 			session->has_data = (strstr(msg_sdp, "DTLS/SCTP") != NULL);
 			/* Check if this user will simulcast */
 			json_t *msg_simulcast = json_object_get(msg->jsep, "simulcast");
-			if(msg_simulcast && janus_get_codec_pt(msg_sdp, "vp8") > 0) {
+			if(msg_simulcast) {
 				JANUS_LOG(LOG_VERB, "VideoCall callee (%s) cannot do simulcast.\n", session->username);
 			} else {
 				janus_rtp_simulcasting_cleanup(NULL, session->ssrc, session->rid, &session->rid_mutex);
@@ -1533,7 +1534,7 @@ static void *janus_videocall_handler(void *data) {
 				session->sim_context.templayer_target = json_integer_value(temporal);
 				JANUS_LOG(LOG_VERB, "Setting video temporal layer to let through (simulcast): %d (was %d)\n",
 					session->sim_context.templayer_target, session->sim_context.templayer);
-				if(session->vcodec == JANUS_VIDEOCODEC_VP8 && session->sim_context.templayer_target == session->sim_context.templayer) {
+				if(session->sim_context.templayer_target == session->sim_context.templayer) {
 					/* No need to do anything, we're already getting the right temporal, so notify the user */
 					json_t *event = json_object();
 					json_object_set_new(event, "videocall", json_string("event"));
@@ -1569,7 +1570,7 @@ static void *janus_videocall_handler(void *data) {
 				session->has_data = (strstr(msg_sdp, "DTLS/SCTP") != NULL);
 				/* Check if this user will simulcast */
 				json_t *msg_simulcast = json_object_get(msg->jsep, "simulcast");
-				if(msg_simulcast && janus_get_codec_pt(msg_sdp, "vp8") > 0) {
+				if(msg_simulcast) {
 					JANUS_LOG(LOG_VERB, "VideoCall callee (%s) cannot do simulcast.\n", session->username);
 				} else {
 					janus_rtp_simulcasting_cleanup(NULL, session->ssrc, session->rid, &session->rid_mutex);