From e57e28dbb386f05b7b96e070f3b00ebf709c46ff Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Mon, 22 May 2023 19:14:35 +0900 Subject: [PATCH] Initial support for VP9 and AV1 simulcast (and fix for broken AV1/SVC) (#3218) --- html/echotest.js | 2 +- html/janus.js | 6 +- src/plugins/duktape/janus-sdp.js | 2 + src/plugins/janus_duktape.c | 13 +- src/plugins/janus_echotest.c | 11 +- src/plugins/janus_lua.c | 13 +- src/plugins/janus_recordplay.c | 7 +- src/plugins/janus_streaming.c | 3 +- src/plugins/janus_videocall.c | 9 +- src/plugins/janus_videoroom.c | 67 +++++----- src/plugins/lua/janus-sdp.lua | 2 + src/rtp.c | 205 +++++++++++++++++++++++++++++-- src/rtp.h | 109 ++++++++-------- src/rtpfwd.c | 2 +- 14 files changed, 316 insertions(+), 135 deletions(-) diff --git a/html/echotest.js b/html/echotest.js index bdd880f68c..3954d8f2f5 100644 --- a/html/echotest.js +++ b/html/echotest.js @@ -203,7 +203,7 @@ $(document).ready(function() { if((substream !== null && substream !== undefined) || (temporal !== null && temporal !== undefined)) { if(!simulcastStarted) { simulcastStarted = true; - addSimulcastSvcButtons(msg["videocodec"] === "vp8"); + addSimulcastSvcButtons(msg["videocodec"] === "vp8" || msg["videocodec"] === "vp9" || msg["videocodec"] === "av1"); } // We just received notice that there's been a switch, update the buttons updateSimulcastSvcButtons(substream, temporal); diff --git a/html/janus.js b/html/janus.js index 344ae52686..8dde64eaf5 100644 --- a/html/janus.js +++ b/html/janus.js @@ -2497,9 +2497,9 @@ function Janus(gatewayCallbacks) { direction: 'sendrecv', streams: [config.myStream], sendEncodings: track.sendEncodings || [ - { rid: 'h', active: true, maxBitrate: maxBitrates.high }, - { rid: 'm', active: true, maxBitrate: maxBitrates.medium, scaleResolutionDownBy: 2 }, - { rid: 'l', active: true, maxBitrate: maxBitrates.low, scaleResolutionDownBy: 4 } + { rid: 'h', active: true, scalabilityMode: 'L1T2', maxBitrate: maxBitrates.high }, + { rid: 'm', active: true, scalabilityMode: 'L1T2', maxBitrate: maxBitrates.medium, scaleResolutionDownBy: 2 }, + { rid: 'l', active: true, scalabilityMode: 'L1T2', maxBitrate: maxBitrates.low, scaleResolutionDownBy: 4 } ] }); } else { diff --git a/src/plugins/duktape/janus-sdp.js b/src/plugins/duktape/janus-sdp.js index 9e833ee80f..c6985e414a 100644 --- a/src/plugins/duktape/janus-sdp.js +++ b/src/plugins/duktape/janus-sdp.js @@ -399,6 +399,8 @@ JANUSSDP.generateAnswer = function(offer, options) { answer.push({ type: "a", name: a.name, value: value }); } else if(options.enableAudioLevel !== false && a.value.indexOf("urn:ietf:params:rtp-hdrext:ssrc-audio-level") !== -1) { answer.push({ type: "a", name: a.name, value: value }); + } else if(options.enableAudioLevel !== false && a.value.indexOf("dependency-descriptor-rtp-header-extension") !== -1) { + answer.push({ type: "a", name: a.name, value: value }); } } } else { diff --git a/src/plugins/janus_duktape.c b/src/plugins/janus_duktape.c index 6cb915ede6..d56f1d6ed2 100644 --- a/src/plugins/janus_duktape.c +++ b/src/plugins/janus_duktape.c @@ -619,10 +619,6 @@ static duk_ret_t janus_duktape_method_pushevent(duk_context *ctx) { janus_sdp_find_first_codec(parsed_sdp, JANUS_SDP_VIDEO, -1, &vcodec); if(vcodec) session->vcodec = janus_videocodec_from_name(vcodec); - if(session->vcodec != JANUS_VIDEOCODEC_VP8 && session->vcodec != JANUS_VIDEOCODEC_H264) { - /* VP8 r H.264 were not negotiated, if simulcasting was enabled then disable it here */ - janus_rtp_simulcasting_cleanup(&session->rid_extmap_id, session->ssrc, session->rid, &session->rid_mutex); - } } janus_sdp_destroy(parsed_sdp); /* Send asynchronously */ @@ -2250,10 +2246,6 @@ struct janus_plugin_result *janus_duktape_handle_message(janus_plugin_session *h janus_sdp_find_first_codec(parsed_sdp, JANUS_SDP_VIDEO, -1, &vcodec); if(vcodec) session->vcodec = janus_videocodec_from_name(vcodec); - if(session->vcodec != JANUS_VIDEOCODEC_VP8 && session->vcodec != JANUS_VIDEOCODEC_H264) { - /* VP8 r H.264 were not negotiated, if simulcasting was enabled then disable it here */ - janus_rtp_simulcasting_cleanup(&session->rid_extmap_id, session->ssrc, session->rid, &session->rid_mutex); - } janus_sdp_destroy(parsed_sdp); } if(json_is_true(json_object_get(jsep, "e2ee"))) @@ -2469,7 +2461,7 @@ void janus_duktape_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp * } else { /* We're simulcasting, save the best video quality */ gboolean save = janus_rtp_simulcasting_context_process_rtp(&session->rec_simctx, - buf, len, session->ssrc, session->rid, session->vcodec, &session->rec_ctx, &session->rid_mutex); + buf, len, NULL, 0, session->ssrc, session->rid, session->vcodec, &session->rec_ctx, &session->rid_mutex); if(save) { uint32_t seq_number = ntohs(rtp->seq_number); uint32_t timestamp = ntohl(rtp->timestamp); @@ -2791,7 +2783,8 @@ static void janus_duktape_relay_rtp_packet(gpointer data, gpointer user_data) { return; /* Process this packet: don't relay if it's not the SSRC/layer we wanted to handle */ gboolean relay = janus_rtp_simulcasting_context_process_rtp(&session->sim_context, - (char *)packet->data, packet->length, packet->ssrc, NULL, sender->vcodec, &session->vrtpctx, NULL); + (char *)packet->data, packet->length, packet->extensions.dd_content, packet->extensions.dd_len, + packet->ssrc, NULL, sender->vcodec, &session->vrtpctx, NULL); if(session->sim_context.need_pli && sender->handle) { /* Send a PLI */ JANUS_LOG(LOG_VERB, "We need a PLI for the simulcast context\n"); diff --git a/src/plugins/janus_echotest.c b/src/plugins/janus_echotest.c index 605fc672ee..4ef6907014 100644 --- a/src/plugins/janus_echotest.c +++ b/src/plugins/janus_echotest.c @@ -624,11 +624,12 @@ void janus_echotest_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp if(simulcast) { /* Process this simulcast packet: don't relay if it's not the SSRC/layer we wanted to handle */ relay = janus_rtp_simulcasting_context_process_rtp(&session->sim_context, - buf, len, session->ssrc, session->rid, session->vcodec, &session->context, &session->rid_mutex); + buf, len, packet->extensions.dd_content, packet->extensions.dd_len, + session->ssrc, session->rid, session->vcodec, &session->context, &session->rid_mutex); } else { /* Process this SVC packet: don't relay if it's not the layer we wanted to handle */ relay = janus_rtp_svc_context_process_rtp(&session->svc_context, - buf, len, session->vcodec, NULL, &session->context); + buf, len, packet->extensions.dd_content, packet->extensions.dd_len, session->vcodec, NULL, &session->context); } if(session->sim_context.need_pli || session->svc_context.need_pli) { /* Send a PLI */ @@ -1088,7 +1089,7 @@ static void *janus_echotest_handler(void *data) { session->sim_context.templayer_target = json_integer_value(temporal); JANUS_LOG(LOG_VERB, "Setting video temporal layer to let through (simulcast): %d (was %d)\n", session->sim_context.templayer_target, session->sim_context.templayer); - if(session->vcodec == JANUS_VIDEOCODEC_VP8 && session->sim_context.templayer_target == session->sim_context.templayer) { + if(session->sim_context.templayer_target == session->sim_context.templayer) { /* No need to do anything, we're already getting the right temporal, so notify the user */ json_t *event = json_object(); json_object_set_new(event, "echotest", json_string("event")); @@ -1280,10 +1281,6 @@ static void *janus_echotest_handler(void *data) { session->vcodec = janus_videocodec_from_name(vcodec); session->has_audio = session->acodec != JANUS_AUDIOCODEC_NONE; session->has_video = session->vcodec != JANUS_VIDEOCODEC_NONE; - if(session->vcodec != JANUS_VIDEOCODEC_VP8 && session->vcodec != JANUS_VIDEOCODEC_H264) { - /* VP8 r H.264 were not negotiated, if simulcasting was enabled then disable it here */ - janus_rtp_simulcasting_cleanup(NULL, session->ssrc, session->rid, &session->rid_mutex); - } g_free(session->vfmtp); session->vfmtp = NULL; if(session->has_video) { diff --git a/src/plugins/janus_lua.c b/src/plugins/janus_lua.c index bbe503a900..e8f47a1068 100644 --- a/src/plugins/janus_lua.c +++ b/src/plugins/janus_lua.c @@ -547,10 +547,6 @@ static int janus_lua_method_pushevent(lua_State *s) { janus_sdp_find_first_codec(parsed_sdp, JANUS_SDP_VIDEO, -1, &vcodec); if(vcodec) session->vcodec = janus_videocodec_from_name(vcodec); - if(session->vcodec != JANUS_VIDEOCODEC_VP8 && session->vcodec != JANUS_VIDEOCODEC_H264) { - /* VP8 r H.264 were not negotiated, if simulcasting was enabled then disable it here */ - janus_rtp_simulcasting_cleanup(&session->rid_extmap_id, session->ssrc, session->rid, &session->rid_mutex); - } } janus_sdp_destroy(parsed_sdp); /* Send asynchronously */ @@ -1936,10 +1932,6 @@ struct janus_plugin_result *janus_lua_handle_message(janus_plugin_session *handl janus_sdp_find_first_codec(parsed_sdp, JANUS_SDP_VIDEO, -1, &vcodec); if(vcodec) session->vcodec = janus_videocodec_from_name(vcodec); - if(session->vcodec != JANUS_VIDEOCODEC_VP8 && session->vcodec != JANUS_VIDEOCODEC_H264) { - /* VP8 r H.264 were not negotiated, if simulcasting was enabled then disable it here */ - janus_rtp_simulcasting_cleanup(&session->rid_extmap_id, session->ssrc, session->rid, &session->rid_mutex); - } janus_sdp_destroy(parsed_sdp); } if(json_is_true(json_object_get(jsep, "e2ee"))) @@ -2127,7 +2119,7 @@ void janus_lua_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *rtp_ } else { /* We're simulcasting, save the best video quality */ gboolean save = janus_rtp_simulcasting_context_process_rtp(&session->rec_simctx, - buf, len, session->ssrc, session->rid, session->vcodec, &session->rec_ctx, &session->rid_mutex); + buf, len, NULL, 0, session->ssrc, session->rid, session->vcodec, &session->rec_ctx, &session->rid_mutex); if(save) { uint32_t seq_number = ntohs(rtp->seq_number); uint32_t timestamp = ntohl(rtp->timestamp); @@ -2421,7 +2413,8 @@ static void janus_lua_relay_rtp_packet(gpointer data, gpointer user_data) { return; /* Process this packet: don't relay if it's not the SSRC/layer we wanted to handle */ gboolean relay = janus_rtp_simulcasting_context_process_rtp(&session->sim_context, - (char *)packet->data, packet->length, packet->ssrc, NULL, sender->vcodec, &session->vrtpctx, NULL); + (char *)packet->data, packet->length, packet->extensions.dd_content, packet->extensions.dd_len, + packet->ssrc, NULL, sender->vcodec, &session->vrtpctx, NULL); if(session->sim_context.need_pli && sender->handle) { /* Send a PLI */ JANUS_LOG(LOG_VERB, "We need a PLI for the simulcast context\n"); diff --git a/src/plugins/janus_recordplay.c b/src/plugins/janus_recordplay.c index d55bfe0487..0812a0bdd9 100644 --- a/src/plugins/janus_recordplay.c +++ b/src/plugins/janus_recordplay.c @@ -1304,7 +1304,8 @@ void janus_recordplay_incoming_rtp(janus_plugin_session *handle, janus_plugin_rt uint32_t ssrc = ntohl(header->ssrc); /* Process this packet: don't save if it's not the SSRC/layer we wanted to handle */ gboolean save = janus_rtp_simulcasting_context_process_rtp(&session->sim_context, - buf, len, session->ssrc, session->rid, session->recording->vcodec, &session->context, &session->rid_mutex); + buf, len, NULL, 0, session->ssrc, session->rid, session->recording->vcodec, + &session->context, &session->rid_mutex); if(session->sim_context.need_pli) { /* Send a PLI */ JANUS_LOG(LOG_VERB, "We need a PLI for the simulcast context\n"); @@ -1879,10 +1880,6 @@ static void *janus_recordplay_handler(void *data) { janus_mutex_unlock(&session->rid_mutex); session->sim_context.substream_target = 2; /* Let's aim for the highest quality */ session->sim_context.templayer_target = 2; /* Let's aim for all temporal layers */ - if(rec->vcodec != JANUS_VIDEOCODEC_VP8 && rec->vcodec != JANUS_VIDEOCODEC_H264) { - /* VP8 r H.264 were not negotiated, if simulcasting was enabled then disable it here */ - janus_rtp_simulcasting_cleanup(NULL, session->ssrc, session->rid, &session->rid_mutex); - } /* FIXME We're stopping at the first item, there may be more */ break; } diff --git a/src/plugins/janus_streaming.c b/src/plugins/janus_streaming.c index 461e7df6be..670677b21e 100644 --- a/src/plugins/janus_streaming.c +++ b/src/plugins/janus_streaming.c @@ -10030,7 +10030,8 @@ static void janus_streaming_relay_rtp_packet(gpointer data, gpointer user_data) return; /* Process this packet: don't relay if it's not the SSRC/layer we wanted to handle */ gboolean relay = janus_rtp_simulcasting_context_process_rtp(&s->sim_context, - (char *)packet->data, packet->length, packet->ssrc, NULL, packet->codec, &s->context, NULL); + (char *)packet->data, packet->length, NULL, 0, + packet->ssrc, NULL, packet->codec, &s->context, NULL); if(!relay) { /* Did a lot of time pass before we could relay a packet? */ gint64 now = janus_get_monotonic_time(); diff --git a/src/plugins/janus_videocall.c b/src/plugins/janus_videocall.c index c048e0d185..4fa5aad6e1 100644 --- a/src/plugins/janus_videocall.c +++ b/src/plugins/janus_videocall.c @@ -778,7 +778,8 @@ void janus_videocall_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp /* Process this packet: don't relay if it's not the SSRC/layer we wanted to handle * The caveat is that the targets in OUR simulcast context are the PEER's targets */ gboolean relay = janus_rtp_simulcasting_context_process_rtp(&peer->sim_context, - buf, len, session->ssrc, session->rid, session->vcodec, &peer->context, &session->rid_mutex); + buf, len, packet->extensions.dd_content, packet->extensions.dd_len, + session->ssrc, session->rid, session->vcodec, &peer->context, &session->rid_mutex); /* Do we need to drop this? */ if(!relay) return; @@ -1374,7 +1375,7 @@ static void *janus_videocall_handler(void *data) { session->has_data = (strstr(msg_sdp, "DTLS/SCTP") != NULL); /* Check if this user will simulcast */ json_t *msg_simulcast = json_object_get(msg->jsep, "simulcast"); - if(msg_simulcast && janus_get_codec_pt(msg_sdp, "vp8") > 0) { + if(msg_simulcast) { JANUS_LOG(LOG_VERB, "VideoCall callee (%s) cannot do simulcast.\n", session->username); } else { janus_rtp_simulcasting_cleanup(NULL, session->ssrc, session->rid, &session->rid_mutex); @@ -1533,7 +1534,7 @@ static void *janus_videocall_handler(void *data) { session->sim_context.templayer_target = json_integer_value(temporal); JANUS_LOG(LOG_VERB, "Setting video temporal layer to let through (simulcast): %d (was %d)\n", session->sim_context.templayer_target, session->sim_context.templayer); - if(session->vcodec == JANUS_VIDEOCODEC_VP8 && session->sim_context.templayer_target == session->sim_context.templayer) { + if(session->sim_context.templayer_target == session->sim_context.templayer) { /* No need to do anything, we're already getting the right temporal, so notify the user */ json_t *event = json_object(); json_object_set_new(event, "videocall", json_string("event")); @@ -1569,7 +1570,7 @@ static void *janus_videocall_handler(void *data) { session->has_data = (strstr(msg_sdp, "DTLS/SCTP") != NULL); /* Check if this user will simulcast */ json_t *msg_simulcast = json_object_get(msg->jsep, "simulcast"); - if(msg_simulcast && janus_get_codec_pt(msg_sdp, "vp8") > 0) { + if(msg_simulcast) { JANUS_LOG(LOG_VERB, "VideoCall callee (%s) cannot do simulcast.\n", session->username); } else { janus_rtp_simulcasting_cleanup(NULL, session->ssrc, session->rid, &session->rid_mutex); diff --git a/src/plugins/janus_videoroom.c b/src/plugins/janus_videoroom.c index 6a4896ad33..a7a7c2b550 100644 --- a/src/plugins/janus_videoroom.c +++ b/src/plugins/janus_videoroom.c @@ -530,8 +530,8 @@ room-: { "codec" : "", "description" : "", "moderated" : , - "simulcast" : "", - "svc" : "", + "simulcast" : "", + "svc" : "", "talking" : , }, // Other streams, if any @@ -675,8 +675,8 @@ room-: { "codec" : "", "description" : "", "moderated" : , - "simulcast" : "", - "svc" : "", + "simulcast" : "", + "svc" : "", "talking" : , }, // Other streams, if any @@ -1365,8 +1365,8 @@ room-: { "substream" : , "temporal" : , "fallback" : , - "spatial_layer" : , - "temporal_layer" : , + "spatial_layer" : , + "temporal_layer" : , "audio_level_average" : "", "audio_active_packets" : "", "min_delay" : , @@ -1388,7 +1388,7 @@ room-: { * that for them to work you'll have to specify the \c mid as well, as the same * subscription may be receiving simulcast stream from multiple publishers. * The \c spatial_layer and \c temporal_layer have exactly the same meaning, - * but within the context of VP9-SVC publishers, and will have no effect + * but within the context of SVC publishers, and will have no effect * on subscriptions associated to regular publishers. * * As anticipated, \c configure is also the request you use when you want @@ -1802,11 +1802,11 @@ static struct janus_json_parameter configure_stream_parameters[] = { /* For talk detection */ {"audio_level_averge", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"audio_active_packets", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, - /* For VP8 (or H.264) simulcast */ + /* For simulcast */ {"substream", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"temporal", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"fallback", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, - /* For VP9 SVC */ + /* For SVC */ {"spatial_layer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"temporal_layer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, /* For the playout-delay RTP extension, if negotiated */ @@ -1835,11 +1835,11 @@ static struct janus_json_parameter subscriber_parameters[] = { {"offer_audio", JANUS_JSON_BOOL, 0}, {"offer_video", JANUS_JSON_BOOL, 0}, {"offer_data", JANUS_JSON_BOOL, 0}, - /* For VP8 (or H.264) simulcast */ + /* For simulcast */ {"substream", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"temporal", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"fallback", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, - /* For VP9 SVC */ + /* For SVC */ {"spatial_layer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"temporal_layer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, }; @@ -1847,10 +1847,10 @@ static struct janus_json_parameter subscriber_stream_parameters[] = { {"mid", JANUS_JSON_STRING, 0}, {"crossrefid", JANUS_JSON_STRING, 0}, {"send", JANUS_JSON_BOOL, 0}, - /* For VP8 (or H.264) simulcast */ + /* For simulcast */ {"substream", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"temporal", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, - /* For VP9 SVC */ + /* For SVC */ {"spatial_layer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"temporal_layer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, /* For the playout-delay RTP extension, if negotiated */ @@ -1876,10 +1876,10 @@ static struct janus_json_parameter switch_update_parameters[] = { //~ {"feed", JANUS_JSON_INTEGER, JANUS_JSON_PARAM_REQUIRED | JANUS_JSON_PARAM_POSITIVE}, {"mid", JANUS_JSON_STRING, JANUS_JSON_PARAM_REQUIRED}, {"sub_mid", JANUS_JSON_STRING, JANUS_JSON_PARAM_REQUIRED}, - /* For VP8 (or H.264) simulcast */ + /* For simulcast */ {"substream", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"temporal", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, - /* For VP9 SVC */ + /* For SVC */ {"spatial_layer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"temporal_layer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE} }; @@ -2129,8 +2129,8 @@ typedef struct janus_videoroom_publisher_stream { gboolean opusfec; /* Whether this stream is sending inband Opus FEC */ gboolean opusdtx; /* Whether this publisher is using Opus DTX (Discontinuous Transmission) */ gboolean opusstereo; /* Whether this publisher is doing stereo Opus */ - gboolean simulcast, svc; /* Whether this stream uses simulcast or VP9 SVC */ - uint32_t vssrc[3]; /* Only needed in case VP8 (or H.264) simulcasting is involved */ + gboolean simulcast, svc; /* Whether this stream uses simulcast or SVC */ + uint32_t vssrc[3]; /* Only needed in case simulcasting is involved */ char *rid[3]; /* Only needed if simulcasting is rid-based */ int rid_extmap_id; /* rid extmap ID */ janus_mutex rid_mutex; /* Mutex to protect access to the rid array and the extmap ID */ @@ -2252,7 +2252,7 @@ typedef struct janus_videoroom_rtp_relay_packet { janus_plugin_rtp_extensions extensions; /* Whether simulcast is involved */ gboolean simulcast; - /* The following are only relevant if we're doing VP9 SVC*/ + /* The following are only relevant if we're doing SVC*/ gboolean svc; janus_vp9_svc_info svc_info; /* The following is only relevant for datachannels */ @@ -4083,7 +4083,7 @@ json_t *janus_videoroom_query_session(janus_plugin_session *handle) { if(ps->simulcast) json_object_set_new(m, "simulcast", json_true()); if(ps->svc) - json_object_set_new(m, "vp9-svc", json_true()); + json_object_set_new(m, "svc", json_true()); if(ps->rc && ps->rc->filename) json_object_set_new(m, "recording", json_string(ps->rc->filename)); if(ps->audio_level_extmap_id > 0) { @@ -8004,7 +8004,8 @@ static void janus_videoroom_incoming_rtp_internal(janus_videoroom_session *sessi } else { /* We're simulcasting, save the best video quality */ gboolean save = janus_rtp_simulcasting_context_process_rtp(&ps->rec_simctx, - buf, len, ps->vssrc, ps->rid, ps->vcodec, &ps->rec_ctx, &ps->rid_mutex); + buf, len, pkt->extensions.dd_content, pkt->extensions.dd_len, + ps->vssrc, ps->rid, ps->vcodec, &ps->rec_ctx, &ps->rid_mutex); if(save) { uint32_t seq_number = ntohs(rtp->seq_number); uint32_t timestamp = ntohl(rtp->timestamp); @@ -8036,10 +8037,14 @@ static void janus_videoroom_incoming_rtp_internal(janus_videoroom_session *sessi janus_videoroom_publisher_dereference_nodebug(participant); return; } - gboolean found = FALSE; - memset(&packet.svc_info, 0, sizeof(packet.svc_info)); - if(janus_vp9_parse_svc(payload, plen, &found, &packet.svc_info) == 0) { - packet.svc = found; + if(ps->vcodec == JANUS_VIDEOCODEC_VP9) { + gboolean found = FALSE; + memset(&packet.svc_info, 0, sizeof(packet.svc_info)); + if(janus_vp9_parse_svc(payload, plen, &found, &packet.svc_info) == 0) { + packet.svc = found; + } + } else if(ps->vcodec == JANUS_VIDEOCODEC_AV1) { + packet.svc = (pkt->extensions.dd_len > 0); } } if(video && ps->simulcast) @@ -10892,7 +10897,7 @@ static void *janus_videoroom_handler(void *data) { janus_videoroom_reqpli(ps, "Simulcasting substream change"); } } - if(ps->vcodec == JANUS_VIDEOCODEC_VP8 && ps->simulcast && sc_temporal) { + if(ps->simulcast && sc_temporal) { stream->sim_context.templayer_target = json_integer_value(sc_temporal); JANUS_LOG(LOG_VERB, "Setting video temporal layer to let through (simulcast): %d (was %d)\n", stream->sim_context.templayer_target, stream->sim_context.templayer); @@ -11803,8 +11808,7 @@ static void *janus_videoroom_handler(void *data) { } } /* Check if simulcast or SVC is in place */ - if(msg_simulcast != NULL && json_array_size(msg_simulcast) > 0 && - (ps->vcodec == JANUS_VIDEOCODEC_VP8 || ps->vcodec == JANUS_VIDEOCODEC_H264)) { + if(msg_simulcast != NULL && json_array_size(msg_simulcast) > 0) { size_t i = 0; for(i=0; irid_mutex); } } else if(msg_svc != NULL && json_array_size(msg_svc) > 0 && - ps->vcodec == JANUS_VIDEOCODEC_VP9) { + (ps->vcodec == JANUS_VIDEOCODEC_VP9 || ps->vcodec == JANUS_VIDEOCODEC_AV1)) { size_t i = 0; for(i=0; ivideoorient_ext ? JANUS_RTP_EXTMAP_VIDEO_ORIENTATION : NULL, JANUS_SDP_OA_ACCEPT_EXTMAP, videoroom->playoutdelay_ext ? JANUS_RTP_EXTMAP_PLAYOUT_DELAY : NULL, JANUS_SDP_OA_ACCEPT_EXTMAP, videoroom->transport_wide_cc_ext ? JANUS_RTP_EXTMAP_TRANSPORT_WIDE_CC : NULL, @@ -12157,7 +12162,8 @@ static void janus_videoroom_relay_rtp_packet(gpointer data, gpointer user_data) /* Process this packet: don't relay if it's not the layer we wanted to handle */ janus_rtp_header rtp = *(packet->data); gboolean relay = janus_rtp_svc_context_process_rtp(&stream->svc_context, - (char *)packet->data, packet->length, ps->vcodec, &packet->svc_info, &stream->context); + (char *)packet->data, packet->length, packet->extensions.dd_content, packet->extensions.dd_len, + ps->vcodec, &packet->svc_info, &stream->context); if(stream->svc_context.need_pli) { /* Send a PLI */ JANUS_LOG(LOG_VERB, "We need a PLI for the SVC context\n"); @@ -12209,7 +12215,8 @@ static void janus_videoroom_relay_rtp_packet(gpointer data, gpointer user_data) return; /* Process this packet: don't relay if it's not the SSRC/layer we wanted to handle */ gboolean relay = janus_rtp_simulcasting_context_process_rtp(&stream->sim_context, - (char *)packet->data, packet->length, packet->ssrc, NULL, ps->vcodec, &stream->context, &ps->rid_mutex); + (char *)packet->data, packet->length, packet->extensions.dd_content, packet->extensions.dd_len, + packet->ssrc, NULL, ps->vcodec, &stream->context, &ps->rid_mutex); if(!relay) { /* Did a lot of time pass before we could relay a packet? */ gint64 now = janus_get_monotonic_time(); diff --git a/src/plugins/lua/janus-sdp.lua b/src/plugins/lua/janus-sdp.lua index 85f062f946..03666ed0b6 100644 --- a/src/plugins/lua/janus-sdp.lua +++ b/src/plugins/lua/janus-sdp.lua @@ -446,6 +446,8 @@ function JANUSSDP.generateAnswer(offer, options) answer[#answer+1] = { type = "a", name = a.name, value = value } elseif options.enableAudioLevel ~= false and a.value:find("urn:ietf:params:rtp-hdrext:ssrc-audio-level", 1, true) then answer[#answer+1] = a + elseif options.enableAudioLevel ~= false and a.value:find("dependency-descriptor-rtp-header-extension", 1, true) then + answer[#answer+1] = a end end else diff --git a/src/rtp.c b/src/rtp.c index 52e8296d07..086ff19054 100644 --- a/src/rtp.c +++ b/src/rtp.c @@ -1023,6 +1023,9 @@ void janus_rtp_simulcasting_context_reset(janus_rtp_simulcasting_context *contex if(context == NULL) return; /* Reset the context values */ + janus_av1_svc_context_reset(&context->av1_context[0]); + janus_av1_svc_context_reset(&context->av1_context[1]); + janus_av1_svc_context_reset(&context->av1_context[2]); memset(context, 0, sizeof(*context)); context->rid_ext_id = -1; context->substream = -1; @@ -1081,7 +1084,7 @@ void janus_rtp_simulcasting_cleanup(int *rid_ext_id, uint32_t *ssrcs, char **rid } gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_context *context, - char *buf, int len, uint32_t *ssrcs, char **rids, + char *buf, int len, uint8_t *dd_content, int dd_len, uint32_t *ssrcs, char **rids, janus_videocodec vcodec, janus_rtp_switching_context *sc, janus_mutex *rid_mutex) { if(!context || !buf || len < 1) return FALSE; @@ -1146,7 +1149,9 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte if(context->substream == -1) { if((vcodec == JANUS_VIDEOCODEC_VP8 && janus_vp8_is_keyframe(payload, plen)) || (vcodec == JANUS_VIDEOCODEC_VP9 && janus_vp9_is_keyframe(payload, plen)) || - (vcodec == JANUS_VIDEOCODEC_H264 && janus_h264_is_keyframe(payload, plen))) { + (vcodec == JANUS_VIDEOCODEC_H264 && janus_h264_is_keyframe(payload, plen)) || + (vcodec == JANUS_VIDEOCODEC_AV1 && janus_av1_is_keyframe(payload, plen)) || + (vcodec == JANUS_VIDEOCODEC_H265 && janus_h265_is_keyframe(payload, plen))) { context->substream = substream; /* Notify the caller that the substream changed */ context->changed_substream = TRUE; @@ -1161,7 +1166,9 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte (context->substream > target && substream < context->substream)) && ((vcodec == JANUS_VIDEOCODEC_VP8 && janus_vp8_is_keyframe(payload, plen)) || (vcodec == JANUS_VIDEOCODEC_VP9 && janus_vp9_is_keyframe(payload, plen)) || - (vcodec == JANUS_VIDEOCODEC_H264 && janus_h264_is_keyframe(payload, plen)))) { + (vcodec == JANUS_VIDEOCODEC_H264 && janus_h264_is_keyframe(payload, plen)) || + (vcodec == JANUS_VIDEOCODEC_AV1 && janus_av1_is_keyframe(payload, plen)) || + (vcodec == JANUS_VIDEOCODEC_H265 && janus_h265_is_keyframe(payload, plen)))) { JANUS_LOG(LOG_VERB, "Received keyframe on #%d (SSRC %"SCNu32"), switching (was #%d/%"SCNu32")\n", substream, ssrc, context->substream, *(ssrcs + context->substream)); context->substream = substream; @@ -1206,7 +1213,7 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte return FALSE; } context->last_relayed = janus_get_monotonic_time(); - /* Temporal layers are only available for VP8, so don't do anything else for other codecs */ + /* Temporal layers are only easily available for some codecs */ if(vcodec == JANUS_VIDEOCODEC_VP8) { /* Check if there's any temporal scalability to take into account */ gboolean m = FALSE; @@ -1232,6 +1239,73 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte return FALSE; } } + } else if(vcodec == JANUS_VIDEOCODEC_VP9) { + /* We use the VP9 SVC parser to extract info on temporal layers */ + gboolean found = FALSE; + janus_vp9_svc_info svc_info = { 0 }; + if(janus_vp9_parse_svc(payload, plen, &found, &svc_info) == 0 && found) { + int temporal_layer = context->templayer; + if(context->templayer_target > context->templayer) { + /* We need to upscale */ + if(svc_info.ubit && svc_info.bbit && + svc_info.temporal_layer > context->templayer && + svc_info.temporal_layer <= context->templayer_target) { + context->templayer = svc_info.temporal_layer; + temporal_layer = context->templayer; + context->changed_temporal = TRUE; + } + } else if(context->templayer_target < context->templayer) { + /* We need to downscale */ + if(svc_info.ebit && svc_info.temporal_layer == context->templayer_target) { + context->templayer = context->templayer_target; + context->changed_temporal = TRUE; + } + } + if(temporal_layer < svc_info.temporal_layer) { + JANUS_LOG(LOG_HUGE, "Dropping packet (it's temporal layer %d, but we're capping at %d)\n", + svc_info.temporal_layer, context->templayer); + /* We increase the base sequence number, or there will be gaps when delivering later */ + if(sc) + sc->base_seq++; + return FALSE; + } + } + } else if(vcodec == JANUS_VIDEOCODEC_AV1 && dd_content != NULL && dd_len > 0) { + /* Use the Dependency Descriptor to check temporal layers */ + janus_av1_svc_context *av1ctx = NULL; + if(context->substream >= 0 && context->substream <= 2) + av1ctx = &context->av1_context[context->substream]; + if(av1ctx != NULL) { + uint8_t template = 0; + if(janus_av1_svc_context_process_dd(av1ctx, dd_content, dd_len, &template, NULL)) { + janus_av1_svc_template *t = g_hash_table_lookup(av1ctx->templates, GUINT_TO_POINTER(template)); + if(t) { + int temporal_layer = context->templayer; + if(context->templayer_target > context->templayer) { + /* We need to upscale */ + if(t->temporal > context->templayer && t->temporal <= context->templayer_target) { + context->templayer = t->temporal; + temporal_layer = context->templayer; + context->changed_temporal = TRUE; + } + } else if(context->templayer_target < context->templayer) { + /* We need to downscale */ + if(t->temporal == context->templayer_target) { + context->templayer = context->templayer_target; + context->changed_temporal = TRUE; + } + } + if(temporal_layer < t->temporal) { + JANUS_LOG(LOG_HUGE, "Dropping packet (it's temporal layer %d, but we're capping at %d)\n", + t->temporal, context->templayer); + /* We increase the base sequence number, or there will be gaps when delivering later */ + if(sc) + sc->base_seq++; + return FALSE; + } + } + } + } } /* If we got here, the packet can be relayed */ return TRUE; @@ -1242,14 +1316,16 @@ void janus_rtp_svc_context_reset(janus_rtp_svc_context *context) { if(context == NULL) return; /* Reset the context values */ + janus_av1_svc_context_reset(&context->dd_context); memset(context, 0, sizeof(*context)); context->spatial = -1; context->temporal = -1; } -gboolean janus_rtp_svc_context_process_rtp(janus_rtp_svc_context *context, char *buf, int len, +gboolean janus_rtp_svc_context_process_rtp(janus_rtp_svc_context *context, + char *buf, int len, uint8_t *dd_content, int dd_len, janus_videocodec vcodec, janus_vp9_svc_info *info, janus_rtp_switching_context *sc) { - if(!context || !buf || len < 1 || vcodec != JANUS_VIDEOCODEC_VP9) + if(!context || !buf || len < 1 || (vcodec != JANUS_VIDEOCODEC_VP9 && vcodec != JANUS_VIDEOCODEC_AV1)) return FALSE; janus_rtp_header *header = (janus_rtp_header *)buf; /* Reset the flags */ @@ -1262,7 +1338,108 @@ gboolean janus_rtp_svc_context_process_rtp(janus_rtp_svc_context *context, char char *payload = janus_rtp_payload(buf, len, &plen); if(payload == NULL) return FALSE; - /* If we don't have any info parsed from the VP9 payload header, get it now */ + /* Check if we should use the Dependency Descriptor */ + if(vcodec == JANUS_VIDEOCODEC_AV1) { + /* We do, make sure the data is there */ + if(dd_content == NULL || dd_len < 1) { + /* No Dependency Descriptor, relay as it is */ + return TRUE; + } + uint8_t template = 0, ebit = 0; + if(!janus_av1_svc_context_process_dd(&context->dd_context, dd_content, dd_len, &template, &ebit)) { + /* We couldn't parse the Dependency Descriptor, relay as it is */ + return TRUE; + } + janus_av1_svc_template *t = g_hash_table_lookup(context->dd_context.templates, GUINT_TO_POINTER(template)); + if(t == NULL) { + /* We couldn't find the template, relay as it is */ + return TRUE; + } + /* Now let's check if we should let the packet through or not */ + gboolean keyframe = janus_av1_is_keyframe((const char *)payload, plen); + gboolean override_mark_bit = FALSE, has_marker_bit = header->markerbit; + int spatial_layer = context->spatial; + if(t->spatial >= 0 && t->spatial <= 2) + context->last_spatial_layer[t->spatial] = now; + if(context->spatial_target > context->spatial) { + JANUS_LOG(LOG_HUGE, "We need to upscale spatially: (%d < %d)\n", + context->spatial, context->spatial_target); + /* We need to upscale: wait for a keyframe */ + if(keyframe) { + int new_spatial_layer = context->spatial_target; + while(new_spatial_layer > context->spatial && new_spatial_layer > 0) { + if(now - context->last_spatial_layer[new_spatial_layer] >= (context->drop_trigger ? context->drop_trigger : 250000)) { + /* We haven't received packets from this layer for a while, try a lower layer */ + JANUS_LOG(LOG_HUGE, "Haven't received packets from layer %d for a while, trying %d instead...\n", + new_spatial_layer, new_spatial_layer-1); + new_spatial_layer--; + } else { + break; + } + } + if(new_spatial_layer > context->spatial) { + JANUS_LOG(LOG_HUGE, " -- Upscaling spatial layer: %d --> %d (need %d)\n", + context->spatial, new_spatial_layer, context->spatial_target); + context->spatial = new_spatial_layer; + spatial_layer = context->spatial; + context->changed_spatial = TRUE; + } + } + } else if(context->spatial_target < context->spatial) { + /* We need to scale: wait for a keyframe */ + JANUS_LOG(LOG_HUGE, "We need to downscale spatially: (%d > %d)\n", + context->spatial, context->spatial_target); + /* Check the E bit to see if this is an end-of-frame */ + if(ebit) { + JANUS_LOG(LOG_HUGE, " -- Downscaling spatial layer: %d --> %d\n", + context->spatial, context->spatial_target); + context->spatial = context->spatial_target; + context->changed_spatial = TRUE; + } + } + if(spatial_layer < t->spatial) { + /* Drop the packet: update the context to make sure sequence number is increased normally later */ + JANUS_LOG(LOG_HUGE, "Dropping packet (spatial layer %d < %d)\n", spatial_layer, t->spatial); + if(sc) + sc->base_seq++; + return FALSE; + } else if(ebit && spatial_layer == t->spatial) { + /* If we stop at layer 0, we need a marker bit now, as the one from layer 1 will not be received */ + override_mark_bit = TRUE; + } + int temporal = context->temporal; + if(context->temporal_target > context->temporal) { + /* We need to upscale */ + if(t->temporal > context->temporal && t->temporal <= context->temporal_target) { + context->temporal = t->temporal; + temporal = context->temporal; + context->changed_temporal = TRUE; + } + } else if(context->temporal_target < context->temporal) { + /* We need to downscale */ + if(t->temporal == context->temporal_target) { + context->temporal = context->temporal_target; + context->changed_temporal = TRUE; + } + } + if(temporal < t->temporal) { + JANUS_LOG(LOG_HUGE, "Dropping packet (it's temporal layer %d, but we're capping at %d)\n", + t->temporal, context->temporal); + /* We increase the base sequence number, or there will be gaps when delivering later */ + if(sc) + sc->base_seq++; + return FALSE; + } + /* If we got here, we can send the frame: this doesn't necessarily mean it's + * one of the layers the user wants, as there may be dependencies involved */ + JANUS_LOG(LOG_HUGE, "Sending packet (spatial=%d, temporal=%d)\n", + t->spatial, t->temporal); + if(override_mark_bit && !has_marker_bit) + header->markerbit = 1; + return TRUE; + } + /* If we got here, it's VP9, for which we parse the payload manually: + * if we don't have any info parsed from the VP9 payload header, get it now */ janus_vp9_svc_info svc_info = { 0 }; if(!info) { gboolean found = FALSE; @@ -1390,7 +1567,7 @@ void janus_av1_svc_context_reset(janus_av1_svc_context *context) { } gboolean janus_av1_svc_context_process_dd(janus_av1_svc_context *context, - uint8_t *dd, int dd_len, uint8_t *template_id) { + uint8_t *dd, int dd_len, uint8_t *template_id, uint8_t *ebit) { if(!context || !dd || dd_len < 3) return FALSE; @@ -1400,9 +1577,11 @@ gboolean janus_av1_svc_context_process_dd(janus_av1_svc_context *context, /* mandatory_descriptor_fields() */ uint8_t start = janus_bitstream_getbit(dd, offset++); uint8_t end = janus_bitstream_getbit(dd, offset++); + if(ebit) + *ebit = end; uint8_t template = janus_bitstream_getbits(dd, 6, &offset); uint16_t frame = janus_bitstream_getbits(dd, 16, &offset); - JANUS_LOG(LOG_WARN, " -- s=%u, e=%u, t=%u, f=%u\n", + JANUS_LOG(LOG_HUGE, " -- s=%u, e=%u, t=%u, f=%u\n", start, end, template, frame); if(blen > 24) { /* extended_descriptor_fields() */ @@ -1434,7 +1613,7 @@ gboolean janus_av1_svc_context_process_dd(janus_av1_svc_context *context, } t->spatial = spatial_layers; t->temporal = temporal_layers; - JANUS_LOG(LOG_WARN, " -- -- -- [%u] spatial=%u, temporal=%u\n", + JANUS_LOG(LOG_HUGE, " -- -- -- [%u] spatial=%u, temporal=%u\n", tcnt, t->spatial, t->temporal); if(nlidc == 1) { temporal_layers++; @@ -1458,14 +1637,14 @@ gboolean janus_av1_svc_context_process_dd(janus_av1_svc_context *context, } /* frame_dependency_definition() */ uint8_t tindex = (template + 64 - context->tioff) % 64; - janus_av1_svc_template *t = g_hash_table_lookup(context->templates, - GUINT_TO_POINTER(tindex)); + janus_av1_svc_template *t = context->templates ? g_hash_table_lookup(context->templates, + GUINT_TO_POINTER(tindex)) : NULL; if(t == NULL) { JANUS_LOG(LOG_WARN, "Invalid template ID '%u' (count is %u), ignoring packet...\n", tindex, context->tcnt); return FALSE; } - JANUS_LOG(LOG_WARN, " -- spatial=%u, temporal=%u (tindex %u)\n", + JANUS_LOG(LOG_HUGE, " -- spatial=%u, temporal=%u (tindex %u)\n", t->spatial, t->temporal, t->id); /* FIXME We currently don't care about the other fields */ diff --git a/src/rtp.h b/src/rtp.h index 4b9a0dbba2..cebb8b3c86 100644 --- a/src/rtp.h +++ b/src/rtp.h @@ -295,6 +295,54 @@ int janus_rtp_skew_compensate_audio(janus_rtp_header *header, janus_rtp_switchin int janus_rtp_skew_compensate_video(janus_rtp_header *header, janus_rtp_switching_context *context, gint64 now); +/** @name Janus AV1-SVC processing methods + */ +///@{ +/*! \brief Helper struct for processing and tracking AV1-SVC streams */ +typedef struct janus_av1_svc_context { + /*! \brief Number of templates advertised via Dependency Descriptor */ + uint8_t tcnt; + /*! \brief Template ID offset, as advertised via Dependency Descriptor */ + uint8_t tioff; + /*! \brief Map of templates advertised via Dependency Descriptor, indexed by ID */ + GHashTable *templates; + /*! \brief How many spatial and temporal layers are available */ + int spatial_layers, temporal_layers; + /*! \brief Whether this context changed since the last update */ + gboolean updated; +} janus_av1_svc_context; + +/*! \brief Helper struct to track SVC templates + * \note This is very incomplete, since we only track the spatial and + * temporal layer associated with a specific template ID for now */ +typedef struct janus_av1_svc_template { + /*! \brief Template ID */ + uint8_t id; + /*! \brief Spatial layer associated to this template */ + int spatial; + /*! \brief Temporal layer associated to this template */ + int temporal; +} janus_av1_svc_template; + +/*! \brief Set (or reset) the context fields to their default values + * @param[in] context The context to (re)set */ +void janus_av1_svc_context_reset(janus_av1_svc_context *context); + +/*! \brief Process a Dependency Descriptor payload, updating the SVC context accordingly + * \note At the moment, this code is quite naive, as it mostly looks at the target + * spatial/temporal layers, and the one written in the Dependency Descriptor data. + * In the future, this should become more sophisticated, and use additional + * information like dependency chains and stuff like that + * @param[in] context The av1svc context to use + * @param[in] dd Pointer to the Dependency Descriptor data + * @param[in] dd_len The length of the Dependendy Descriptor data + * @param[out] template_id Pointer to the ID of the template referenced in this packet + * @param[out] ebit Whether this packet is an end of frame or not + * @returns TRUE if the packet is valid, FALSE if it should be dropped instead */ +gboolean janus_av1_svc_context_process_dd(janus_av1_svc_context *context, + uint8_t *dd, int dd_len, uint8_t *template_id, uint8_t *ebit); +///@} + /** @name Janus simulcast processing methods */ ///@{ @@ -302,6 +350,8 @@ int janus_rtp_skew_compensate_video(janus_rtp_header *header, janus_rtp_switchin typedef struct janus_rtp_simulcasting_context { /*! \brief RTP Stream extension ID, if any */ gint rid_ext_id; + /*! \brief Dependency Descriptors contexts, if any */ + janus_av1_svc_context av1_context[3]; /*! \brief Which simulcast substream we should forward back */ int substream; /*! \brief As above, but to handle transitions (e.g., wait for keyframe, or get this if available) */ @@ -348,6 +398,8 @@ void janus_rtp_simulcasting_cleanup(int *rid_ext_id, uint32_t *ssrcs, char **rid * @param[in] context The simulcasting context to use * @param[in] buf The RTP packet to process * @param[in] len The length of the RTP packet (header, extension and payload) + * @param[in] dd_content The Dependency Descriptor RTP extension data, if available + * @param[in] dd_len Length of the Dependency Descriptor data, if available * @param[in] ssrcs The simulcast SSRCs to refer to (may be updated if rids are involved) * @param[in] rids The simulcast rids to refer to, if any * @param[in] vcodec Video codec of the RTP payload @@ -355,7 +407,7 @@ void janus_rtp_simulcasting_cleanup(int *rid_ext_id, uint32_t *ssrcs, char **rid * @param[in] rid_mutex A mutex that must be acquired before reading the rids array, if any * @returns TRUE if the packet should be relayed, FALSE if it should be dropped instead */ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_context *context, - char *buf, int len, uint32_t *ssrcs, char **rids, + char *buf, int len, uint8_t *dd_content, int dd_len, uint32_t *ssrcs, char **rids, janus_videocodec vcodec, janus_rtp_switching_context *sc, janus_mutex *rid_mutex); ///@} @@ -364,6 +416,8 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte ///@{ /*! \brief Helper struct for processing and tracking VP9-SVC streams */ typedef struct janus_rtp_svc_context { + /*! \brief Dependency Descriptor context, in case it's needed */ + struct janus_av1_svc_context dd_context; /*! \brief Which SVC spatial layer we should forward back */ int spatial; /*! \brief As above, but to handle transitions (e.g., wait for keyframe, or get this if available) */ @@ -394,60 +448,15 @@ void janus_rtp_svc_context_reset(janus_rtp_svc_context *context); * @param[in] context The VP9 SVC context to use * @param[in] buf The RTP packet to process * @param[in] len The length of the RTP packet (header, extension and payload) + * @param[in] dd_content The Dependency Descriptor RTP extension data, if available + * @param[in] dd_len Length of the Dependency Descriptor data, if available * @param[in] vcodec Video codec of the RTP payload * @param[in] info Parsed info on VP9-SVC, if any * @param[in] sc RTP switching context to refer to, if any * @returns TRUE if the packet should be relayed, FALSE if it should be dropped instead */ -gboolean janus_rtp_svc_context_process_rtp(janus_rtp_svc_context *context, char *buf, int len, +gboolean janus_rtp_svc_context_process_rtp(janus_rtp_svc_context *context, + char *buf, int len, uint8_t *dd_content, int dd_len, janus_videocodec vcodec, janus_vp9_svc_info *info, janus_rtp_switching_context *sc); ///@} -/** @name Janus AV1-SVC processing methods (still WIP) - */ -///@{ -/*! \brief Helper struct for processing and tracking AV1-SVC streams */ -typedef struct janus_av1_svc_context { - /*! \brief Number of templates advertised via Dependency Descriptor */ - uint8_t tcnt; - /*! \brief Template ID offset, as advertised via Dependency Descriptor */ - uint8_t tioff; - /*! \brief Map of templates advertised via Dependency Descriptor, indexed by ID */ - GHashTable *templates; - /*! \brief How many spatial and temporal layers are available */ - int spatial_layers, temporal_layers; - /*! \brief Whether this context changed since the last update */ - gboolean updated; -} janus_av1_svc_context; - -/*! \brief Helper struct to track SVC templates - * \note This is very incomplete, since we only track the spatial and - * temporal layer associated with a specific template ID for now */ -typedef struct janus_av1_svc_template { - /*! \brief Template ID */ - uint8_t id; - /*! \brief Spatial layer associated to this template */ - int spatial; - /*! \brief Temporal layer associated to this template */ - int temporal; -} janus_av1_svc_template; - -/*! \brief Set (or reset) the context fields to their default values - * @param[in] context The context to (re)set */ -void janus_av1_svc_context_reset(janus_av1_svc_context *context); - -/*! \brief Process a Dependency Descriptor payload, updating the SVC context accordingly - * \note At the moment, this code is quite naive, as it mostly looks at the target - * spatial/temporal layers, and the one written in the Dependency Descriptor data. - * In the future, this should become more sophisticated, and use additional - * information like dependency chains and stuff like that - * @param[in] context The av1svc context to use - * @param[in] dd Pointer to the Dependency Descriptor data - * @param[in] dd_len The length of the Dependendy Descriptor data - * @param[out] template_id Pointer to the ID of the template referenced in this packet - * @returns TRUE if the packet is valid, FALSE if it should be dropped instead */ -gboolean janus_av1_svc_context_process_dd(janus_av1_svc_context *context, - uint8_t *dd, int dd_len, uint8_t *template_id); -///@} - - #endif diff --git a/src/rtpfwd.c b/src/rtpfwd.c index b5819e174a..c347055a8b 100644 --- a/src/rtpfwd.c +++ b/src/rtpfwd.c @@ -354,7 +354,7 @@ void janus_rtp_forwarder_send_rtp_full(janus_rtp_forwarder *rf, char *buffer, in if(rf->is_video && rf->simulcast) { /* This is video and we're simulcasting, check if we need to forward this frame */ if(!janus_rtp_simulcasting_context_process_rtp(&rf->sim_context, - buffer, len, ssrcs, rids, vcodec, &rf->rtp_context, rid_mutex)) { + buffer, len, NULL, 0, ssrcs, rids, vcodec, &rf->rtp_context, rid_mutex)) { /* There was an error processing simulcasting for this packet */ return; }