From 73c3de200de8b67a36ec5984fd9379f38e0c2ee5 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Wed, 10 May 2023 18:27:41 +0200 Subject: [PATCH 1/6] Initial support for VP9 and AV1 simulcast --- html/janus.js | 6 +++--- src/plugins/janus_echotest.c | 6 +----- src/rtp.c | 8 ++++++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/html/janus.js b/html/janus.js index 344ae52686..8dde64eaf5 100644 --- a/html/janus.js +++ b/html/janus.js @@ -2497,9 +2497,9 @@ function Janus(gatewayCallbacks) { direction: 'sendrecv', streams: [config.myStream], sendEncodings: track.sendEncodings || [ - { rid: 'h', active: true, maxBitrate: maxBitrates.high }, - { rid: 'm', active: true, maxBitrate: maxBitrates.medium, scaleResolutionDownBy: 2 }, - { rid: 'l', active: true, maxBitrate: maxBitrates.low, scaleResolutionDownBy: 4 } + { rid: 'h', active: true, scalabilityMode: 'L1T2', maxBitrate: maxBitrates.high }, + { rid: 'm', active: true, scalabilityMode: 'L1T2', maxBitrate: maxBitrates.medium, scaleResolutionDownBy: 2 }, + { rid: 'l', active: true, scalabilityMode: 'L1T2', maxBitrate: maxBitrates.low, scaleResolutionDownBy: 4 } ] }); } else { diff --git a/src/plugins/janus_echotest.c b/src/plugins/janus_echotest.c index 605fc672ee..e5c90571fa 100644 --- a/src/plugins/janus_echotest.c +++ b/src/plugins/janus_echotest.c @@ -1088,7 +1088,7 @@ static void *janus_echotest_handler(void *data) { session->sim_context.templayer_target = json_integer_value(temporal); JANUS_LOG(LOG_VERB, "Setting video temporal layer to let through (simulcast): %d (was %d)\n", session->sim_context.templayer_target, session->sim_context.templayer); - if(session->vcodec == JANUS_VIDEOCODEC_VP8 && session->sim_context.templayer_target == session->sim_context.templayer) { + if(session->sim_context.templayer_target == session->sim_context.templayer) { /* No need to do anything, we're already getting the right temporal, so notify the user */ json_t *event = json_object(); json_object_set_new(event, "echotest", json_string("event")); @@ -1280,10 +1280,6 @@ static void *janus_echotest_handler(void *data) { session->vcodec = janus_videocodec_from_name(vcodec); session->has_audio = session->acodec != JANUS_AUDIOCODEC_NONE; session->has_video = session->vcodec != JANUS_VIDEOCODEC_NONE; - if(session->vcodec != JANUS_VIDEOCODEC_VP8 && session->vcodec != JANUS_VIDEOCODEC_H264) { - /* VP8 r H.264 were not negotiated, if simulcasting was enabled then disable it here */ - janus_rtp_simulcasting_cleanup(NULL, session->ssrc, session->rid, &session->rid_mutex); - } g_free(session->vfmtp); session->vfmtp = NULL; if(session->has_video) { diff --git a/src/rtp.c b/src/rtp.c index 52e8296d07..e71a48a443 100644 --- a/src/rtp.c +++ b/src/rtp.c @@ -1146,7 +1146,9 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte if(context->substream == -1) { if((vcodec == JANUS_VIDEOCODEC_VP8 && janus_vp8_is_keyframe(payload, plen)) || (vcodec == JANUS_VIDEOCODEC_VP9 && janus_vp9_is_keyframe(payload, plen)) || - (vcodec == JANUS_VIDEOCODEC_H264 && janus_h264_is_keyframe(payload, plen))) { + (vcodec == JANUS_VIDEOCODEC_H264 && janus_h264_is_keyframe(payload, plen)) || + (vcodec == JANUS_VIDEOCODEC_AV1 && janus_av1_is_keyframe(payload, plen)) || + (vcodec == JANUS_VIDEOCODEC_H265 && janus_h265_is_keyframe(payload, plen))) { context->substream = substream; /* Notify the caller that the substream changed */ context->changed_substream = TRUE; @@ -1161,7 +1163,9 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte (context->substream > target && substream < context->substream)) && ((vcodec == JANUS_VIDEOCODEC_VP8 && janus_vp8_is_keyframe(payload, plen)) || (vcodec == JANUS_VIDEOCODEC_VP9 && janus_vp9_is_keyframe(payload, plen)) || - (vcodec == JANUS_VIDEOCODEC_H264 && janus_h264_is_keyframe(payload, plen)))) { + (vcodec == JANUS_VIDEOCODEC_H264 && janus_h264_is_keyframe(payload, plen)) || + (vcodec == JANUS_VIDEOCODEC_AV1 && janus_av1_is_keyframe(payload, plen)) || + (vcodec == JANUS_VIDEOCODEC_H265 && janus_h265_is_keyframe(payload, plen)))) { JANUS_LOG(LOG_VERB, "Received keyframe on #%d (SSRC %"SCNu32"), switching (was #%d/%"SCNu32")\n", substream, ssrc, context->substream, *(ssrcs + context->substream)); context->substream = substream; From c64793b130efc77179940e459beb8d691f489703 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Thu, 11 May 2023 11:27:55 +0200 Subject: [PATCH 2/6] Read VP9 temporal layers from payload descriptor (SVC code) --- html/echotest.js | 2 +- src/rtp.c | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/html/echotest.js b/html/echotest.js index bdd880f68c..5ccc310485 100644 --- a/html/echotest.js +++ b/html/echotest.js @@ -203,7 +203,7 @@ $(document).ready(function() { if((substream !== null && substream !== undefined) || (temporal !== null && temporal !== undefined)) { if(!simulcastStarted) { simulcastStarted = true; - addSimulcastSvcButtons(msg["videocodec"] === "vp8"); + addSimulcastSvcButtons(msg["videocodec"] === "vp8" || msg["videocodec"] === "vp9"); } // We just received notice that there's been a switch, update the buttons updateSimulcastSvcButtons(substream, temporal); diff --git a/src/rtp.c b/src/rtp.c index e71a48a443..e447448711 100644 --- a/src/rtp.c +++ b/src/rtp.c @@ -1210,7 +1210,7 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte return FALSE; } context->last_relayed = janus_get_monotonic_time(); - /* Temporal layers are only available for VP8, so don't do anything else for other codecs */ + /* Temporal layers are only easily available for some codecs */ if(vcodec == JANUS_VIDEOCODEC_VP8) { /* Check if there's any temporal scalability to take into account */ gboolean m = FALSE; @@ -1236,6 +1236,37 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte return FALSE; } } + } else if(vcodec == JANUS_VIDEOCODEC_VP9) { + /* We use the VP9 SVC parser to extract info on temporal layers */ + gboolean found = FALSE; + janus_vp9_svc_info svc_info = { 0 }; + if(janus_vp9_parse_svc(payload, plen, &found, &svc_info) == 0 && found) { + int temporal_layer = context->templayer; + if(context->templayer_target > context->templayer) { + /* We need to upscale */ + if(svc_info.ubit && svc_info.bbit && + svc_info.temporal_layer > context->templayer && + svc_info.temporal_layer <= context->templayer_target) { + context->templayer = svc_info.temporal_layer; + temporal_layer = context->templayer; + context->changed_temporal = TRUE; + } + } else if(context->templayer_target < context->templayer) { + /* We need to downscale */ + if(svc_info.ebit && svc_info.temporal_layer == context->templayer_target) { + context->templayer = context->templayer_target; + context->changed_temporal = TRUE; + } + } + if(temporal_layer < svc_info.temporal_layer) { + JANUS_LOG(LOG_HUGE, "Dropping packet (it's temporal layer %d, but we're capping at %d)\n", + svc_info.temporal_layer, context->templayer); + /* We increase the base sequence number, or there will be gaps when delivering later */ + if(sc) + sc->base_seq++; + return FALSE; + } + } } /* If we got here, the packet can be relayed */ return TRUE; From cf6eb00ac715a1be10b1949ea62022cfb5b389f8 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Thu, 11 May 2023 19:54:04 +0200 Subject: [PATCH 3/6] Use Dependency Descriptor for temporal layers in AV1 simulcasting, if available --- src/plugins/janus_echotest.c | 48 ++++++++++++++++++++++++++++++++++++ src/rtp.c | 10 ++++---- 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/src/plugins/janus_echotest.c b/src/plugins/janus_echotest.c index e5c90571fa..6c88210ff1 100644 --- a/src/plugins/janus_echotest.c +++ b/src/plugins/janus_echotest.c @@ -253,6 +253,7 @@ typedef struct janus_echotest_session { janus_vp8_simulcast_context vp8_context; gboolean svc; janus_rtp_svc_context svc_context; + janus_av1_svc_context av1_context[3]; /* Dependency Descriptors can be used in VP9/AV1 simulcast too */ janus_recorder *arc; /* The Janus recorder instance for this user's audio, if enabled */ janus_recorder *vrc; /* The Janus recorder instance for this user's video, if enabled */ janus_recorder *drc; /* The Janus recorder instance for this user's data, if enabled */ @@ -280,6 +281,9 @@ static void janus_echotest_session_free(const janus_refcount *session_ref) { g_free(session->vfmtp); janus_mutex_destroy(&session->rid_mutex); janus_mutex_destroy(&session->rec_mutex); + janus_av1_svc_context_reset(&session->av1_context[0]); + janus_av1_svc_context_reset(&session->av1_context[1]); + janus_av1_svc_context_reset(&session->av1_context[2]); janus_rtp_simulcasting_cleanup(NULL, NULL, session->rid, NULL); g_free(session); } @@ -448,6 +452,9 @@ void janus_echotest_create_session(janus_plugin_session *handle, int *error) { janus_rtp_simulcasting_context_reset(&session->sim_context); janus_vp8_simulcast_context_reset(&session->vp8_context); janus_rtp_svc_context_reset(&session->svc_context); + janus_av1_svc_context_reset(&session->av1_context[0]); + janus_av1_svc_context_reset(&session->av1_context[1]); + janus_av1_svc_context_reset(&session->av1_context[2]); janus_mutex_init(&session->rid_mutex); session->min_delay = -1; session->max_delay = -1; @@ -625,6 +632,41 @@ void janus_echotest_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp /* Process this simulcast packet: don't relay if it's not the SSRC/layer we wanted to handle */ relay = janus_rtp_simulcasting_context_process_rtp(&session->sim_context, buf, len, session->ssrc, session->rid, session->vcodec, &session->context, &session->rid_mutex); + if(relay && session->vcodec == JANUS_VIDEOCODEC_AV1 && packet->extensions.dd_len > 0) { + /* Use the Dependency Descriptor to check temporal layers */ + janus_av1_svc_context *av1ctx = NULL; + if(session->sim_context.substream >= 0 && session->sim_context.substream <= 2) + av1ctx = &session->av1_context[session->sim_context.substream]; + uint8_t template = 0; + if(janus_av1_svc_context_process_dd(av1ctx, packet->extensions.dd_content, + packet->extensions.dd_len, &template)) { + janus_av1_svc_template *t = g_hash_table_lookup(av1ctx->templates, GUINT_TO_POINTER(template)); + if(t) { + int temporal_layer = session->sim_context.templayer; + if(session->sim_context.templayer_target > session->sim_context.templayer) { + /* We need to upscale */ + if(t->temporal > session->sim_context.templayer && t->temporal <= session->sim_context.templayer_target) { + session->sim_context.templayer = t->temporal; + temporal_layer = session->sim_context.templayer; + session->sim_context.changed_temporal = TRUE; + } + } else if(session->sim_context.templayer_target < session->sim_context.templayer) { + /* We need to downscale */ + if(t->temporal == session->sim_context.templayer_target) { + session->sim_context.templayer = session->sim_context.templayer_target; + session->sim_context.changed_temporal = TRUE; + } + } + if(temporal_layer < t->temporal) { + JANUS_LOG(LOG_HUGE, "Dropping packet (it's temporal layer %d, but we're capping at %d)\n", + t->temporal, session->sim_context.templayer); + /* We increase the base sequence number, or there will be gaps when delivering later */ + session->context.base_seq++; + relay = FALSE; + } + } + } + } } else { /* Process this SVC packet: don't relay if it's not the layer we wanted to handle */ relay = janus_rtp_svc_context_process_rtp(&session->svc_context, @@ -973,6 +1015,9 @@ static void *janus_echotest_handler(void *data) { janus_rtp_simulcasting_prepare(s, &rid_ext_id, session->ssrc, session->rid); session->sim_context.rid_ext_id = rid_ext_id; janus_mutex_unlock(&session->rid_mutex); + janus_av1_svc_context_reset(&session->av1_context[0]); + janus_av1_svc_context_reset(&session->av1_context[1]); + janus_av1_svc_context_reset(&session->av1_context[2]); session->sim_context.substream_target = 2; /* Let's aim for the highest quality */ session->sim_context.templayer_target = 2; /* Let's aim for all temporal layers */ /* FIXME We're stopping at the first item, there may be more */ @@ -990,6 +1035,9 @@ static void *janus_echotest_handler(void *data) { janus_rtp_svc_context_reset(&session->svc_context); session->svc_context.spatial_target = 2; /* FIXME Actually depends on the scalabilityMode */ session->svc_context.temporal_target = 2; /* FIXME Actually depends on the scalabilityMode */ + janus_av1_svc_context_reset(&session->av1_context[0]); + janus_av1_svc_context_reset(&session->av1_context[1]); + janus_av1_svc_context_reset(&session->av1_context[2]); session->svc = TRUE; } /* FIXME We're stopping at the first item, there may be more */ diff --git a/src/rtp.c b/src/rtp.c index e447448711..5c2b85801d 100644 --- a/src/rtp.c +++ b/src/rtp.c @@ -1437,7 +1437,7 @@ gboolean janus_av1_svc_context_process_dd(janus_av1_svc_context *context, uint8_t end = janus_bitstream_getbit(dd, offset++); uint8_t template = janus_bitstream_getbits(dd, 6, &offset); uint16_t frame = janus_bitstream_getbits(dd, 16, &offset); - JANUS_LOG(LOG_WARN, " -- s=%u, e=%u, t=%u, f=%u\n", + JANUS_LOG(LOG_HUGE, " -- s=%u, e=%u, t=%u, f=%u\n", start, end, template, frame); if(blen > 24) { /* extended_descriptor_fields() */ @@ -1469,7 +1469,7 @@ gboolean janus_av1_svc_context_process_dd(janus_av1_svc_context *context, } t->spatial = spatial_layers; t->temporal = temporal_layers; - JANUS_LOG(LOG_WARN, " -- -- -- [%u] spatial=%u, temporal=%u\n", + JANUS_LOG(LOG_HUGE, " -- -- -- [%u] spatial=%u, temporal=%u\n", tcnt, t->spatial, t->temporal); if(nlidc == 1) { temporal_layers++; @@ -1493,14 +1493,14 @@ gboolean janus_av1_svc_context_process_dd(janus_av1_svc_context *context, } /* frame_dependency_definition() */ uint8_t tindex = (template + 64 - context->tioff) % 64; - janus_av1_svc_template *t = g_hash_table_lookup(context->templates, - GUINT_TO_POINTER(tindex)); + janus_av1_svc_template *t = context->templates ? g_hash_table_lookup(context->templates, + GUINT_TO_POINTER(tindex)) : NULL; if(t == NULL) { JANUS_LOG(LOG_WARN, "Invalid template ID '%u' (count is %u), ignoring packet...\n", tindex, context->tcnt); return FALSE; } - JANUS_LOG(LOG_WARN, " -- spatial=%u, temporal=%u (tindex %u)\n", + JANUS_LOG(LOG_HUGE, " -- spatial=%u, temporal=%u (tindex %u)\n", t->spatial, t->temporal, t->id); /* FIXME We currently don't care about the other fields */ From f282022f9acb3dcc440664722c96e0872bc1e823 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Thu, 11 May 2023 20:08:56 +0200 Subject: [PATCH 4/6] Show temporal layer buttons in EchoTest demos also for VP9 and AV1 --- html/echotest.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/html/echotest.js b/html/echotest.js index 5ccc310485..3954d8f2f5 100644 --- a/html/echotest.js +++ b/html/echotest.js @@ -203,7 +203,7 @@ $(document).ready(function() { if((substream !== null && substream !== undefined) || (temporal !== null && temporal !== undefined)) { if(!simulcastStarted) { simulcastStarted = true; - addSimulcastSvcButtons(msg["videocodec"] === "vp8" || msg["videocodec"] === "vp9"); + addSimulcastSvcButtons(msg["videocodec"] === "vp8" || msg["videocodec"] === "vp9" || msg["videocodec"] === "av1"); } // We just received notice that there's been a switch, update the buttons updateSimulcastSvcButtons(substream, temporal); From 8de2c781d23e735d4c98f5e1e93eab62111db5ac Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Fri, 12 May 2023 18:42:18 +0200 Subject: [PATCH 5/6] Made AV1/SVC finally work --- src/plugins/janus_echotest.c | 7 +-- src/plugins/janus_videoroom.c | 2 +- src/rtp.c | 111 ++++++++++++++++++++++++++++++++-- src/rtp.h | 93 ++++++++++++++-------------- 4 files changed, 159 insertions(+), 54 deletions(-) diff --git a/src/plugins/janus_echotest.c b/src/plugins/janus_echotest.c index 6c88210ff1..ab48bf5e57 100644 --- a/src/plugins/janus_echotest.c +++ b/src/plugins/janus_echotest.c @@ -639,7 +639,7 @@ void janus_echotest_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp av1ctx = &session->av1_context[session->sim_context.substream]; uint8_t template = 0; if(janus_av1_svc_context_process_dd(av1ctx, packet->extensions.dd_content, - packet->extensions.dd_len, &template)) { + packet->extensions.dd_len, &template, NULL)) { janus_av1_svc_template *t = g_hash_table_lookup(av1ctx->templates, GUINT_TO_POINTER(template)); if(t) { int temporal_layer = session->sim_context.templayer; @@ -670,7 +670,7 @@ void janus_echotest_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp } else { /* Process this SVC packet: don't relay if it's not the layer we wanted to handle */ relay = janus_rtp_svc_context_process_rtp(&session->svc_context, - buf, len, session->vcodec, NULL, &session->context); + buf, len, packet->extensions.dd_content, packet->extensions.dd_len, session->vcodec, NULL, &session->context); } if(session->sim_context.need_pli || session->svc_context.need_pli) { /* Send a PLI */ @@ -1035,9 +1035,6 @@ static void *janus_echotest_handler(void *data) { janus_rtp_svc_context_reset(&session->svc_context); session->svc_context.spatial_target = 2; /* FIXME Actually depends on the scalabilityMode */ session->svc_context.temporal_target = 2; /* FIXME Actually depends on the scalabilityMode */ - janus_av1_svc_context_reset(&session->av1_context[0]); - janus_av1_svc_context_reset(&session->av1_context[1]); - janus_av1_svc_context_reset(&session->av1_context[2]); session->svc = TRUE; } /* FIXME We're stopping at the first item, there may be more */ diff --git a/src/plugins/janus_videoroom.c b/src/plugins/janus_videoroom.c index 73d85defe8..6c459a0f11 100644 --- a/src/plugins/janus_videoroom.c +++ b/src/plugins/janus_videoroom.c @@ -12149,7 +12149,7 @@ static void janus_videoroom_relay_rtp_packet(gpointer data, gpointer user_data) /* Process this packet: don't relay if it's not the layer we wanted to handle */ janus_rtp_header rtp = *(packet->data); gboolean relay = janus_rtp_svc_context_process_rtp(&stream->svc_context, - (char *)packet->data, packet->length, ps->vcodec, &packet->svc_info, &stream->context); + (char *)packet->data, packet->length, NULL, NULL, ps->vcodec, &packet->svc_info, &stream->context); if(stream->svc_context.need_pli) { /* Send a PLI */ JANUS_LOG(LOG_VERB, "We need a PLI for the SVC context\n"); diff --git a/src/rtp.c b/src/rtp.c index 5c2b85801d..14b7008fce 100644 --- a/src/rtp.c +++ b/src/rtp.c @@ -1277,14 +1277,16 @@ void janus_rtp_svc_context_reset(janus_rtp_svc_context *context) { if(context == NULL) return; /* Reset the context values */ + janus_av1_svc_context_reset(&context->dd_context); memset(context, 0, sizeof(*context)); context->spatial = -1; context->temporal = -1; } -gboolean janus_rtp_svc_context_process_rtp(janus_rtp_svc_context *context, char *buf, int len, +gboolean janus_rtp_svc_context_process_rtp(janus_rtp_svc_context *context, + char *buf, int len, uint8_t *dd_content, int dd_len, janus_videocodec vcodec, janus_vp9_svc_info *info, janus_rtp_switching_context *sc) { - if(!context || !buf || len < 1 || vcodec != JANUS_VIDEOCODEC_VP9) + if(!context || !buf || len < 1 || (vcodec != JANUS_VIDEOCODEC_VP9 && vcodec != JANUS_VIDEOCODEC_AV1)) return FALSE; janus_rtp_header *header = (janus_rtp_header *)buf; /* Reset the flags */ @@ -1297,7 +1299,106 @@ gboolean janus_rtp_svc_context_process_rtp(janus_rtp_svc_context *context, char char *payload = janus_rtp_payload(buf, len, &plen); if(payload == NULL) return FALSE; - /* If we don't have any info parsed from the VP9 payload header, get it now */ + /* Check if we should use the Dependency Descriptor */ + if(vcodec == JANUS_VIDEOCODEC_AV1) { + /* We do, make sure the data is there */ + if(dd_content == NULL || dd_len < 1) + return FALSE; + uint8_t template = 0, ebit = 0; + if(!janus_av1_svc_context_process_dd(&context->dd_context, dd_content, dd_len, &template, &ebit)) { + /* We couldn't parse the Dependency Descriptor, relay as it is */ + return TRUE; + } + janus_av1_svc_template *t = g_hash_table_lookup(context->dd_context.templates, GUINT_TO_POINTER(template)); + if(t == NULL) { + /* We couldn't find the template, relay as it is */ + return TRUE; + } + /* Now let's check if we should let the packet through or not */ + gboolean keyframe = janus_av1_is_keyframe((const char *)payload, plen); + gboolean override_mark_bit = FALSE, has_marker_bit = header->markerbit; + int spatial_layer = context->spatial; + if(t->spatial >= 0 && t->spatial <= 2) + context->last_spatial_layer[t->spatial] = now; + if(context->spatial_target > context->spatial) { + JANUS_LOG(LOG_HUGE, "We need to upscale spatially: (%d < %d)\n", + context->spatial, context->spatial_target); + /* We need to upscale: wait for a keyframe */ + if(keyframe) { + int new_spatial_layer = context->spatial_target; + while(new_spatial_layer > context->spatial && new_spatial_layer > 0) { + if(now - context->last_spatial_layer[new_spatial_layer] >= (context->drop_trigger ? context->drop_trigger : 250000)) { + /* We haven't received packets from this layer for a while, try a lower layer */ + JANUS_LOG(LOG_HUGE, "Haven't received packets from layer %d for a while, trying %d instead...\n", + new_spatial_layer, new_spatial_layer-1); + new_spatial_layer--; + } else { + break; + } + } + if(new_spatial_layer > context->spatial) { + JANUS_LOG(LOG_HUGE, " -- Upscaling spatial layer: %d --> %d (need %d)\n", + context->spatial, new_spatial_layer, context->spatial_target); + context->spatial = new_spatial_layer; + spatial_layer = context->spatial; + context->changed_spatial = TRUE; + } + } + } else if(context->spatial_target < context->spatial) { + /* We need to scale: wait for a keyframe */ + JANUS_LOG(LOG_HUGE, "We need to downscale spatially: (%d > %d)\n", + context->spatial, context->spatial_target); + /* Check the E bit to see if this is an end-of-frame */ + if(ebit) { + JANUS_LOG(LOG_HUGE, " -- Downscaling spatial layer: %d --> %d\n", + context->spatial, context->spatial_target); + context->spatial = context->spatial_target; + context->changed_spatial = TRUE; + } + } + if(spatial_layer < t->spatial) { + /* Drop the packet: update the context to make sure sequence number is increased normally later */ + JANUS_LOG(LOG_HUGE, "Dropping packet (spatial layer %d < %d)\n", spatial_layer, t->spatial); + if(sc) + sc->base_seq++; + return FALSE; + } else if(ebit && spatial_layer == t->spatial) { + /* If we stop at layer 0, we need a marker bit now, as the one from layer 1 will not be received */ + override_mark_bit = TRUE; + } + int temporal = context->temporal; + if(context->temporal_target > context->temporal) { + /* We need to upscale */ + if(t->temporal > context->temporal && t->temporal <= context->temporal_target) { + context->temporal = t->temporal; + temporal = context->temporal; + context->changed_temporal = TRUE; + } + } else if(context->temporal_target < context->temporal) { + /* We need to downscale */ + if(t->temporal == context->temporal_target) { + context->temporal = context->temporal_target; + context->changed_temporal = TRUE; + } + } + if(temporal < t->temporal) { + JANUS_LOG(LOG_HUGE, "Dropping packet (it's temporal layer %d, but we're capping at %d)\n", + t->temporal, context->temporal); + /* We increase the base sequence number, or there will be gaps when delivering later */ + if(sc) + sc->base_seq++; + return FALSE; + } + /* If we got here, we can send the frame: this doesn't necessarily mean it's + * one of the layers the user wants, as there may be dependencies involved */ + JANUS_LOG(LOG_HUGE, "Sending packet (spatial=%d, temporal=%d)\n", + t->spatial, t->temporal); + if(override_mark_bit && !has_marker_bit) + header->markerbit = 1; + return TRUE; + } + /* If we got here, it's VP9, for which we parse the payload manually: + * if we don't have any info parsed from the VP9 payload header, get it now */ janus_vp9_svc_info svc_info = { 0 }; if(!info) { gboolean found = FALSE; @@ -1425,7 +1526,7 @@ void janus_av1_svc_context_reset(janus_av1_svc_context *context) { } gboolean janus_av1_svc_context_process_dd(janus_av1_svc_context *context, - uint8_t *dd, int dd_len, uint8_t *template_id) { + uint8_t *dd, int dd_len, uint8_t *template_id, uint8_t *ebit) { if(!context || !dd || dd_len < 3) return FALSE; @@ -1435,6 +1536,8 @@ gboolean janus_av1_svc_context_process_dd(janus_av1_svc_context *context, /* mandatory_descriptor_fields() */ uint8_t start = janus_bitstream_getbit(dd, offset++); uint8_t end = janus_bitstream_getbit(dd, offset++); + if(ebit) + *ebit = end; uint8_t template = janus_bitstream_getbits(dd, 6, &offset); uint16_t frame = janus_bitstream_getbits(dd, 16, &offset); JANUS_LOG(LOG_HUGE, " -- s=%u, e=%u, t=%u, f=%u\n", diff --git a/src/rtp.h b/src/rtp.h index 4b9a0dbba2..778089f29c 100644 --- a/src/rtp.h +++ b/src/rtp.h @@ -359,49 +359,6 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte janus_videocodec vcodec, janus_rtp_switching_context *sc, janus_mutex *rid_mutex); ///@} -/** @name Janus SVC processing methods - */ -///@{ -/*! \brief Helper struct for processing and tracking VP9-SVC streams */ -typedef struct janus_rtp_svc_context { - /*! \brief Which SVC spatial layer we should forward back */ - int spatial; - /*! \brief As above, but to handle transitions (e.g., wait for keyframe, or get this if available) */ - int spatial_target; - /*! \brief Which SVC temporal layer we should forward back */ - int temporal; - /*! \brief As above, but to handle transitions (e.g., wait for keyframe) */ - int temporal_target; - /*! \brief How much time (in us, default 250000) without receiving packets will make us drop to the substream below */ - guint32 drop_trigger; - /*! \brief When we relayed the last packet (used to detect when layers become unavailable) */ - gint64 last_spatial_layer[3]; - /*! \brief Whether the spatial layer has changed after processing a packet */ - gboolean changed_spatial; - /*! \brief Whether the temporal layer has changed after processing a packet */ - gboolean changed_temporal; - /*! \brief Whether we need to send the user a keyframe request (PLI) */ - gboolean need_pli; -} janus_rtp_svc_context; - -/*! \brief Set (or reset) the context fields to their default values - * @param[in] context The context to (re)set */ -void janus_rtp_svc_context_reset(janus_rtp_svc_context *context); - -/*! \brief Process an RTP packet, and decide whether this should be relayed or not, updating the context accordingly - * \note Calling this method resets the \c changed_spatial , \c changed_temporal and \c need_pli - * properties, and updates them according to the decisions made after processing the packet - * @param[in] context The VP9 SVC context to use - * @param[in] buf The RTP packet to process - * @param[in] len The length of the RTP packet (header, extension and payload) - * @param[in] vcodec Video codec of the RTP payload - * @param[in] info Parsed info on VP9-SVC, if any - * @param[in] sc RTP switching context to refer to, if any - * @returns TRUE if the packet should be relayed, FALSE if it should be dropped instead */ -gboolean janus_rtp_svc_context_process_rtp(janus_rtp_svc_context *context, char *buf, int len, - janus_videocodec vcodec, janus_vp9_svc_info *info, janus_rtp_switching_context *sc); -///@} - /** @name Janus AV1-SVC processing methods (still WIP) */ ///@{ @@ -444,10 +401,58 @@ void janus_av1_svc_context_reset(janus_av1_svc_context *context); * @param[in] dd Pointer to the Dependency Descriptor data * @param[in] dd_len The length of the Dependendy Descriptor data * @param[out] template_id Pointer to the ID of the template referenced in this packet + * @param[out] ebit Whether this packet is an end of frame or not * @returns TRUE if the packet is valid, FALSE if it should be dropped instead */ gboolean janus_av1_svc_context_process_dd(janus_av1_svc_context *context, - uint8_t *dd, int dd_len, uint8_t *template_id); + uint8_t *dd, int dd_len, uint8_t *template_id, uint8_t *ebit); ///@} +/** @name Janus SVC processing methods + */ +///@{ +/*! \brief Helper struct for processing and tracking VP9-SVC streams */ +typedef struct janus_rtp_svc_context { + /*! \brief Dependency Descriptor context, in case it's needed */ + struct janus_av1_svc_context dd_context; + /*! \brief Which SVC spatial layer we should forward back */ + int spatial; + /*! \brief As above, but to handle transitions (e.g., wait for keyframe, or get this if available) */ + int spatial_target; + /*! \brief Which SVC temporal layer we should forward back */ + int temporal; + /*! \brief As above, but to handle transitions (e.g., wait for keyframe) */ + int temporal_target; + /*! \brief How much time (in us, default 250000) without receiving packets will make us drop to the substream below */ + guint32 drop_trigger; + /*! \brief When we relayed the last packet (used to detect when layers become unavailable) */ + gint64 last_spatial_layer[3]; + /*! \brief Whether the spatial layer has changed after processing a packet */ + gboolean changed_spatial; + /*! \brief Whether the temporal layer has changed after processing a packet */ + gboolean changed_temporal; + /*! \brief Whether we need to send the user a keyframe request (PLI) */ + gboolean need_pli; +} janus_rtp_svc_context; + +/*! \brief Set (or reset) the context fields to their default values + * @param[in] context The context to (re)set */ +void janus_rtp_svc_context_reset(janus_rtp_svc_context *context); + +/*! \brief Process an RTP packet, and decide whether this should be relayed or not, updating the context accordingly + * \note Calling this method resets the \c changed_spatial , \c changed_temporal and \c need_pli + * properties, and updates them according to the decisions made after processing the packet + * @param[in] context The VP9 SVC context to use + * @param[in] buf The RTP packet to process + * @param[in] len The length of the RTP packet (header, extension and payload) + * @param[in] dd_content The Dependency Descriptor RTP extension data, if available + * @param[in] dd_len Length of the Dependency Descriptor data, if available + * @param[in] vcodec Video codec of the RTP payload + * @param[in] info Parsed info on VP9-SVC, if any + * @param[in] sc RTP switching context to refer to, if any + * @returns TRUE if the packet should be relayed, FALSE if it should be dropped instead */ +gboolean janus_rtp_svc_context_process_rtp(janus_rtp_svc_context *context, + char *buf, int len, uint8_t *dd_content, int dd_len, + janus_videocodec vcodec, janus_vp9_svc_info *info, janus_rtp_switching_context *sc); +///@} #endif From a001ded49fb495e5f4d3f7e90b7cbb76cbea814f Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Tue, 16 May 2023 12:38:54 +0200 Subject: [PATCH 6/6] Streamlined simulcast usage when involving DD, and added support to other plugins --- src/plugins/duktape/janus-sdp.js | 2 + src/plugins/janus_duktape.c | 13 +--- src/plugins/janus_echotest.c | 48 +-------------- src/plugins/janus_lua.c | 13 +--- src/plugins/janus_recordplay.c | 7 +-- src/plugins/janus_streaming.c | 3 +- src/plugins/janus_videocall.c | 9 +-- src/plugins/janus_videoroom.c | 67 +++++++++++--------- src/plugins/lua/janus-sdp.lua | 2 + src/rtp.c | 47 +++++++++++++- src/rtp.h | 102 ++++++++++++++++--------------- src/rtpfwd.c | 2 +- 12 files changed, 156 insertions(+), 159 deletions(-) diff --git a/src/plugins/duktape/janus-sdp.js b/src/plugins/duktape/janus-sdp.js index 9e833ee80f..c6985e414a 100644 --- a/src/plugins/duktape/janus-sdp.js +++ b/src/plugins/duktape/janus-sdp.js @@ -399,6 +399,8 @@ JANUSSDP.generateAnswer = function(offer, options) { answer.push({ type: "a", name: a.name, value: value }); } else if(options.enableAudioLevel !== false && a.value.indexOf("urn:ietf:params:rtp-hdrext:ssrc-audio-level") !== -1) { answer.push({ type: "a", name: a.name, value: value }); + } else if(options.enableAudioLevel !== false && a.value.indexOf("dependency-descriptor-rtp-header-extension") !== -1) { + answer.push({ type: "a", name: a.name, value: value }); } } } else { diff --git a/src/plugins/janus_duktape.c b/src/plugins/janus_duktape.c index ce72dd47b0..39a2d2c815 100644 --- a/src/plugins/janus_duktape.c +++ b/src/plugins/janus_duktape.c @@ -619,10 +619,6 @@ static duk_ret_t janus_duktape_method_pushevent(duk_context *ctx) { janus_sdp_find_first_codec(parsed_sdp, JANUS_SDP_VIDEO, -1, &vcodec); if(vcodec) session->vcodec = janus_videocodec_from_name(vcodec); - if(session->vcodec != JANUS_VIDEOCODEC_VP8 && session->vcodec != JANUS_VIDEOCODEC_H264) { - /* VP8 r H.264 were not negotiated, if simulcasting was enabled then disable it here */ - janus_rtp_simulcasting_cleanup(&session->rid_extmap_id, session->ssrc, session->rid, &session->rid_mutex); - } } janus_sdp_destroy(parsed_sdp); /* Send asynchronously */ @@ -2240,10 +2236,6 @@ struct janus_plugin_result *janus_duktape_handle_message(janus_plugin_session *h janus_sdp_find_first_codec(parsed_sdp, JANUS_SDP_VIDEO, -1, &vcodec); if(vcodec) session->vcodec = janus_videocodec_from_name(vcodec); - if(session->vcodec != JANUS_VIDEOCODEC_VP8 && session->vcodec != JANUS_VIDEOCODEC_H264) { - /* VP8 r H.264 were not negotiated, if simulcasting was enabled then disable it here */ - janus_rtp_simulcasting_cleanup(&session->rid_extmap_id, session->ssrc, session->rid, &session->rid_mutex); - } janus_sdp_destroy(parsed_sdp); } if(json_is_true(json_object_get(jsep, "e2ee"))) @@ -2459,7 +2451,7 @@ void janus_duktape_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp * } else { /* We're simulcasting, save the best video quality */ gboolean save = janus_rtp_simulcasting_context_process_rtp(&session->rec_simctx, - buf, len, session->ssrc, session->rid, session->vcodec, &session->rec_ctx, &session->rid_mutex); + buf, len, NULL, 0, session->ssrc, session->rid, session->vcodec, &session->rec_ctx, &session->rid_mutex); if(save) { uint32_t seq_number = ntohs(rtp->seq_number); uint32_t timestamp = ntohl(rtp->timestamp); @@ -2781,7 +2773,8 @@ static void janus_duktape_relay_rtp_packet(gpointer data, gpointer user_data) { return; /* Process this packet: don't relay if it's not the SSRC/layer we wanted to handle */ gboolean relay = janus_rtp_simulcasting_context_process_rtp(&session->sim_context, - (char *)packet->data, packet->length, packet->ssrc, NULL, sender->vcodec, &session->vrtpctx, NULL); + (char *)packet->data, packet->length, packet->extensions.dd_content, packet->extensions.dd_len, + packet->ssrc, NULL, sender->vcodec, &session->vrtpctx, NULL); if(session->sim_context.need_pli && sender->handle) { /* Send a PLI */ JANUS_LOG(LOG_VERB, "We need a PLI for the simulcast context\n"); diff --git a/src/plugins/janus_echotest.c b/src/plugins/janus_echotest.c index ab48bf5e57..4ef6907014 100644 --- a/src/plugins/janus_echotest.c +++ b/src/plugins/janus_echotest.c @@ -253,7 +253,6 @@ typedef struct janus_echotest_session { janus_vp8_simulcast_context vp8_context; gboolean svc; janus_rtp_svc_context svc_context; - janus_av1_svc_context av1_context[3]; /* Dependency Descriptors can be used in VP9/AV1 simulcast too */ janus_recorder *arc; /* The Janus recorder instance for this user's audio, if enabled */ janus_recorder *vrc; /* The Janus recorder instance for this user's video, if enabled */ janus_recorder *drc; /* The Janus recorder instance for this user's data, if enabled */ @@ -281,9 +280,6 @@ static void janus_echotest_session_free(const janus_refcount *session_ref) { g_free(session->vfmtp); janus_mutex_destroy(&session->rid_mutex); janus_mutex_destroy(&session->rec_mutex); - janus_av1_svc_context_reset(&session->av1_context[0]); - janus_av1_svc_context_reset(&session->av1_context[1]); - janus_av1_svc_context_reset(&session->av1_context[2]); janus_rtp_simulcasting_cleanup(NULL, NULL, session->rid, NULL); g_free(session); } @@ -452,9 +448,6 @@ void janus_echotest_create_session(janus_plugin_session *handle, int *error) { janus_rtp_simulcasting_context_reset(&session->sim_context); janus_vp8_simulcast_context_reset(&session->vp8_context); janus_rtp_svc_context_reset(&session->svc_context); - janus_av1_svc_context_reset(&session->av1_context[0]); - janus_av1_svc_context_reset(&session->av1_context[1]); - janus_av1_svc_context_reset(&session->av1_context[2]); janus_mutex_init(&session->rid_mutex); session->min_delay = -1; session->max_delay = -1; @@ -631,42 +624,8 @@ void janus_echotest_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp if(simulcast) { /* Process this simulcast packet: don't relay if it's not the SSRC/layer we wanted to handle */ relay = janus_rtp_simulcasting_context_process_rtp(&session->sim_context, - buf, len, session->ssrc, session->rid, session->vcodec, &session->context, &session->rid_mutex); - if(relay && session->vcodec == JANUS_VIDEOCODEC_AV1 && packet->extensions.dd_len > 0) { - /* Use the Dependency Descriptor to check temporal layers */ - janus_av1_svc_context *av1ctx = NULL; - if(session->sim_context.substream >= 0 && session->sim_context.substream <= 2) - av1ctx = &session->av1_context[session->sim_context.substream]; - uint8_t template = 0; - if(janus_av1_svc_context_process_dd(av1ctx, packet->extensions.dd_content, - packet->extensions.dd_len, &template, NULL)) { - janus_av1_svc_template *t = g_hash_table_lookup(av1ctx->templates, GUINT_TO_POINTER(template)); - if(t) { - int temporal_layer = session->sim_context.templayer; - if(session->sim_context.templayer_target > session->sim_context.templayer) { - /* We need to upscale */ - if(t->temporal > session->sim_context.templayer && t->temporal <= session->sim_context.templayer_target) { - session->sim_context.templayer = t->temporal; - temporal_layer = session->sim_context.templayer; - session->sim_context.changed_temporal = TRUE; - } - } else if(session->sim_context.templayer_target < session->sim_context.templayer) { - /* We need to downscale */ - if(t->temporal == session->sim_context.templayer_target) { - session->sim_context.templayer = session->sim_context.templayer_target; - session->sim_context.changed_temporal = TRUE; - } - } - if(temporal_layer < t->temporal) { - JANUS_LOG(LOG_HUGE, "Dropping packet (it's temporal layer %d, but we're capping at %d)\n", - t->temporal, session->sim_context.templayer); - /* We increase the base sequence number, or there will be gaps when delivering later */ - session->context.base_seq++; - relay = FALSE; - } - } - } - } + buf, len, packet->extensions.dd_content, packet->extensions.dd_len, + session->ssrc, session->rid, session->vcodec, &session->context, &session->rid_mutex); } else { /* Process this SVC packet: don't relay if it's not the layer we wanted to handle */ relay = janus_rtp_svc_context_process_rtp(&session->svc_context, @@ -1015,9 +974,6 @@ static void *janus_echotest_handler(void *data) { janus_rtp_simulcasting_prepare(s, &rid_ext_id, session->ssrc, session->rid); session->sim_context.rid_ext_id = rid_ext_id; janus_mutex_unlock(&session->rid_mutex); - janus_av1_svc_context_reset(&session->av1_context[0]); - janus_av1_svc_context_reset(&session->av1_context[1]); - janus_av1_svc_context_reset(&session->av1_context[2]); session->sim_context.substream_target = 2; /* Let's aim for the highest quality */ session->sim_context.templayer_target = 2; /* Let's aim for all temporal layers */ /* FIXME We're stopping at the first item, there may be more */ diff --git a/src/plugins/janus_lua.c b/src/plugins/janus_lua.c index bbe503a900..e8f47a1068 100644 --- a/src/plugins/janus_lua.c +++ b/src/plugins/janus_lua.c @@ -547,10 +547,6 @@ static int janus_lua_method_pushevent(lua_State *s) { janus_sdp_find_first_codec(parsed_sdp, JANUS_SDP_VIDEO, -1, &vcodec); if(vcodec) session->vcodec = janus_videocodec_from_name(vcodec); - if(session->vcodec != JANUS_VIDEOCODEC_VP8 && session->vcodec != JANUS_VIDEOCODEC_H264) { - /* VP8 r H.264 were not negotiated, if simulcasting was enabled then disable it here */ - janus_rtp_simulcasting_cleanup(&session->rid_extmap_id, session->ssrc, session->rid, &session->rid_mutex); - } } janus_sdp_destroy(parsed_sdp); /* Send asynchronously */ @@ -1936,10 +1932,6 @@ struct janus_plugin_result *janus_lua_handle_message(janus_plugin_session *handl janus_sdp_find_first_codec(parsed_sdp, JANUS_SDP_VIDEO, -1, &vcodec); if(vcodec) session->vcodec = janus_videocodec_from_name(vcodec); - if(session->vcodec != JANUS_VIDEOCODEC_VP8 && session->vcodec != JANUS_VIDEOCODEC_H264) { - /* VP8 r H.264 were not negotiated, if simulcasting was enabled then disable it here */ - janus_rtp_simulcasting_cleanup(&session->rid_extmap_id, session->ssrc, session->rid, &session->rid_mutex); - } janus_sdp_destroy(parsed_sdp); } if(json_is_true(json_object_get(jsep, "e2ee"))) @@ -2127,7 +2119,7 @@ void janus_lua_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *rtp_ } else { /* We're simulcasting, save the best video quality */ gboolean save = janus_rtp_simulcasting_context_process_rtp(&session->rec_simctx, - buf, len, session->ssrc, session->rid, session->vcodec, &session->rec_ctx, &session->rid_mutex); + buf, len, NULL, 0, session->ssrc, session->rid, session->vcodec, &session->rec_ctx, &session->rid_mutex); if(save) { uint32_t seq_number = ntohs(rtp->seq_number); uint32_t timestamp = ntohl(rtp->timestamp); @@ -2421,7 +2413,8 @@ static void janus_lua_relay_rtp_packet(gpointer data, gpointer user_data) { return; /* Process this packet: don't relay if it's not the SSRC/layer we wanted to handle */ gboolean relay = janus_rtp_simulcasting_context_process_rtp(&session->sim_context, - (char *)packet->data, packet->length, packet->ssrc, NULL, sender->vcodec, &session->vrtpctx, NULL); + (char *)packet->data, packet->length, packet->extensions.dd_content, packet->extensions.dd_len, + packet->ssrc, NULL, sender->vcodec, &session->vrtpctx, NULL); if(session->sim_context.need_pli && sender->handle) { /* Send a PLI */ JANUS_LOG(LOG_VERB, "We need a PLI for the simulcast context\n"); diff --git a/src/plugins/janus_recordplay.c b/src/plugins/janus_recordplay.c index d55bfe0487..0812a0bdd9 100644 --- a/src/plugins/janus_recordplay.c +++ b/src/plugins/janus_recordplay.c @@ -1304,7 +1304,8 @@ void janus_recordplay_incoming_rtp(janus_plugin_session *handle, janus_plugin_rt uint32_t ssrc = ntohl(header->ssrc); /* Process this packet: don't save if it's not the SSRC/layer we wanted to handle */ gboolean save = janus_rtp_simulcasting_context_process_rtp(&session->sim_context, - buf, len, session->ssrc, session->rid, session->recording->vcodec, &session->context, &session->rid_mutex); + buf, len, NULL, 0, session->ssrc, session->rid, session->recording->vcodec, + &session->context, &session->rid_mutex); if(session->sim_context.need_pli) { /* Send a PLI */ JANUS_LOG(LOG_VERB, "We need a PLI for the simulcast context\n"); @@ -1879,10 +1880,6 @@ static void *janus_recordplay_handler(void *data) { janus_mutex_unlock(&session->rid_mutex); session->sim_context.substream_target = 2; /* Let's aim for the highest quality */ session->sim_context.templayer_target = 2; /* Let's aim for all temporal layers */ - if(rec->vcodec != JANUS_VIDEOCODEC_VP8 && rec->vcodec != JANUS_VIDEOCODEC_H264) { - /* VP8 r H.264 were not negotiated, if simulcasting was enabled then disable it here */ - janus_rtp_simulcasting_cleanup(NULL, session->ssrc, session->rid, &session->rid_mutex); - } /* FIXME We're stopping at the first item, there may be more */ break; } diff --git a/src/plugins/janus_streaming.c b/src/plugins/janus_streaming.c index 461e7df6be..670677b21e 100644 --- a/src/plugins/janus_streaming.c +++ b/src/plugins/janus_streaming.c @@ -10030,7 +10030,8 @@ static void janus_streaming_relay_rtp_packet(gpointer data, gpointer user_data) return; /* Process this packet: don't relay if it's not the SSRC/layer we wanted to handle */ gboolean relay = janus_rtp_simulcasting_context_process_rtp(&s->sim_context, - (char *)packet->data, packet->length, packet->ssrc, NULL, packet->codec, &s->context, NULL); + (char *)packet->data, packet->length, NULL, 0, + packet->ssrc, NULL, packet->codec, &s->context, NULL); if(!relay) { /* Did a lot of time pass before we could relay a packet? */ gint64 now = janus_get_monotonic_time(); diff --git a/src/plugins/janus_videocall.c b/src/plugins/janus_videocall.c index c048e0d185..4fa5aad6e1 100644 --- a/src/plugins/janus_videocall.c +++ b/src/plugins/janus_videocall.c @@ -778,7 +778,8 @@ void janus_videocall_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp /* Process this packet: don't relay if it's not the SSRC/layer we wanted to handle * The caveat is that the targets in OUR simulcast context are the PEER's targets */ gboolean relay = janus_rtp_simulcasting_context_process_rtp(&peer->sim_context, - buf, len, session->ssrc, session->rid, session->vcodec, &peer->context, &session->rid_mutex); + buf, len, packet->extensions.dd_content, packet->extensions.dd_len, + session->ssrc, session->rid, session->vcodec, &peer->context, &session->rid_mutex); /* Do we need to drop this? */ if(!relay) return; @@ -1374,7 +1375,7 @@ static void *janus_videocall_handler(void *data) { session->has_data = (strstr(msg_sdp, "DTLS/SCTP") != NULL); /* Check if this user will simulcast */ json_t *msg_simulcast = json_object_get(msg->jsep, "simulcast"); - if(msg_simulcast && janus_get_codec_pt(msg_sdp, "vp8") > 0) { + if(msg_simulcast) { JANUS_LOG(LOG_VERB, "VideoCall callee (%s) cannot do simulcast.\n", session->username); } else { janus_rtp_simulcasting_cleanup(NULL, session->ssrc, session->rid, &session->rid_mutex); @@ -1533,7 +1534,7 @@ static void *janus_videocall_handler(void *data) { session->sim_context.templayer_target = json_integer_value(temporal); JANUS_LOG(LOG_VERB, "Setting video temporal layer to let through (simulcast): %d (was %d)\n", session->sim_context.templayer_target, session->sim_context.templayer); - if(session->vcodec == JANUS_VIDEOCODEC_VP8 && session->sim_context.templayer_target == session->sim_context.templayer) { + if(session->sim_context.templayer_target == session->sim_context.templayer) { /* No need to do anything, we're already getting the right temporal, so notify the user */ json_t *event = json_object(); json_object_set_new(event, "videocall", json_string("event")); @@ -1569,7 +1570,7 @@ static void *janus_videocall_handler(void *data) { session->has_data = (strstr(msg_sdp, "DTLS/SCTP") != NULL); /* Check if this user will simulcast */ json_t *msg_simulcast = json_object_get(msg->jsep, "simulcast"); - if(msg_simulcast && janus_get_codec_pt(msg_sdp, "vp8") > 0) { + if(msg_simulcast) { JANUS_LOG(LOG_VERB, "VideoCall callee (%s) cannot do simulcast.\n", session->username); } else { janus_rtp_simulcasting_cleanup(NULL, session->ssrc, session->rid, &session->rid_mutex); diff --git a/src/plugins/janus_videoroom.c b/src/plugins/janus_videoroom.c index 6c459a0f11..01ea6a74cd 100644 --- a/src/plugins/janus_videoroom.c +++ b/src/plugins/janus_videoroom.c @@ -530,8 +530,8 @@ room-: { "codec" : "", "description" : "", "moderated" : , - "simulcast" : "", - "svc" : "", + "simulcast" : "", + "svc" : "", "talking" : , }, // Other streams, if any @@ -675,8 +675,8 @@ room-: { "codec" : "", "description" : "", "moderated" : , - "simulcast" : "", - "svc" : "", + "simulcast" : "", + "svc" : "", "talking" : , }, // Other streams, if any @@ -1365,8 +1365,8 @@ room-: { "substream" : , "temporal" : , "fallback" : , - "spatial_layer" : , - "temporal_layer" : , + "spatial_layer" : , + "temporal_layer" : , "audio_level_average" : "", "audio_active_packets" : "", "min_delay" : , @@ -1388,7 +1388,7 @@ room-: { * that for them to work you'll have to specify the \c mid as well, as the same * subscription may be receiving simulcast stream from multiple publishers. * The \c spatial_layer and \c temporal_layer have exactly the same meaning, - * but within the context of VP9-SVC publishers, and will have no effect + * but within the context of SVC publishers, and will have no effect * on subscriptions associated to regular publishers. * * As anticipated, \c configure is also the request you use when you want @@ -1802,11 +1802,11 @@ static struct janus_json_parameter configure_stream_parameters[] = { /* For talk detection */ {"audio_level_averge", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"audio_active_packets", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, - /* For VP8 (or H.264) simulcast */ + /* For simulcast */ {"substream", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"temporal", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"fallback", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, - /* For VP9 SVC */ + /* For SVC */ {"spatial_layer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"temporal_layer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, /* For the playout-delay RTP extension, if negotiated */ @@ -1835,11 +1835,11 @@ static struct janus_json_parameter subscriber_parameters[] = { {"offer_audio", JANUS_JSON_BOOL, 0}, {"offer_video", JANUS_JSON_BOOL, 0}, {"offer_data", JANUS_JSON_BOOL, 0}, - /* For VP8 (or H.264) simulcast */ + /* For simulcast */ {"substream", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"temporal", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"fallback", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, - /* For VP9 SVC */ + /* For SVC */ {"spatial_layer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"temporal_layer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, }; @@ -1847,10 +1847,10 @@ static struct janus_json_parameter subscriber_stream_parameters[] = { {"mid", JANUS_JSON_STRING, 0}, {"crossrefid", JANUS_JSON_STRING, 0}, {"send", JANUS_JSON_BOOL, 0}, - /* For VP8 (or H.264) simulcast */ + /* For simulcast */ {"substream", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"temporal", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, - /* For VP9 SVC */ + /* For SVC */ {"spatial_layer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"temporal_layer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, /* For the playout-delay RTP extension, if negotiated */ @@ -1876,10 +1876,10 @@ static struct janus_json_parameter switch_update_parameters[] = { //~ {"feed", JANUS_JSON_INTEGER, JANUS_JSON_PARAM_REQUIRED | JANUS_JSON_PARAM_POSITIVE}, {"mid", JANUS_JSON_STRING, JANUS_JSON_PARAM_REQUIRED}, {"sub_mid", JANUS_JSON_STRING, JANUS_JSON_PARAM_REQUIRED}, - /* For VP8 (or H.264) simulcast */ + /* For simulcast */ {"substream", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"temporal", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, - /* For VP9 SVC */ + /* For SVC */ {"spatial_layer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"temporal_layer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE} }; @@ -2129,8 +2129,8 @@ typedef struct janus_videoroom_publisher_stream { gboolean opusfec; /* Whether this stream is sending inband Opus FEC */ gboolean opusdtx; /* Whether this publisher is using Opus DTX (Discontinuous Transmission) */ gboolean opusstereo; /* Whether this publisher is doing stereo Opus */ - gboolean simulcast, svc; /* Whether this stream uses simulcast or VP9 SVC */ - uint32_t vssrc[3]; /* Only needed in case VP8 (or H.264) simulcasting is involved */ + gboolean simulcast, svc; /* Whether this stream uses simulcast or SVC */ + uint32_t vssrc[3]; /* Only needed in case simulcasting is involved */ char *rid[3]; /* Only needed if simulcasting is rid-based */ int rid_extmap_id; /* rid extmap ID */ janus_mutex rid_mutex; /* Mutex to protect access to the rid array and the extmap ID */ @@ -2252,7 +2252,7 @@ typedef struct janus_videoroom_rtp_relay_packet { janus_plugin_rtp_extensions extensions; /* Whether simulcast is involved */ gboolean simulcast; - /* The following are only relevant if we're doing VP9 SVC*/ + /* The following are only relevant if we're doing SVC*/ gboolean svc; janus_vp9_svc_info svc_info; /* The following is only relevant for datachannels */ @@ -4077,7 +4077,7 @@ json_t *janus_videoroom_query_session(janus_plugin_session *handle) { if(ps->simulcast) json_object_set_new(m, "simulcast", json_true()); if(ps->svc) - json_object_set_new(m, "vp9-svc", json_true()); + json_object_set_new(m, "svc", json_true()); if(ps->rc && ps->rc->filename) json_object_set_new(m, "recording", json_string(ps->rc->filename)); if(ps->audio_level_extmap_id > 0) { @@ -7998,7 +7998,8 @@ static void janus_videoroom_incoming_rtp_internal(janus_videoroom_session *sessi } else { /* We're simulcasting, save the best video quality */ gboolean save = janus_rtp_simulcasting_context_process_rtp(&ps->rec_simctx, - buf, len, ps->vssrc, ps->rid, ps->vcodec, &ps->rec_ctx, &ps->rid_mutex); + buf, len, pkt->extensions.dd_content, pkt->extensions.dd_len, + ps->vssrc, ps->rid, ps->vcodec, &ps->rec_ctx, &ps->rid_mutex); if(save) { uint32_t seq_number = ntohs(rtp->seq_number); uint32_t timestamp = ntohl(rtp->timestamp); @@ -8030,10 +8031,14 @@ static void janus_videoroom_incoming_rtp_internal(janus_videoroom_session *sessi janus_videoroom_publisher_dereference_nodebug(participant); return; } - gboolean found = FALSE; - memset(&packet.svc_info, 0, sizeof(packet.svc_info)); - if(janus_vp9_parse_svc(payload, plen, &found, &packet.svc_info) == 0) { - packet.svc = found; + if(ps->vcodec == JANUS_VIDEOCODEC_VP9) { + gboolean found = FALSE; + memset(&packet.svc_info, 0, sizeof(packet.svc_info)); + if(janus_vp9_parse_svc(payload, plen, &found, &packet.svc_info) == 0) { + packet.svc = found; + } + } else if(ps->vcodec == JANUS_VIDEOCODEC_AV1) { + packet.svc = (pkt->extensions.dd_len > 0); } } if(video && ps->simulcast) @@ -10884,7 +10889,7 @@ static void *janus_videoroom_handler(void *data) { janus_videoroom_reqpli(ps, "Simulcasting substream change"); } } - if(ps->vcodec == JANUS_VIDEOCODEC_VP8 && ps->simulcast && sc_temporal) { + if(ps->simulcast && sc_temporal) { stream->sim_context.templayer_target = json_integer_value(sc_temporal); JANUS_LOG(LOG_VERB, "Setting video temporal layer to let through (simulcast): %d (was %d)\n", stream->sim_context.templayer_target, stream->sim_context.templayer); @@ -11795,8 +11800,7 @@ static void *janus_videoroom_handler(void *data) { } } /* Check if simulcast or SVC is in place */ - if(msg_simulcast != NULL && json_array_size(msg_simulcast) > 0 && - (ps->vcodec == JANUS_VIDEOCODEC_VP8 || ps->vcodec == JANUS_VIDEOCODEC_H264)) { + if(msg_simulcast != NULL && json_array_size(msg_simulcast) > 0) { size_t i = 0; for(i=0; irid_mutex); } } else if(msg_svc != NULL && json_array_size(msg_svc) > 0 && - ps->vcodec == JANUS_VIDEOCODEC_VP9) { + (ps->vcodec == JANUS_VIDEOCODEC_VP9 || ps->vcodec == JANUS_VIDEOCODEC_AV1)) { size_t i = 0; for(i=0; ivideoorient_ext ? JANUS_RTP_EXTMAP_VIDEO_ORIENTATION : NULL, JANUS_SDP_OA_ACCEPT_EXTMAP, videoroom->playoutdelay_ext ? JANUS_RTP_EXTMAP_PLAYOUT_DELAY : NULL, JANUS_SDP_OA_ACCEPT_EXTMAP, videoroom->transport_wide_cc_ext ? JANUS_RTP_EXTMAP_TRANSPORT_WIDE_CC : NULL, @@ -12149,7 +12154,8 @@ static void janus_videoroom_relay_rtp_packet(gpointer data, gpointer user_data) /* Process this packet: don't relay if it's not the layer we wanted to handle */ janus_rtp_header rtp = *(packet->data); gboolean relay = janus_rtp_svc_context_process_rtp(&stream->svc_context, - (char *)packet->data, packet->length, NULL, NULL, ps->vcodec, &packet->svc_info, &stream->context); + (char *)packet->data, packet->length, packet->extensions.dd_content, packet->extensions.dd_len, + ps->vcodec, &packet->svc_info, &stream->context); if(stream->svc_context.need_pli) { /* Send a PLI */ JANUS_LOG(LOG_VERB, "We need a PLI for the SVC context\n"); @@ -12201,7 +12207,8 @@ static void janus_videoroom_relay_rtp_packet(gpointer data, gpointer user_data) return; /* Process this packet: don't relay if it's not the SSRC/layer we wanted to handle */ gboolean relay = janus_rtp_simulcasting_context_process_rtp(&stream->sim_context, - (char *)packet->data, packet->length, packet->ssrc, NULL, ps->vcodec, &stream->context, &ps->rid_mutex); + (char *)packet->data, packet->length, packet->extensions.dd_content, packet->extensions.dd_len, + packet->ssrc, NULL, ps->vcodec, &stream->context, &ps->rid_mutex); if(!relay) { /* Did a lot of time pass before we could relay a packet? */ gint64 now = janus_get_monotonic_time(); diff --git a/src/plugins/lua/janus-sdp.lua b/src/plugins/lua/janus-sdp.lua index 85f062f946..03666ed0b6 100644 --- a/src/plugins/lua/janus-sdp.lua +++ b/src/plugins/lua/janus-sdp.lua @@ -446,6 +446,8 @@ function JANUSSDP.generateAnswer(offer, options) answer[#answer+1] = { type = "a", name = a.name, value = value } elseif options.enableAudioLevel ~= false and a.value:find("urn:ietf:params:rtp-hdrext:ssrc-audio-level", 1, true) then answer[#answer+1] = a + elseif options.enableAudioLevel ~= false and a.value:find("dependency-descriptor-rtp-header-extension", 1, true) then + answer[#answer+1] = a end end else diff --git a/src/rtp.c b/src/rtp.c index 14b7008fce..086ff19054 100644 --- a/src/rtp.c +++ b/src/rtp.c @@ -1023,6 +1023,9 @@ void janus_rtp_simulcasting_context_reset(janus_rtp_simulcasting_context *contex if(context == NULL) return; /* Reset the context values */ + janus_av1_svc_context_reset(&context->av1_context[0]); + janus_av1_svc_context_reset(&context->av1_context[1]); + janus_av1_svc_context_reset(&context->av1_context[2]); memset(context, 0, sizeof(*context)); context->rid_ext_id = -1; context->substream = -1; @@ -1081,7 +1084,7 @@ void janus_rtp_simulcasting_cleanup(int *rid_ext_id, uint32_t *ssrcs, char **rid } gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_context *context, - char *buf, int len, uint32_t *ssrcs, char **rids, + char *buf, int len, uint8_t *dd_content, int dd_len, uint32_t *ssrcs, char **rids, janus_videocodec vcodec, janus_rtp_switching_context *sc, janus_mutex *rid_mutex) { if(!context || !buf || len < 1) return FALSE; @@ -1267,6 +1270,42 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte return FALSE; } } + } else if(vcodec == JANUS_VIDEOCODEC_AV1 && dd_content != NULL && dd_len > 0) { + /* Use the Dependency Descriptor to check temporal layers */ + janus_av1_svc_context *av1ctx = NULL; + if(context->substream >= 0 && context->substream <= 2) + av1ctx = &context->av1_context[context->substream]; + if(av1ctx != NULL) { + uint8_t template = 0; + if(janus_av1_svc_context_process_dd(av1ctx, dd_content, dd_len, &template, NULL)) { + janus_av1_svc_template *t = g_hash_table_lookup(av1ctx->templates, GUINT_TO_POINTER(template)); + if(t) { + int temporal_layer = context->templayer; + if(context->templayer_target > context->templayer) { + /* We need to upscale */ + if(t->temporal > context->templayer && t->temporal <= context->templayer_target) { + context->templayer = t->temporal; + temporal_layer = context->templayer; + context->changed_temporal = TRUE; + } + } else if(context->templayer_target < context->templayer) { + /* We need to downscale */ + if(t->temporal == context->templayer_target) { + context->templayer = context->templayer_target; + context->changed_temporal = TRUE; + } + } + if(temporal_layer < t->temporal) { + JANUS_LOG(LOG_HUGE, "Dropping packet (it's temporal layer %d, but we're capping at %d)\n", + t->temporal, context->templayer); + /* We increase the base sequence number, or there will be gaps when delivering later */ + if(sc) + sc->base_seq++; + return FALSE; + } + } + } + } } /* If we got here, the packet can be relayed */ return TRUE; @@ -1302,8 +1341,10 @@ gboolean janus_rtp_svc_context_process_rtp(janus_rtp_svc_context *context, /* Check if we should use the Dependency Descriptor */ if(vcodec == JANUS_VIDEOCODEC_AV1) { /* We do, make sure the data is there */ - if(dd_content == NULL || dd_len < 1) - return FALSE; + if(dd_content == NULL || dd_len < 1) { + /* No Dependency Descriptor, relay as it is */ + return TRUE; + } uint8_t template = 0, ebit = 0; if(!janus_av1_svc_context_process_dd(&context->dd_context, dd_content, dd_len, &template, &ebit)) { /* We couldn't parse the Dependency Descriptor, relay as it is */ diff --git a/src/rtp.h b/src/rtp.h index 778089f29c..cebb8b3c86 100644 --- a/src/rtp.h +++ b/src/rtp.h @@ -295,6 +295,54 @@ int janus_rtp_skew_compensate_audio(janus_rtp_header *header, janus_rtp_switchin int janus_rtp_skew_compensate_video(janus_rtp_header *header, janus_rtp_switching_context *context, gint64 now); +/** @name Janus AV1-SVC processing methods + */ +///@{ +/*! \brief Helper struct for processing and tracking AV1-SVC streams */ +typedef struct janus_av1_svc_context { + /*! \brief Number of templates advertised via Dependency Descriptor */ + uint8_t tcnt; + /*! \brief Template ID offset, as advertised via Dependency Descriptor */ + uint8_t tioff; + /*! \brief Map of templates advertised via Dependency Descriptor, indexed by ID */ + GHashTable *templates; + /*! \brief How many spatial and temporal layers are available */ + int spatial_layers, temporal_layers; + /*! \brief Whether this context changed since the last update */ + gboolean updated; +} janus_av1_svc_context; + +/*! \brief Helper struct to track SVC templates + * \note This is very incomplete, since we only track the spatial and + * temporal layer associated with a specific template ID for now */ +typedef struct janus_av1_svc_template { + /*! \brief Template ID */ + uint8_t id; + /*! \brief Spatial layer associated to this template */ + int spatial; + /*! \brief Temporal layer associated to this template */ + int temporal; +} janus_av1_svc_template; + +/*! \brief Set (or reset) the context fields to their default values + * @param[in] context The context to (re)set */ +void janus_av1_svc_context_reset(janus_av1_svc_context *context); + +/*! \brief Process a Dependency Descriptor payload, updating the SVC context accordingly + * \note At the moment, this code is quite naive, as it mostly looks at the target + * spatial/temporal layers, and the one written in the Dependency Descriptor data. + * In the future, this should become more sophisticated, and use additional + * information like dependency chains and stuff like that + * @param[in] context The av1svc context to use + * @param[in] dd Pointer to the Dependency Descriptor data + * @param[in] dd_len The length of the Dependendy Descriptor data + * @param[out] template_id Pointer to the ID of the template referenced in this packet + * @param[out] ebit Whether this packet is an end of frame or not + * @returns TRUE if the packet is valid, FALSE if it should be dropped instead */ +gboolean janus_av1_svc_context_process_dd(janus_av1_svc_context *context, + uint8_t *dd, int dd_len, uint8_t *template_id, uint8_t *ebit); +///@} + /** @name Janus simulcast processing methods */ ///@{ @@ -302,6 +350,8 @@ int janus_rtp_skew_compensate_video(janus_rtp_header *header, janus_rtp_switchin typedef struct janus_rtp_simulcasting_context { /*! \brief RTP Stream extension ID, if any */ gint rid_ext_id; + /*! \brief Dependency Descriptors contexts, if any */ + janus_av1_svc_context av1_context[3]; /*! \brief Which simulcast substream we should forward back */ int substream; /*! \brief As above, but to handle transitions (e.g., wait for keyframe, or get this if available) */ @@ -348,6 +398,8 @@ void janus_rtp_simulcasting_cleanup(int *rid_ext_id, uint32_t *ssrcs, char **rid * @param[in] context The simulcasting context to use * @param[in] buf The RTP packet to process * @param[in] len The length of the RTP packet (header, extension and payload) + * @param[in] dd_content The Dependency Descriptor RTP extension data, if available + * @param[in] dd_len Length of the Dependency Descriptor data, if available * @param[in] ssrcs The simulcast SSRCs to refer to (may be updated if rids are involved) * @param[in] rids The simulcast rids to refer to, if any * @param[in] vcodec Video codec of the RTP payload @@ -355,58 +407,10 @@ void janus_rtp_simulcasting_cleanup(int *rid_ext_id, uint32_t *ssrcs, char **rid * @param[in] rid_mutex A mutex that must be acquired before reading the rids array, if any * @returns TRUE if the packet should be relayed, FALSE if it should be dropped instead */ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_context *context, - char *buf, int len, uint32_t *ssrcs, char **rids, + char *buf, int len, uint8_t *dd_content, int dd_len, uint32_t *ssrcs, char **rids, janus_videocodec vcodec, janus_rtp_switching_context *sc, janus_mutex *rid_mutex); ///@} -/** @name Janus AV1-SVC processing methods (still WIP) - */ -///@{ -/*! \brief Helper struct for processing and tracking AV1-SVC streams */ -typedef struct janus_av1_svc_context { - /*! \brief Number of templates advertised via Dependency Descriptor */ - uint8_t tcnt; - /*! \brief Template ID offset, as advertised via Dependency Descriptor */ - uint8_t tioff; - /*! \brief Map of templates advertised via Dependency Descriptor, indexed by ID */ - GHashTable *templates; - /*! \brief How many spatial and temporal layers are available */ - int spatial_layers, temporal_layers; - /*! \brief Whether this context changed since the last update */ - gboolean updated; -} janus_av1_svc_context; - -/*! \brief Helper struct to track SVC templates - * \note This is very incomplete, since we only track the spatial and - * temporal layer associated with a specific template ID for now */ -typedef struct janus_av1_svc_template { - /*! \brief Template ID */ - uint8_t id; - /*! \brief Spatial layer associated to this template */ - int spatial; - /*! \brief Temporal layer associated to this template */ - int temporal; -} janus_av1_svc_template; - -/*! \brief Set (or reset) the context fields to their default values - * @param[in] context The context to (re)set */ -void janus_av1_svc_context_reset(janus_av1_svc_context *context); - -/*! \brief Process a Dependency Descriptor payload, updating the SVC context accordingly - * \note At the moment, this code is quite naive, as it mostly looks at the target - * spatial/temporal layers, and the one written in the Dependency Descriptor data. - * In the future, this should become more sophisticated, and use additional - * information like dependency chains and stuff like that - * @param[in] context The av1svc context to use - * @param[in] dd Pointer to the Dependency Descriptor data - * @param[in] dd_len The length of the Dependendy Descriptor data - * @param[out] template_id Pointer to the ID of the template referenced in this packet - * @param[out] ebit Whether this packet is an end of frame or not - * @returns TRUE if the packet is valid, FALSE if it should be dropped instead */ -gboolean janus_av1_svc_context_process_dd(janus_av1_svc_context *context, - uint8_t *dd, int dd_len, uint8_t *template_id, uint8_t *ebit); -///@} - /** @name Janus SVC processing methods */ ///@{ diff --git a/src/rtpfwd.c b/src/rtpfwd.c index b5819e174a..c347055a8b 100644 --- a/src/rtpfwd.c +++ b/src/rtpfwd.c @@ -354,7 +354,7 @@ void janus_rtp_forwarder_send_rtp_full(janus_rtp_forwarder *rf, char *buffer, in if(rf->is_video && rf->simulcast) { /* This is video and we're simulcasting, check if we need to forward this frame */ if(!janus_rtp_simulcasting_context_process_rtp(&rf->sim_context, - buffer, len, ssrcs, rids, vcodec, &rf->rtp_context, rid_mutex)) { + buffer, len, NULL, 0, ssrcs, rids, vcodec, &rf->rtp_context, rid_mutex)) { /* There was an error processing simulcasting for this packet */ return; }