Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Initial support for VP9 and AV1 simulcast (and fix for broken AV1/SVC) #3218

Merged
merged 6 commits into from
May 22, 2023
Merged

Initial support for VP9 and AV1 simulcast (and fix for broken AV1/SVC) #3218

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
73c3de2
Initial support for VP9 and AV1 simulcast
lminiero May 10, 2023
c64793b
Read VP9 temporal layers from payload descriptor (SVC code)
lminiero May 11, 2023
cf6eb00
Use Dependency Descriptor for temporal layers in AV1 simulcasting, if…
lminiero May 11, 2023
f282022
Show temporal layer buttons in EchoTest demos also for VP9 and AV1
lminiero May 11, 2023
8de2c78
Made AV1/SVC finally work
lminiero May 12, 2023
a001ded
Streamlined simulcast usage when involving DD, and added support to o…
lminiero May 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion html/echotest.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,7 @@ $(document).ready(function() {
if((substream !== null && substream !== undefined) || (temporal !== null && temporal !== undefined)) {
if(!simulcastStarted) {
simulcastStarted = true;
addSimulcastSvcButtons(msg["videocodec"] === "vp8");
addSimulcastSvcButtons(msg["videocodec"] === "vp8" || msg["videocodec"] === "vp9" || msg["videocodec"] === "av1");
}
// We just received notice that there's been a switch, update the buttons
updateSimulcastSvcButtons(substream, temporal);
Expand Down
6 changes: 3 additions & 3 deletions html/janus.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2497,9 +2497,9 @@ function Janus(gatewayCallbacks) {
direction: 'sendrecv',
streams: [config.myStream],
sendEncodings: track.sendEncodings || [
{ rid: 'h', active: true, maxBitrate: maxBitrates.high },
{ rid: 'm', active: true, maxBitrate: maxBitrates.medium, scaleResolutionDownBy: 2 },
{ rid: 'l', active: true, maxBitrate: maxBitrates.low, scaleResolutionDownBy: 4 }
{ rid: 'h', active: true, scalabilityMode: 'L1T2', maxBitrate: maxBitrates.high },
{ rid: 'm', active: true, scalabilityMode: 'L1T2', maxBitrate: maxBitrates.medium, scaleResolutionDownBy: 2 },
{ rid: 'l', active: true, scalabilityMode: 'L1T2', maxBitrate: maxBitrates.low, scaleResolutionDownBy: 4 }
]
});
} else {
Expand Down
2 changes: 2 additions & 0 deletions src/plugins/duktape/janus-sdp.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -399,6 +399,8 @@ JANUSSDP.generateAnswer = function(offer, options) {
answer.push({ type: "a", name: a.name, value: value });
} else if(options.enableAudioLevel !== false && a.value.indexOf("urn:ietf:params:rtp-hdrext:ssrc-audio-level") !== -1) {
answer.push({ type: "a", name: a.name, value: value });
} else if(options.enableAudioLevel !== false && a.value.indexOf("dependency-descriptor-rtp-header-extension") !== -1) {
answer.push({ type: "a", name: a.name, value: value });
}
}
} else {
Expand Down
13 changes: 3 additions & 10 deletions src/plugins/janus_duktape.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -619,10 +619,6 @@ static duk_ret_t janus_duktape_method_pushevent(duk_context *ctx) {
janus_sdp_find_first_codec(parsed_sdp, JANUS_SDP_VIDEO, -1, &vcodec);
if(vcodec)
session->vcodec = janus_videocodec_from_name(vcodec);
if(session->vcodec != JANUS_VIDEOCODEC_VP8 && session->vcodec != JANUS_VIDEOCODEC_H264) {
/* VP8 r H.264 were not negotiated, if simulcasting was enabled then disable it here */
janus_rtp_simulcasting_cleanup(&session->rid_extmap_id, session->ssrc, session->rid, &session->rid_mutex);
}
}
janus_sdp_destroy(parsed_sdp);
/* Send asynchronously */
Expand Down Expand Up @@ -2240,10 +2236,6 @@ struct janus_plugin_result *janus_duktape_handle_message(janus_plugin_session *h
janus_sdp_find_first_codec(parsed_sdp, JANUS_SDP_VIDEO, -1, &vcodec);
if(vcodec)
session->vcodec = janus_videocodec_from_name(vcodec);
if(session->vcodec != JANUS_VIDEOCODEC_VP8 && session->vcodec != JANUS_VIDEOCODEC_H264) {
/* VP8 r H.264 were not negotiated, if simulcasting was enabled then disable it here */
janus_rtp_simulcasting_cleanup(&session->rid_extmap_id, session->ssrc, session->rid, &session->rid_mutex);
}
janus_sdp_destroy(parsed_sdp);
}
if(json_is_true(json_object_get(jsep, "e2ee")))
Expand Down Expand Up @@ -2459,7 +2451,7 @@ void janus_duktape_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *
} else {
/* We're simulcasting, save the best video quality */
gboolean save = janus_rtp_simulcasting_context_process_rtp(&session->rec_simctx,
buf, len, session->ssrc, session->rid, session->vcodec, &session->rec_ctx, &session->rid_mutex);
buf, len, NULL, 0, session->ssrc, session->rid, session->vcodec, &session->rec_ctx, &session->rid_mutex);
if(save) {
uint32_t seq_number = ntohs(rtp->seq_number);
uint32_t timestamp = ntohl(rtp->timestamp);
Expand Down Expand Up @@ -2781,7 +2773,8 @@ static void janus_duktape_relay_rtp_packet(gpointer data, gpointer user_data) {
return;
/* Process this packet: don't relay if it's not the SSRC/layer we wanted to handle */
gboolean relay = janus_rtp_simulcasting_context_process_rtp(&session->sim_context,
(char *)packet->data, packet->length, packet->ssrc, NULL, sender->vcodec, &session->vrtpctx, NULL);
(char *)packet->data, packet->length, packet->extensions.dd_content, packet->extensions.dd_len,
packet->ssrc, NULL, sender->vcodec, &session->vrtpctx, NULL);
if(session->sim_context.need_pli && sender->handle) {
/* Send a PLI */
JANUS_LOG(LOG_VERB, "We need a PLI for the simulcast context\n");
Expand Down
11 changes: 4 additions & 7 deletions src/plugins/janus_echotest.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -624,11 +624,12 @@ void janus_echotest_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp
if(simulcast) {
/* Process this simulcast packet: don't relay if it's not the SSRC/layer we wanted to handle */
relay = janus_rtp_simulcasting_context_process_rtp(&session->sim_context,
buf, len, session->ssrc, session->rid, session->vcodec, &session->context, &session->rid_mutex);
buf, len, packet->extensions.dd_content, packet->extensions.dd_len,
session->ssrc, session->rid, session->vcodec, &session->context, &session->rid_mutex);
} else {
/* Process this SVC packet: don't relay if it's not the layer we wanted to handle */
relay = janus_rtp_svc_context_process_rtp(&session->svc_context,
buf, len, session->vcodec, NULL, &session->context);
buf, len, packet->extensions.dd_content, packet->extensions.dd_len, session->vcodec, NULL, &session->context);
}
if(session->sim_context.need_pli || session->svc_context.need_pli) {
/* Send a PLI */
Expand Down Expand Up @@ -1088,7 +1089,7 @@ static void *janus_echotest_handler(void *data) {
session->sim_context.templayer_target = json_integer_value(temporal);
JANUS_LOG(LOG_VERB, "Setting video temporal layer to let through (simulcast): %d (was %d)\n",
session->sim_context.templayer_target, session->sim_context.templayer);
if(session->vcodec == JANUS_VIDEOCODEC_VP8 && session->sim_context.templayer_target == session->sim_context.templayer) {
if(session->sim_context.templayer_target == session->sim_context.templayer) {
/* No need to do anything, we're already getting the right temporal, so notify the user */
json_t *event = json_object();
json_object_set_new(event, "echotest", json_string("event"));
Expand Down Expand Up @@ -1280,10 +1281,6 @@ static void *janus_echotest_handler(void *data) {
session->vcodec = janus_videocodec_from_name(vcodec);
session->has_audio = session->acodec != JANUS_AUDIOCODEC_NONE;
session->has_video = session->vcodec != JANUS_VIDEOCODEC_NONE;
if(session->vcodec != JANUS_VIDEOCODEC_VP8 && session->vcodec != JANUS_VIDEOCODEC_H264) {
/* VP8 r H.264 were not negotiated, if simulcasting was enabled then disable it here */
janus_rtp_simulcasting_cleanup(NULL, session->ssrc, session->rid, &session->rid_mutex);
}
g_free(session->vfmtp);
session->vfmtp = NULL;
if(session->has_video) {
Expand Down
13 changes: 3 additions & 10 deletions src/plugins/janus_lua.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -547,10 +547,6 @@ static int janus_lua_method_pushevent(lua_State *s) {
janus_sdp_find_first_codec(parsed_sdp, JANUS_SDP_VIDEO, -1, &vcodec);
if(vcodec)
session->vcodec = janus_videocodec_from_name(vcodec);
if(session->vcodec != JANUS_VIDEOCODEC_VP8 && session->vcodec != JANUS_VIDEOCODEC_H264) {
/* VP8 r H.264 were not negotiated, if simulcasting was enabled then disable it here */
janus_rtp_simulcasting_cleanup(&session->rid_extmap_id, session->ssrc, session->rid, &session->rid_mutex);
}
}
janus_sdp_destroy(parsed_sdp);
/* Send asynchronously */
Expand Down Expand Up @@ -1936,10 +1932,6 @@ struct janus_plugin_result *janus_lua_handle_message(janus_plugin_session *handl
janus_sdp_find_first_codec(parsed_sdp, JANUS_SDP_VIDEO, -1, &vcodec);
if(vcodec)
session->vcodec = janus_videocodec_from_name(vcodec);
if(session->vcodec != JANUS_VIDEOCODEC_VP8 && session->vcodec != JANUS_VIDEOCODEC_H264) {
/* VP8 r H.264 were not negotiated, if simulcasting was enabled then disable it here */
janus_rtp_simulcasting_cleanup(&session->rid_extmap_id, session->ssrc, session->rid, &session->rid_mutex);
}
janus_sdp_destroy(parsed_sdp);
}
if(json_is_true(json_object_get(jsep, "e2ee")))
Expand Down Expand Up @@ -2127,7 +2119,7 @@ void janus_lua_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *rtp_
} else {
/* We're simulcasting, save the best video quality */
gboolean save = janus_rtp_simulcasting_context_process_rtp(&session->rec_simctx,
buf, len, session->ssrc, session->rid, session->vcodec, &session->rec_ctx, &session->rid_mutex);
buf, len, NULL, 0, session->ssrc, session->rid, session->vcodec, &session->rec_ctx, &session->rid_mutex);
if(save) {
uint32_t seq_number = ntohs(rtp->seq_number);
uint32_t timestamp = ntohl(rtp->timestamp);
Expand Down Expand Up @@ -2421,7 +2413,8 @@ static void janus_lua_relay_rtp_packet(gpointer data, gpointer user_data) {
return;
/* Process this packet: don't relay if it's not the SSRC/layer we wanted to handle */
gboolean relay = janus_rtp_simulcasting_context_process_rtp(&session->sim_context,
(char *)packet->data, packet->length, packet->ssrc, NULL, sender->vcodec, &session->vrtpctx, NULL);
(char *)packet->data, packet->length, packet->extensions.dd_content, packet->extensions.dd_len,
packet->ssrc, NULL, sender->vcodec, &session->vrtpctx, NULL);
if(session->sim_context.need_pli && sender->handle) {
/* Send a PLI */
JANUS_LOG(LOG_VERB, "We need a PLI for the simulcast context\n");
Expand Down
7 changes: 2 additions & 5 deletions src/plugins/janus_recordplay.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1304,7 +1304,8 @@ void janus_recordplay_incoming_rtp(janus_plugin_session *handle, janus_plugin_rt
uint32_t ssrc = ntohl(header->ssrc);
/* Process this packet: don't save if it's not the SSRC/layer we wanted to handle */
gboolean save = janus_rtp_simulcasting_context_process_rtp(&session->sim_context,
buf, len, session->ssrc, session->rid, session->recording->vcodec, &session->context, &session->rid_mutex);
buf, len, NULL, 0, session->ssrc, session->rid, session->recording->vcodec,
&session->context, &session->rid_mutex);
if(session->sim_context.need_pli) {
/* Send a PLI */
JANUS_LOG(LOG_VERB, "We need a PLI for the simulcast context\n");
Expand Down Expand Up @@ -1879,10 +1880,6 @@ static void *janus_recordplay_handler(void *data) {
janus_mutex_unlock(&session->rid_mutex);
session->sim_context.substream_target = 2; /* Let's aim for the highest quality */
session->sim_context.templayer_target = 2; /* Let's aim for all temporal layers */
if(rec->vcodec != JANUS_VIDEOCODEC_VP8 && rec->vcodec != JANUS_VIDEOCODEC_H264) {
/* VP8 r H.264 were not negotiated, if simulcasting was enabled then disable it here */
janus_rtp_simulcasting_cleanup(NULL, session->ssrc, session->rid, &session->rid_mutex);
}
/* FIXME We're stopping at the first item, there may be more */
break;
}
Expand Down
3 changes: 2 additions & 1 deletion src/plugins/janus_streaming.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10030,7 +10030,8 @@ static void janus_streaming_relay_rtp_packet(gpointer data, gpointer user_data)
return;
/* Process this packet: don't relay if it's not the SSRC/layer we wanted to handle */
gboolean relay = janus_rtp_simulcasting_context_process_rtp(&s->sim_context,
(char *)packet->data, packet->length, packet->ssrc, NULL, packet->codec, &s->context, NULL);
(char *)packet->data, packet->length, NULL, 0,
packet->ssrc, NULL, packet->codec, &s->context, NULL);
if(!relay) {
/* Did a lot of time pass before we could relay a packet? */
gint64 now = janus_get_monotonic_time();
Expand Down
9 changes: 5 additions & 4 deletions src/plugins/janus_videocall.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -778,7 +778,8 @@ void janus_videocall_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp
/* Process this packet: don't relay if it's not the SSRC/layer we wanted to handle
* The caveat is that the targets in OUR simulcast context are the PEER's targets */
gboolean relay = janus_rtp_simulcasting_context_process_rtp(&peer->sim_context,
buf, len, session->ssrc, session->rid, session->vcodec, &peer->context, &session->rid_mutex);
buf, len, packet->extensions.dd_content, packet->extensions.dd_len,
session->ssrc, session->rid, session->vcodec, &peer->context, &session->rid_mutex);
/* Do we need to drop this? */
if(!relay)
return;
Expand Down Expand Up @@ -1374,7 +1375,7 @@ static void *janus_videocall_handler(void *data) {
session->has_data = (strstr(msg_sdp, "DTLS/SCTP") != NULL);
/* Check if this user will simulcast */
json_t *msg_simulcast = json_object_get(msg->jsep, "simulcast");
if(msg_simulcast && janus_get_codec_pt(msg_sdp, "vp8") > 0) {
if(msg_simulcast) {
JANUS_LOG(LOG_VERB, "VideoCall callee (%s) cannot do simulcast.\n", session->username);
} else {
janus_rtp_simulcasting_cleanup(NULL, session->ssrc, session->rid, &session->rid_mutex);
Expand Down Expand Up @@ -1533,7 +1534,7 @@ static void *janus_videocall_handler(void *data) {
session->sim_context.templayer_target = json_integer_value(temporal);
JANUS_LOG(LOG_VERB, "Setting video temporal layer to let through (simulcast): %d (was %d)\n",
session->sim_context.templayer_target, session->sim_context.templayer);
if(session->vcodec == JANUS_VIDEOCODEC_VP8 && session->sim_context.templayer_target == session->sim_context.templayer) {
if(session->sim_context.templayer_target == session->sim_context.templayer) {
/* No need to do anything, we're already getting the right temporal, so notify the user */
json_t *event = json_object();
json_object_set_new(event, "videocall", json_string("event"));
Expand Down Expand Up @@ -1569,7 +1570,7 @@ static void *janus_videocall_handler(void *data) {
session->has_data = (strstr(msg_sdp, "DTLS/SCTP") != NULL);
/* Check if this user will simulcast */
json_t *msg_simulcast = json_object_get(msg->jsep, "simulcast");
if(msg_simulcast && janus_get_codec_pt(msg_sdp, "vp8") > 0) {
if(msg_simulcast) {
JANUS_LOG(LOG_VERB, "VideoCall callee (%s) cannot do simulcast.\n", session->username);
} else {
janus_rtp_simulcasting_cleanup(NULL, session->ssrc, session->rid, &session->rid_mutex);
Expand Down
Loading