From 1102cf0898deb444a5aed06dcc3d2bbb2979c27b Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Thu, 10 Jun 2021 11:14:18 +0200 Subject: [PATCH] Fixed broken switching when using different payload types in Streaming plugin --- plugins/janus_streaming.c | 67 +++++++++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 20 deletions(-) diff --git a/plugins/janus_streaming.c b/plugins/janus_streaming.c index 45b0ac89bd..01c3ea292f 100644 --- a/plugins/janus_streaming.c +++ b/plugins/janus_streaming.c @@ -1017,6 +1017,7 @@ typedef struct janus_streaming_rtp_relay_packet { uint32_t ssrc[3]; janus_videocodec codec; int substream; + int ptype; uint32_t timestamp; uint16_t seq_number; /* The following are only relevant for VP9 SVC*/ @@ -1229,6 +1230,7 @@ typedef struct janus_streaming_session { volatile gint started; volatile gint paused; gboolean audio, video, data; /* Whether audio, video and/or data must be sent to this listener */ + int audio_pt, video_pt; janus_rtp_switching_context context; janus_rtp_simulcasting_context sim_context; janus_vp8_simulcast_context vp8_context; @@ -2323,6 +2325,8 @@ void janus_streaming_create_session(janus_plugin_session *handle, int *error) { g_atomic_int_set(&session->paused, 0); g_atomic_int_set(&session->destroyed, 0); g_atomic_int_set(&session->hangingup, 0); + session->audio_pt = -1; + session->video_pt = -1; handle->plugin_handle = session; janus_refcount_init(&session->ref, janus_streaming_session_free); janus_mutex_lock(&sessions_mutex); @@ -4531,6 +4535,8 @@ static void janus_streaming_hangup_media_internal(janus_plugin_session *handle) g_atomic_int_set(&session->stopping, 1); g_atomic_int_set(&session->started, 0); g_atomic_int_set(&session->paused, 0); + session->audio_pt = -1; + session->video_pt = -1; janus_rtp_switching_context_reset(&session->context); janus_rtp_simulcasting_context_reset(&session->sim_context); janus_vp8_simulcast_context_reset(&session->vp8_context); @@ -4871,6 +4877,12 @@ static void *janus_streaming_handler(void *data) { session->target_temporal_layer, session->temporal_layer); } } + /* Initialize the payload types this subscriber will expect */ + session->audio_pt = -1; + if(mp->codecs.audio_pt >= 0 && session->audio) + session->audio_pt = mp->codecs.audio_pt; + if(mp->codecs.video_pt >= 0 && session->video) + session->video_pt = mp->codecs.video_pt; /* If this mountpoint is broadcasting end-to-end encrypted media, * add the info to the JSEP offer we'll be sending them */ session->e2ee = source->e2ee; @@ -4892,22 +4904,22 @@ static void *janus_streaming_handler(void *data) { g_strlcat(sdptemp, buffer, 2048); g_strlcat(sdptemp, "t=0 0\r\n", 2048); if(mp->codecs.audio_pt >= 0 && session->audio) { + int pt = session->audio_pt >= 0 ? session->audio_pt : mp->codecs.audio_pt; /* Add audio line */ g_snprintf(buffer, 512, "m=audio 1 RTP/SAVPF %d\r\n" - "c=IN IP4 1.1.1.1\r\n", - mp->codecs.audio_pt); + "c=IN IP4 1.1.1.1\r\n", pt); g_strlcat(sdptemp, buffer, 2048); if(mp->codecs.audio_rtpmap) { g_snprintf(buffer, 512, "a=rtpmap:%d %s\r\n", - mp->codecs.audio_pt, mp->codecs.audio_rtpmap); + pt, mp->codecs.audio_rtpmap); g_strlcat(sdptemp, buffer, 2048); } if(mp->codecs.audio_fmtp) { g_snprintf(buffer, 512, "a=fmtp:%d %s\r\n", - mp->codecs.audio_pt, mp->codecs.audio_fmtp); + pt, mp->codecs.audio_fmtp); g_strlcat(sdptemp, buffer, 2048); } g_strlcat(sdptemp, "a=sendonly\r\n", 2048); @@ -4915,35 +4927,32 @@ static void *janus_streaming_handler(void *data) { g_strlcat(sdptemp, buffer, 2048); } if(mp->codecs.video_pt > 0 && session->video) { + int pt = session->video_pt > 0 ? session->video_pt : mp->codecs.video_pt; /* Add video line */ g_snprintf(buffer, 512, "m=video 1 RTP/SAVPF %d\r\n" - "c=IN IP4 1.1.1.1\r\n", - mp->codecs.video_pt); + "c=IN IP4 1.1.1.1\r\n", pt); g_strlcat(sdptemp, buffer, 2048); if(mp->codecs.video_rtpmap) { g_snprintf(buffer, 512, "a=rtpmap:%d %s\r\n", - mp->codecs.video_pt, mp->codecs.video_rtpmap); + pt, mp->codecs.video_rtpmap); g_strlcat(sdptemp, buffer, 2048); } if(mp->codecs.video_fmtp) { g_snprintf(buffer, 512, "a=fmtp:%d %s\r\n", - mp->codecs.video_pt, mp->codecs.video_fmtp); + pt, mp->codecs.video_fmtp); g_strlcat(sdptemp, buffer, 2048); } g_snprintf(buffer, 512, - "a=rtcp-fb:%d nack\r\n", - mp->codecs.video_pt); + "a=rtcp-fb:%d nack\r\n", pt); g_strlcat(sdptemp, buffer, 2048); g_snprintf(buffer, 512, - "a=rtcp-fb:%d nack pli\r\n", - mp->codecs.video_pt); + "a=rtcp-fb:%d nack pli\r\n", pt); g_strlcat(sdptemp, buffer, 2048); g_snprintf(buffer, 512, - "a=rtcp-fb:%d goog-remb\r\n", - mp->codecs.video_pt); + "a=rtcp-fb:%d goog-remb\r\n", pt); g_strlcat(sdptemp, buffer, 2048); g_strlcat(sdptemp, "a=sendonly\r\n", 2048); g_snprintf(buffer, 512, "a=extmap:%d %s\r\n", 1, JANUS_RTP_EXTMAP_MID); @@ -7350,7 +7359,8 @@ static void *janus_streaming_ondemand_thread(void *data) { packet.is_rtp = TRUE; packet.is_video = FALSE; packet.is_keyframe = FALSE; - /* Backup the actual timestamp and sequence number */ + /* Backup the actual payload type, timestamp and sequence number */ + packet.ptype = packet.data->type; packet.timestamp = ntohl(packet.data->timestamp); packet.seq_number = ntohs(packet.data->seq_number); /* Go! */ @@ -7499,7 +7509,8 @@ static void *janus_streaming_filesource_thread(void *data) { packet.is_rtp = TRUE; packet.is_video = FALSE; packet.is_keyframe = FALSE; - /* Backup the actual timestamp and sequence number */ + /* Backup the actual payload type, timestamp and sequence number */ + packet.ptype = packet.data->type; packet.timestamp = ntohl(packet.data->timestamp); packet.seq_number = ntohs(packet.data->seq_number); /* Go! */ @@ -7900,11 +7911,11 @@ static void *janus_streaming_relay_thread(void *data) { } if(mountpoint->enabled) { packet.data->ssrc = htonl(ssrc); - /* Backup the actual timestamp and sequence number set by the restreamer, in case switching is involved */ + /* Backup the actual payload type, timestamp and sequence number set by the restreamer, in case switching is involved */ + packet.ptype = packet.data->type; packet.timestamp = ntohl(packet.data->timestamp); packet.seq_number = ntohs(packet.data->seq_number); /* Go! */ - janus_mutex_lock(&mountpoint->mutex); g_list_foreach(mountpoint->helper_threads == 0 ? mountpoint->viewers : mountpoint->threads, mountpoint->helper_threads == 0 ? janus_streaming_relay_rtp_packet : janus_streaming_helper_rtprtcp_packet, @@ -7993,6 +8004,7 @@ static void *janus_streaming_relay_thread(void *data) { pkt->is_video = TRUE; pkt->is_keyframe = TRUE; pkt->length = bytes; + pkt->ptype = rtp->type; pkt->timestamp = source->keyframe.temp_ts; pkt->seq_number = ntohs(rtp->seq_number); source->keyframe.temp_keyframe = g_list_append(source->keyframe.temp_keyframe, pkt); @@ -8041,6 +8053,7 @@ static void *janus_streaming_relay_thread(void *data) { pkt->is_video = TRUE; pkt->is_keyframe = TRUE; pkt->length = bytes; + pkt->ptype = rtp->type; pkt->timestamp = source->keyframe.temp_ts; pkt->seq_number = ntohs(rtp->seq_number); source->keyframe.temp_keyframe = g_list_append(source->keyframe.temp_keyframe, pkt); @@ -8096,7 +8109,8 @@ static void *janus_streaming_relay_thread(void *data) { } if (mountpoint->enabled) { packet.data->ssrc = htonl(ssrc); - /* Backup the actual timestamp and sequence number set by the restreamer, in case switching is involved */ + /* Backup the actual payload type, timestamp and sequence number set by the restreamer, in case switching is involved */ + packet.ptype = packet.data->type; packet.timestamp = ntohl(packet.data->timestamp); packet.seq_number = ntohs(packet.data->seq_number); /* Take note of the simulcast SSRCs */ @@ -8436,6 +8450,8 @@ static void janus_streaming_relay_rtp_packet(gpointer data, gpointer user_data) if(override_mark_bit && !has_marker_bit) { packet->data->markerbit = 1; } + if(session->video_pt > 0) + packet->data->type = session->video_pt; janus_plugin_rtp rtp = { .video = packet->is_video, .buffer = (char *)packet->data, .length = packet->length }; janus_plugin_rtp_extensions_reset(&rtp.extensions); if(gateway != NULL) @@ -8443,7 +8459,8 @@ static void janus_streaming_relay_rtp_packet(gpointer data, gpointer user_data) if(override_mark_bit && !has_marker_bit) { packet->data->markerbit = 0; } - /* Restore the timestamp and sequence number to what the publisher set them to */ + /* Restore the payload type, timestamp and sequence number to what the publisher set them to */ + packet->data->type = packet->ptype; packet->data->timestamp = htonl(packet->timestamp); packet->data->seq_number = htons(packet->seq_number); } else if(packet->simulcast) { @@ -8504,12 +8521,15 @@ static void janus_streaming_relay_rtp_packet(gpointer data, gpointer user_data) janus_vp8_simulcast_descriptor_update(payload, plen, &session->vp8_context, session->sim_context.changed_substream); } + if(session->video_pt > 0) + packet->data->type = session->video_pt; /* Send the packet */ janus_plugin_rtp rtp = { .video = packet->is_video, .buffer = (char *)packet->data, .length = packet->length }; janus_plugin_rtp_extensions_reset(&rtp.extensions); if(gateway != NULL) gateway->relay_rtp(session->handle, &rtp); /* Restore the timestamp and sequence number to what the publisher set them to */ + packet->data->type = packet->ptype; packet->data->timestamp = htonl(packet->timestamp); packet->data->seq_number = htons(packet->seq_number); if(packet->codec == JANUS_VIDEOCODEC_VP8) { @@ -8519,11 +8539,14 @@ static void janus_streaming_relay_rtp_packet(gpointer data, gpointer user_data) } else { /* Fix sequence number and timestamp (switching may be involved) */ janus_rtp_header_update(packet->data, &session->context, TRUE, 0); + if(session->video_pt > 0) + packet->data->type = session->video_pt; janus_plugin_rtp rtp = { .video = packet->is_video, .buffer = (char *)packet->data, .length = packet->length }; janus_plugin_rtp_extensions_reset(&rtp.extensions); if(gateway != NULL) gateway->relay_rtp(session->handle, &rtp); /* Restore the timestamp and sequence number to what the video source set them to */ + packet->data->type = packet->ptype; packet->data->timestamp = htonl(packet->timestamp); packet->data->seq_number = htons(packet->seq_number); } @@ -8532,11 +8555,14 @@ static void janus_streaming_relay_rtp_packet(gpointer data, gpointer user_data) return; /* Fix sequence number and timestamp (switching may be involved) */ janus_rtp_header_update(packet->data, &session->context, FALSE, 0); + if(session->audio_pt >= 0) + packet->data->type = session->audio_pt; janus_plugin_rtp rtp = { .video = packet->is_video, .buffer = (char *)packet->data, .length = packet->length }; janus_plugin_rtp_extensions_reset(&rtp.extensions); if(gateway != NULL) gateway->relay_rtp(session->handle, &rtp); /* Restore the timestamp and sequence number to what the video source set them to */ + packet->data->type = packet->ptype; packet->data->timestamp = htonl(packet->timestamp); packet->data->seq_number = htons(packet->seq_number); } @@ -8609,6 +8635,7 @@ static void janus_streaming_helper_rtprtcp_packet(gpointer data, gpointer user_d copy->ssrc[2] = packet->ssrc[2]; copy->codec = packet->codec; copy->substream = packet->substream; + copy->ptype = packet->ptype; copy->timestamp = packet->timestamp; copy->seq_number = packet->seq_number; g_async_queue_push(helper->queued_packets, copy);