From 24548023f58c8dc6b31f122ddfb72c04ba5e919d Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Wed, 27 Jan 2021 11:22:13 +0100 Subject: [PATCH] Allow marking of RTP extensions in MJR recordings (#2527) --- plugins/janus_recordplay.c | 70 +++++++++++++++++++++++++++++++---- postprocessing/janus-pp-rec.c | 40 +++++++++++++++++++- postprocessing/pp-rtp.h | 3 ++ record.c | 35 +++++++++++++++++- record.h | 10 +++++ 5 files changed, 149 insertions(+), 9 deletions(-) diff --git a/plugins/janus_recordplay.c b/plugins/janus_recordplay.c index 45e4d8c83d..2427d7afe9 100644 --- a/plugins/janus_recordplay.c +++ b/plugins/janus_recordplay.c @@ -411,6 +411,8 @@ typedef struct janus_recordplay_recording { janus_videocodec vcodec; /* Codec used for video, if available */ char *vfmtp; /* Video fmtp, if any */ int video_pt; /* Payload types to use for audio when playing recordings */ + guint8 audiolevel_ext_id; /* Audio level extmap ID */ + guint8 videoorient_ext_id; /* Video orientation extmap ID */ char *drc_file; /* Data file name */ gboolean textdata; /* Whether data format is text */ char *offer; /* The SDP offer that will be sent to watchers */ @@ -507,7 +509,8 @@ void janus_recordplay_send_rtcp_feedback(janus_plugin_session *handle, int video #define VIDEO_PT 100 /* Helper method to check which codec was used in a specific recording (and if it's end-to-end encrypted) */ -static const char *janus_recordplay_parse_codec(const char *dir, const char *filename, char *fmtp, size_t fmtplen, gboolean *e2ee) { +static const char *janus_recordplay_parse_codec(const char *dir, const char *filename, char *fmtp, size_t fmtplen, + uint8_t *audiolevel_ext_id, uint8_t *videoorient_ext_id, gboolean *e2ee) { if(dir == NULL || filename == NULL) return NULL; if(e2ee) @@ -629,6 +632,23 @@ static const char *janus_recordplay_parse_codec(const char *dir, const char *fil fclose(file); return NULL; } + /* Any RTP extension we care about? */ + json_t *exts = json_object_get(info, "x"); + if(exts && !data) { + int extid = 0; + const char *key = NULL, *extmap = NULL; + json_t *value = NULL; + json_object_foreach(exts, key, value) { + if(key == NULL || value == NULL || !json_is_string(value)) + continue; + extid = atoi(key); + extmap = json_string_value(value); + if(!video && !strcasecmp(extmap, JANUS_RTP_EXTMAP_AUDIO_LEVEL) && audiolevel_ext_id != NULL) + *audiolevel_ext_id = extid; + else if(video && !strcasecmp(extmap, JANUS_RTP_EXTMAP_VIDEO_ORIENTATION) && videoorient_ext_id != NULL) + *videoorient_ext_id = extid; + } + } const char *c = json_string_value(codec); if(data) { const char *dtype = NULL; @@ -679,6 +699,9 @@ static int janus_recordplay_generate_offer(janus_recordplay_recording *rec) { offer_data = (rec->drc_file != NULL); char s_name[100]; g_snprintf(s_name, sizeof(s_name), "Recording %"SCNu64, rec->id); + guint8 mid_ext_id = 1; + while(mid_ext_id == rec->audiolevel_ext_id || mid_ext_id == rec->videoorient_ext_id) + mid_ext_id++; janus_sdp *offer = janus_sdp_generate_offer( s_name, "1.1.1.1", JANUS_SDP_OA_AUDIO, offer_audio, @@ -686,13 +709,15 @@ static int janus_recordplay_generate_offer(janus_recordplay_recording *rec) { JANUS_SDP_OA_AUDIO_PT, rec->audio_pt, JANUS_SDP_OA_AUDIO_FMTP, rec->afmtp, JANUS_SDP_OA_AUDIO_DIRECTION, JANUS_SDP_SENDONLY, - JANUS_SDP_OA_AUDIO_EXTENSION, JANUS_RTP_EXTMAP_MID, 1, + JANUS_SDP_OA_AUDIO_EXTENSION, JANUS_RTP_EXTMAP_MID, mid_ext_id, + JANUS_SDP_OA_AUDIO_EXTENSION, JANUS_RTP_EXTMAP_AUDIO_LEVEL, rec->audiolevel_ext_id, JANUS_SDP_OA_VIDEO, offer_video, JANUS_SDP_OA_VIDEO_CODEC, janus_videocodec_name(rec->vcodec), JANUS_SDP_OA_VIDEO_FMTP, rec->vfmtp, JANUS_SDP_OA_VIDEO_PT, rec->video_pt, JANUS_SDP_OA_VIDEO_DIRECTION, JANUS_SDP_SENDONLY, - JANUS_SDP_OA_VIDEO_EXTENSION, JANUS_RTP_EXTMAP_MID, 1, + JANUS_SDP_OA_VIDEO_EXTENSION, JANUS_RTP_EXTMAP_MID, mid_ext_id, + JANUS_SDP_OA_AUDIO_EXTENSION, JANUS_RTP_EXTMAP_VIDEO_ORIENTATION, rec->videoorient_ext_id, JANUS_SDP_OA_DATA, offer_data, JANUS_SDP_OA_DONE); g_free(rec->offer); @@ -1672,6 +1697,30 @@ static void *janus_recordplay_handler(void *data) { rec->audio_pt = 9; } rec->video_pt = VIDEO_PT; + /* Check if relevant extensions are negotiated */ + GList *temp = offer->m_lines; + while(temp) { + /* Which media are available? */ + janus_sdp_mline *m = (janus_sdp_mline *)temp->data; + if(m->type == JANUS_SDP_AUDIO || m->type == JANUS_SDP_VIDEO) { + /* Are the extmaps we care about there? */ + GList *ma = m->attributes; + while(ma) { + janus_sdp_attribute *a = (janus_sdp_attribute *)ma->data; + if(a->name && a->value) { + if(m->type == JANUS_SDP_AUDIO && strstr(a->value, JANUS_RTP_EXTMAP_AUDIO_LEVEL)) { + if(janus_string_to_uint8(a->value, &rec->audiolevel_ext_id) < 0) + JANUS_LOG(LOG_WARN, "Invalid audio-level extension ID: %s\n", a->value); + } else if(m->type == JANUS_SDP_VIDEO && strstr(a->value, JANUS_RTP_EXTMAP_VIDEO_ORIENTATION)) { + if(janus_string_to_uint8(a->value, &rec->videoorient_ext_id) < 0) + JANUS_LOG(LOG_WARN, "Invalid video-orientation extension ID: %s\n", a->value); + } + } + ma = ma->next; + } + } + temp = temp->next; + } /* Create a date string */ time_t t = time(NULL); struct tm *tmv = localtime(&t); @@ -1688,6 +1737,9 @@ static void *janus_recordplay_handler(void *data) { } rec->arc_file = g_strdup(filename); janus_recorder *rc = janus_recorder_create(recordings_path, janus_audiocodec_name(rec->acodec), rec->arc_file); + /* If the audio-level extension has been negotiated, mark it in the recording */ + if(rec->audiolevel_ext_id > 0) + janus_recorder_add_extmap(rc, rec->audiolevel_ext_id, JANUS_RTP_EXTMAP_AUDIO_LEVEL); /* If media is encrypted, mark it in the recording */ if(e2ee) janus_recorder_encrypted(rc); @@ -1703,6 +1755,9 @@ static void *janus_recordplay_handler(void *data) { rec->vrc_file = g_strdup(filename); janus_recorder *rc = janus_recorder_create_full(recordings_path, janus_videocodec_name(rec->vcodec), rec->vfmtp, rec->vrc_file); + /* If the video-orientation extension has been negotiated, mark it in the recording */ + if(rec->videoorient_ext_id > 0) + janus_recorder_add_extmap(rc, rec->videoorient_ext_id, JANUS_RTP_EXTMAP_VIDEO_ORIENTATION); /* If media is encrypted, mark it in the recording */ if(e2ee) janus_recorder_encrypted(rc); @@ -1745,8 +1800,9 @@ static void *janus_recordplay_handler(void *data) { JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_MID, JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_RID, JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_REPAIRED_RID, - JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_FRAME_MARKING, JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_TRANSPORT_WIDE_CC, + JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_AUDIO_LEVEL, + JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_VIDEO_ORIENTATION, JANUS_SDP_OA_DONE); g_free(answer->s_name); char s_name[100]; @@ -2101,7 +2157,7 @@ void janus_recordplay_update_recordings_list(void) { char fmtp[256]; fmtp[0] = '\0'; rec->acodec = janus_audiocodec_from_name(janus_recordplay_parse_codec(recordings_path, - rec->arc_file, fmtp, sizeof(fmtp), &e2ee)); + rec->arc_file, fmtp, sizeof(fmtp), &rec->audiolevel_ext_id, NULL, &e2ee)); if(strlen(fmtp) > 0) rec->afmtp = g_strdup(fmtp); if(e2ee) @@ -2117,7 +2173,7 @@ void janus_recordplay_update_recordings_list(void) { char fmtp[256]; fmtp[0] = '\0'; rec->vcodec = janus_videocodec_from_name(janus_recordplay_parse_codec(recordings_path, - rec->vrc_file, fmtp, sizeof(fmtp), &e2ee)); + rec->vrc_file, fmtp, sizeof(fmtp), NULL, &rec->videoorient_ext_id, &e2ee)); if(strlen(fmtp) > 0) rec->vfmtp = g_strdup(fmtp); if(e2ee) @@ -2129,7 +2185,7 @@ void janus_recordplay_update_recordings_list(void) { if(ext != NULL) *ext = '\0'; const char *textcodec = janus_recordplay_parse_codec(recordings_path, - rec->drc_file, NULL, sizeof(NULL), NULL); + rec->drc_file, NULL, sizeof(NULL), NULL, NULL, NULL); rec->textdata = textcodec && (!strcasecmp(textcodec, "text")); } rec->audio_pt = AUDIO_PT; diff --git a/postprocessing/janus-pp-rec.c b/postprocessing/janus-pp-rec.c index 4aa16939ab..5ff32ae7b4 100644 --- a/postprocessing/janus-pp-rec.c +++ b/postprocessing/janus-pp-rec.c @@ -593,6 +593,44 @@ int main(int argc, char *argv[]) } /* Any codec-specific info? (just informational) */ const char *f = json_string_value(json_object_get(info, "f")); + /* Check if there are RTP extensions */ + json_t *exts = json_object_get(info, "x"); + if(exts != NULL) { + /* There are: check if audio-level and/or video-orientation + * are among them, as we might need them */ + int extid = 0; + const char *key = NULL, *extmap = NULL; + json_t *value = NULL; + json_object_foreach(exts, key, value) { + if(key == NULL || value == NULL || !json_is_string(value)) + continue; + extid = atoi(key); + extmap = json_string_value(value); + if(!strcasecmp(extmap, JANUS_PP_RTP_EXTMAP_AUDIO_LEVEL)) { + /* Audio level */ + if(audio_level_extmap_id != -1) { + if(audio_level_extmap_id != extid) { + JANUS_LOG(LOG_WARN, "Audio level extension ID found in header (%d) is different from the one provided via argument (%d)\n", + audio_level_extmap_id, extid); + } + } else { + audio_level_extmap_id = extid; + JANUS_LOG(LOG_INFO, "Audio level extension ID: %d\n", audio_level_extmap_id); + } + } else if(!strcasecmp(extmap, JANUS_PP_RTP_EXTMAP_VIDEO_ORIENTATION)) { + /* Video orientation */ + if(video_orient_extmap_id != -1) { + if(video_orient_extmap_id != extid) { + JANUS_LOG(LOG_WARN, "Video orientation extension ID found in header (%d) is different from the one provided via argument (%d)\n", + video_orient_extmap_id, extid); + } + } else { + video_orient_extmap_id = extid; + JANUS_LOG(LOG_INFO, "Video orientation extension ID: %d\n", video_orient_extmap_id); + } + } + } + } /* When was the file created? */ json_t *created = json_object_get(info, "s"); if(!created || !json_is_integer(created)) { @@ -639,7 +677,7 @@ int main(int argc, char *argv[]) } /* Now that we know what we're working with, check the extension */ - if(strcasecmp(extension, "opus") && strcasecmp(extension, "wav") && + if(extension && strcasecmp(extension, "opus") && strcasecmp(extension, "wav") && strcasecmp(extension, "webm") && strcasecmp(extension, "mp4") && strcasecmp(extension, "srt") && (!data || (data && textdata))) { /* Unsupported extension? */ diff --git a/postprocessing/pp-rtp.h b/postprocessing/pp-rtp.h index 958f79f48f..121fa739b9 100644 --- a/postprocessing/pp-rtp.h +++ b/postprocessing/pp-rtp.h @@ -24,6 +24,9 @@ #include +#define JANUS_PP_RTP_EXTMAP_AUDIO_LEVEL "urn:ietf:params:rtp-hdrext:ssrc-audio-level" +#define JANUS_PP_RTP_EXTMAP_VIDEO_ORIENTATION "urn:3gpp:video-orientation" + typedef struct janus_pp_rtp_header { #if __BYTE_ORDER == __BIG_ENDIAN diff --git a/record.c b/record.c index 7c5eb7f7f3..d5117a8083 100644 --- a/record.c +++ b/record.c @@ -77,6 +77,8 @@ static void janus_recorder_free(const janus_refcount *recorder_ref) { recorder->codec = NULL; g_free(recorder->fmtp); recorder->fmtp = NULL; + if(recorder->extensions != NULL) + g_hash_table_destroy(recorder->extensions); g_free(recorder); } @@ -253,10 +255,21 @@ janus_recorder *janus_recorder_create_full(const char *dir, const char *codec, c return rc; } +int janus_recorder_add_extmap(janus_recorder *recorder, int id, const char *extmap) { + if(!recorder || g_atomic_int_get(&recorder->header) || id < 1 || id > 15 || extmap == NULL) + return -1; + janus_mutex_lock_nodebug(&recorder->mutex); + if(recorder->extensions == NULL) + recorder->extensions = g_hash_table_new_full(NULL, NULL, NULL, (GDestroyNotify)g_free); + g_hash_table_insert(recorder->extensions, GINT_TO_POINTER(id), g_strdup(extmap)); + janus_mutex_unlock_nodebug(&recorder->mutex); + return 0; +} + int janus_recorder_encrypted(janus_recorder *recorder) { if(!recorder) return -1; - if(!recorder->header) { + if(!g_atomic_int_get(&recorder->header)) { recorder->encrypted = TRUE; return 0; } @@ -295,6 +308,26 @@ int janus_recorder_save_frame(janus_recorder *recorder, char *buffer, uint lengt json_object_set_new(info, "c", json_string(recorder->codec)); /* Media codec */ if(recorder->fmtp) json_object_set_new(info, "f", json_string(recorder->fmtp)); /* Codec-specific info */ + if(recorder->extensions) { + /* Add the extmaps to the JSON object */ + json_t *extmaps = NULL; + GHashTableIter iter; + gpointer key, value; + g_hash_table_iter_init(&iter, recorder->extensions); + while(g_hash_table_iter_next(&iter, &key, &value)) { + int id = GPOINTER_TO_INT(key); + char *extmap = (char *)value; + if(id > 0 && id < 16 && extmap != NULL) { + if(extmaps == NULL) + extmaps = json_object(); + char id_str[3]; + g_snprintf(id_str, sizeof(id_str), "%d", id); + json_object_set_new(extmaps, id_str, json_string(extmap)); + } + } + if(extmaps != NULL) + json_object_set_new(info, "x", extmaps); + } json_object_set_new(info, "s", json_integer(recorder->created)); /* Created time */ json_object_set_new(info, "u", json_integer(janus_get_real_time())); /* First frame written time */ /* If media will be end-to-end encrypted, mark it in the recording header */ diff --git a/record.h b/record.h index 6a4337b2a3..482c9fc060 100644 --- a/record.h +++ b/record.h @@ -48,6 +48,8 @@ typedef struct janus_recorder { char *codec; /*! \brief Codec-specific info (e.g., H.264 or VP9 profile) */ char *fmtp; + /*! \brief List of RTP extensions (as a hashtable, indexed by ID) in this recording */ + GHashTable *extensions; /*! \brief When the recording file has been created and started */ gint64 created, started; /*! \brief Media this instance is recording */ @@ -90,6 +92,14 @@ janus_recorder *janus_recorder_create(const char *dir, const char *codec, const * @param[in] filename Filename to use for the recording * @returns A valid janus_recorder instance in case of success, NULL otherwise */ janus_recorder *janus_recorder_create_full(const char *dir, const char *codec, const char *fmtp, const char *filename); +/*! \brief Add an RTP extension to this recording + * \note This will only be possible BEFORE the first frame is written, as it needs to + * be reflected in the .mjr header: doing this after that will return an error. + * @param[in] recorder The janus_recorder instance to add the extension to + * @param[in] id Numeric ID of the RTP extension + * @param[in] extmap Namespace of the RTP extension + * @returns 0 in case of success, a negative integer otherwise */ +int janus_recorder_add_extmap(janus_recorder *recorder, int id, const char *extmap); /*! \brief Mark this recorder as end-to-end encrypted (e.g., via Insertable Streams) * \note This will only be possible BEFORE the first frame is written, as it needs to * be reflected in the .mjr header: doing this after that will return an error. Also