From e7705b33b60307e96143de048ba1385feec5396d Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Mon, 23 Nov 2020 11:14:15 +0100 Subject: [PATCH 1/7] Spatial audio support in AudioBridge via stereo mixing (WIP) --- conf/janus.plugin.audiobridge.jcfg.sample | 1 + html/audiobridgetest.js | 11 +- plugins/janus_audiobridge.c | 175 +++++++++++++++++----- 3 files changed, 152 insertions(+), 35 deletions(-) diff --git a/conf/janus.plugin.audiobridge.jcfg.sample b/conf/janus.plugin.audiobridge.jcfg.sample index c5f2359ddb..faf440bd2f 100644 --- a/conf/janus.plugin.audiobridge.jcfg.sample +++ b/conf/janus.plugin.audiobridge.jcfg.sample @@ -4,6 +4,7 @@ # secret = "" # pin = "" # sampling_rate = (e.g., 16000 for wideband mixing) +# spatial_audio = true|false (if true, the mix will be stereo to spatially place users, default=false) # audiolevel_ext = true|false (whether the ssrc-audio-level RTP extension must # be negotiated/used or not for new joins, default=true) # audiolevel_event = true|false (whether to emit event to other users or not, default=false) diff --git a/html/audiobridgetest.js b/html/audiobridgetest.js index 6d0de4f7c5..be3e12ff3c 100644 --- a/html/audiobridgetest.js +++ b/html/audiobridgetest.js @@ -57,6 +57,9 @@ var spinner = null; var myroom = 1234; // Demo room if(getQueryStringValue("room") !== "") myroom = parseInt(getQueryStringValue("room")); +var stereo = false; +if(getQueryStringValue("stereo") !== "") + stereo = (getQueryStringValue("stereo") === "true"); var myusername = null; var myid = null; var webrtcUp = false; @@ -146,7 +149,13 @@ $(document).ready(function() { // Publish our stream mixertest.createOffer( { - media: { video: false}, // This is an audio only room + media: { video: false }, // This is an audio only room + customizeSdp: function(jsep) { + if(stereo && jsep.sdp.indexOf("stereo=1") == -1) { + // Make sure that our offer contains stereo too + jsep.sdp = jsep.sdp.replace("useinbandfec=1", "useinbandfec=1;stereo=1"); + } + }, success: function(jsep) { Janus.debug("Got SDP!", jsep); var publish = { request: "configure", muted: false }; diff --git a/plugins/janus_audiobridge.c b/plugins/janus_audiobridge.c index 2ef028321c..7680926c79 100644 --- a/plugins/janus_audiobridge.c +++ b/plugins/janus_audiobridge.c @@ -32,6 +32,7 @@ room-: { secret = pin = sampling_rate = (e.g., 16000 for wideband mixing) + spatial_audio = true|false (if true, the mix will be stereo to spatially place users, default=false) audiolevel_ext = true|false (whether the ssrc-audio-level RTP extension must be negotiated/used or not for new joins, default=true) audiolevel_event = true|false (whether to emit event to other users or not, default=false) @@ -124,6 +125,7 @@ room-: { "is_private" : , "allowed" : [ array of string tokens users can use to join this room, optional], "sampling_rate" : , + "spatial_audio" : , "audiolevel_ext" : , "audiolevel_event" : , "audio_active_packets" : , @@ -321,6 +323,7 @@ room-: { "description" : "", "pin_required" : , "sampling_rate" : , + "spatial_audio" : , "record" : , "num_participants" : }, @@ -354,6 +357,7 @@ room-: { "setup" : , "muted" : , "talking" : , + "spatial_position" : , }, // Other participants ] @@ -642,6 +646,7 @@ room-: { "prebuffer" : , "quality" : <0-10, Opus-related complexity to use, the higher the value, the better the quality (but more CPU); optional, default is 4>, "volume" : 100 increases volume; optional, default is 100 (no volume change)>, + "spatial_position" : , "secret" : "", "audio_level_average" : "", "audio_active_packets" : "" @@ -680,6 +685,7 @@ room-: { "prebuffer" : , "quality" : , "volume" : , + "spatial_position" : , "record": " } @@ -981,6 +987,7 @@ static struct janus_json_parameter create_parameters[] = { {"allowed", JSON_ARRAY, 0}, {"sampling_rate", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"sampling", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, /* We keep this to be backwards compatible */ + {"spatial_audio", JANUS_JSON_BOOL, 0}, {"record", JANUS_JSON_BOOL, 0}, {"record_file", JSON_STRING, 0}, {"permanent", JANUS_JSON_BOOL, 0}, @@ -1017,6 +1024,7 @@ static struct janus_json_parameter join_parameters[] = { {"prebuffer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"quality", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"volume", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, + {"spatial_position", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"audio_level_average", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"audio_active_packets", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"generate_offer", JANUS_JSON_BOOL, 0}, @@ -1027,6 +1035,7 @@ static struct janus_json_parameter configure_parameters[] = { {"prebuffer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"quality", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"volume", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, + {"spatial_position", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"record", JANUS_JSON_BOOL, 0}, {"filename", JSON_STRING, 0}, {"display", JSON_STRING, 0}, @@ -1096,6 +1105,7 @@ typedef struct janus_audiobridge_room { uint32_t room_ssrc; /* SSRC we'll use for packets generated by the mixer */ gboolean is_private; /* Whether this room is 'private' (as in hidden) or not */ uint32_t sampling_rate; /* Sampling rate of the mix (e.g., 16000 for wideband; can be 8, 12, 16, 24 or 48kHz) */ + gboolean spatial_audio; /* Whether the mix will use spatial audio, using stereo */ gboolean audiolevel_ext; /* Whether the ssrc-audio-level extension must be negotiated or not for new joins */ gboolean audiolevel_event; /* Whether to emit event to other users about audiolevel */ uint default_prebuffering; /* Number of packets to buffer before decoding each participant */ @@ -1308,6 +1318,8 @@ typedef struct janus_audiobridge_participant { gboolean muted; /* Whether this participant is muted */ int volume_gain; /* Gain to apply to the input audio (in percentage) */ int opus_complexity; /* Complexity to use in the encoder (by default, DEFAULT_COMPLEXITY) */ + gboolean stereo; /* Whether stereo will be used for spatial audio */ + int spatial_position; /* Panning of this participant in the mix */ /* RTP stuff */ GList *inbuf; /* Incoming audio from this participant, as an ordered list of packets */ GAsyncQueue *outbuf; /* Mixed audio for this participant */ @@ -1798,7 +1810,7 @@ static int janus_audiobridge_resample(int16_t *input, int input_num, int input_r /* Opus settings */ #define OPUS_SAMPLES 960 #define G711_SAMPLES 160 -#define BUFFER_SAMPLES OPUS_SAMPLES*6 +#define BUFFER_SAMPLES OPUS_SAMPLES*12 #define DEFAULT_COMPLEXITY 4 @@ -1844,7 +1856,8 @@ static int janus_audiobridge_create_opus_encoder_if_needed(janus_audiobridge_roo } int error = 0; - audiobridge->rtp_encoder = opus_encoder_create(audiobridge->sampling_rate, 1, OPUS_APPLICATION_VOIP, &error); + audiobridge->rtp_encoder = opus_encoder_create(audiobridge->sampling_rate, + audiobridge->spatial_audio ? 2 : 1, OPUS_APPLICATION_VOIP, &error); if(error != OPUS_OK) { JANUS_LOG(LOG_ERR, "Error creating Opus encoder for RTP forwarder (room %s)\n", audiobridge->room_id_str); return -1; @@ -2086,6 +2099,7 @@ int janus_audiobridge_init(janus_callbacks *callback, const char *config_path) { janus_config_item *desc = janus_config_get(config, cat, janus_config_type_item, "description"); janus_config_item *priv = janus_config_get(config, cat, janus_config_type_item, "is_private"); janus_config_item *sampling = janus_config_get(config, cat, janus_config_type_item, "sampling_rate"); + janus_config_item *spatial = janus_config_get(config, cat, janus_config_type_item, "spatial_audio"); janus_config_item *audiolevel_ext = janus_config_get(config, cat, janus_config_type_item, "audiolevel_ext"); janus_config_item *audiolevel_event = janus_config_get(config, cat, janus_config_type_item, "audiolevel_event"); janus_config_item *audio_active_packets = janus_config_get(config, cat, janus_config_type_item, "audio_active_packets"); @@ -2156,6 +2170,7 @@ int janus_audiobridge_init(janus_callbacks *callback, const char *config_path) { cl = cl->next; continue; } + audiobridge->spatial_audio = spatial && spatial->value && janus_is_true(spatial->value); audiobridge->audiolevel_ext = TRUE; if(audiolevel_ext != NULL && audiolevel_ext->value != NULL) audiobridge->audiolevel_ext = janus_is_true(audiolevel_ext->value); @@ -2446,6 +2461,8 @@ json_t *janus_audiobridge_query_session(janus_plugin_session *handle) { json_object_set_new(info, "queue-out", json_integer(g_async_queue_length(participant->outbuf))); if(participant->last_drop > 0) json_object_set_new(info, "last-drop", json_integer(participant->last_drop)); + if(participant->stereo) + json_object_set_new(info, "spatial_position", json_integer(participant->spatial_position)); if(participant->arc && participant->arc->filename) json_object_set_new(info, "audio-recording", json_string(participant->arc->filename)); if(participant->extmap_id > 0) { @@ -2513,6 +2530,7 @@ static json_t *janus_audiobridge_process_synchronous_request(janus_audiobridge_s json_t *sampling = json_object_get(root, "sampling_rate"); if(sampling == NULL) sampling = json_object_get(root, "sampling"); + json_t *spatial = json_object_get(root, "spatial_audio"); json_t *audiolevel_ext = json_object_get(root, "audiolevel_ext"); json_t *audiolevel_event = json_object_get(root, "audiolevel_event"); json_t *audio_active_packets = json_object_get(root, "audio_active_packets"); @@ -2617,6 +2635,7 @@ static json_t *janus_audiobridge_process_synchronous_request(janus_audiobridge_s audiobridge->sampling_rate = json_integer_value(sampling); else audiobridge->sampling_rate = 16000; + audiobridge->spatial_audio = spatial ? json_is_true(spatial) : FALSE; audiobridge->audiolevel_ext = audiolevel_ext ? json_is_true(audiolevel_ext) : TRUE; audiobridge->audiolevel_event = audiolevel_event ? json_is_true(audiolevel_event) : FALSE; if(audiobridge->audiolevel_event) { @@ -3086,6 +3105,7 @@ static json_t *janus_audiobridge_process_synchronous_request(janus_audiobridge_s json_object_set_new(rl, "room", string_ids ? json_string(room->room_id_str) : json_integer(room->room_id)); json_object_set_new(rl, "description", json_string(room->room_name)); json_object_set_new(rl, "sampling_rate", json_integer(room->sampling_rate)); + json_object_set_new(rl, "spatial_audio", room->spatial_audio ? json_true() : json_false()); json_object_set_new(rl, "pin_required", room->room_pin ? json_true() : json_false()); json_object_set_new(rl, "record", room->record ? json_true() : json_false()); json_object_set_new(rl, "muted", room->muted ? json_true() : json_false()); @@ -4176,7 +4196,10 @@ static json_t *janus_audiobridge_process_synchronous_request(janus_audiobridge_s p->annc->loop = json_is_true(json_object_get(root, "loop")); /* Setup the opus decoder */ int opuserror = 0; - p->decoder = opus_decoder_create(audiobridge->sampling_rate, 1, &opuserror); + p->stereo = audiobridge->spatial_audio; + p->spatial_position = 64; + p->decoder = opus_decoder_create(audiobridge->sampling_rate, + audiobridge->spatial_audio ? 2 : 1, &opuserror); if(opuserror != OPUS_OK) { janus_mutex_unlock(&audiobridge->mutex); janus_mutex_unlock(&rooms_mutex); @@ -4638,17 +4661,18 @@ void janus_audiobridge_incoming_rtp(janus_plugin_session *handle, janus_plugin_r return; if(participant->room && participant->room->muted && !participant->admin) return; - char *buf = packet->buffer; uint16_t len = packet->length; /* - Save the frame if we're recording this leg */ + char *buf = packet->buffer; + uint16_t len = packet->length; + /* Save the frame if we're recording this leg */ janus_recorder_save_frame(participant->arc, buf, len); - if(g_atomic_int_get(&participant->active) && (participant->codec != - JANUS_AUDIOCODEC_OPUS || + if(g_atomic_int_get(&participant->active) && (participant->codec != JANUS_AUDIOCODEC_OPUS || (participant->codec == JANUS_AUDIOCODEC_OPUS && participant->decoder))) { /* First of all, check if a reset on the decoder is due */ if(participant->reset && participant->codec == JANUS_AUDIOCODEC_OPUS) { /* Create a new decoder and get rid of the old one */ int error = 0; - OpusDecoder *decoder = opus_decoder_create(participant->room->sampling_rate, 1, &error); + OpusDecoder *decoder = opus_decoder_create(participant->room->sampling_rate, + participant->stereo ? 2 : 1, &error); if(error != OPUS_OK) { JANUS_LOG(LOG_ERR, "Error resetting Opus decoder...\n"); } else { @@ -5236,6 +5260,7 @@ static void *janus_audiobridge_handler(void *data) { json_t *muted = json_object_get(root, "muted"); json_t *prebuffer = json_object_get(root, "prebuffer"); json_t *gain = json_object_get(root, "volume"); + json_t *spatial = json_object_get(root, "spatial_position"); json_t *quality = json_object_get(root, "quality"); json_t *acodec = json_object_get(root, "codec"); json_t *user_audio_level_average = json_object_get(root, "audio_level_average"); @@ -5248,6 +5273,7 @@ static void *janus_audiobridge_handler(void *data) { audiobridge->default_prebuffering); } int volume = gain ? json_integer_value(gain) : 100; + int spatial_position = spatial ? json_integer_value(spatial) : 64; int complexity = quality ? json_integer_value(quality) : DEFAULT_COMPLEXITY; if(complexity < 1 || complexity > 10) { janus_mutex_unlock(&audiobridge->mutex); @@ -5352,6 +5378,12 @@ static void *janus_audiobridge_handler(void *data) { participant->prebuffer_count = prebuffer_count; participant->volume_gain = volume; participant->opus_complexity = complexity; + participant->stereo = audiobridge->spatial_audio; + participant->spatial_position = spatial_position; + if(participant->spatial_position < 0) + participant->spatial_position = 0; + else if(participant->spatial_position > 100) + participant->spatial_position = 100; participant->user_audio_active_packets = json_integer_value(user_audio_active_packets); participant->user_audio_level_average = json_integer_value(user_audio_level_average); if(participant->outbuf == NULL) @@ -5369,7 +5401,8 @@ static void *janus_audiobridge_handler(void *data) { /* Opus encoder */ int error = 0; if(participant->encoder == NULL) { - participant->encoder = opus_encoder_create(audiobridge->sampling_rate, 1, OPUS_APPLICATION_VOIP, &error); + participant->encoder = opus_encoder_create(audiobridge->sampling_rate, + audiobridge->spatial_audio ? 2 : 1, OPUS_APPLICATION_VOIP, &error); if(error != OPUS_OK) { if(user_id_allocated) g_free(user_id_str); @@ -5403,7 +5436,8 @@ static void *janus_audiobridge_handler(void *data) { if(participant->decoder == NULL) { /* Opus decoder */ error = 0; - participant->decoder = opus_decoder_create(audiobridge->sampling_rate, 1, &error); + participant->decoder = opus_decoder_create(audiobridge->sampling_rate, + audiobridge->spatial_audio ? 2 : 1, &error); if(error != OPUS_OK) { if(user_id_allocated) g_free(user_id_str); @@ -5540,6 +5574,7 @@ static void *janus_audiobridge_handler(void *data) { json_t *prebuffer = json_object_get(root, "prebuffer"); json_t *quality = json_object_get(root, "quality"); json_t *gain = json_object_get(root, "volume"); + json_t *spatial = json_object_get(root, "spatial_position"); json_t *record = json_object_get(root, "record"); json_t *recfile = json_object_get(root, "filename"); json_t *display = json_object_get(root, "display"); @@ -5573,6 +5608,13 @@ static void *janus_audiobridge_handler(void *data) { } if(gain) participant->volume_gain = json_integer_value(gain); + if(spatial) { + participant->spatial_position = json_integer_value(spatial); + if(participant->spatial_position < 0) + participant->spatial_position = 0; + else if(participant->spatial_position > 100) + participant->spatial_position = 100; + } if(quality) { int complexity = json_integer_value(quality); if(complexity < 1 || complexity > 10) { @@ -5821,8 +5863,10 @@ static void *janus_audiobridge_handler(void *data) { const char *display_text = display ? json_string_value(display) : NULL; json_t *muted = json_object_get(root, "muted"); json_t *gain = json_object_get(root, "volume"); + json_t *spatial = json_object_get(root, "spatial_position"); json_t *quality = json_object_get(root, "quality"); int volume = gain ? json_integer_value(gain) : 100; + int spatial_position = spatial ? json_integer_value(spatial) : 64; int complexity = quality ? json_integer_value(quality) : DEFAULT_COMPLEXITY; if(complexity < 1 || complexity > 10) { janus_mutex_unlock(&audiobridge->mutex); @@ -5896,10 +5940,14 @@ static void *janus_audiobridge_handler(void *data) { janus_mutex_lock(&old_audiobridge->mutex); g_hash_table_remove(old_audiobridge->participants, string_ids ? (gpointer)participant->user_id_str : (gpointer)&participant->user_id); - if(old_audiobridge->sampling_rate != audiobridge->sampling_rate) { + if(old_audiobridge->sampling_rate != audiobridge->sampling_rate || + old_audiobridge->spatial_audio != audiobridge->spatial_audio) { /* Create a new one that takes into account the sampling rate we want now */ + participant->stereo = audiobridge->spatial_audio; + participant->spatial_position = 64; int error = 0; - OpusEncoder *new_encoder = opus_encoder_create(audiobridge->sampling_rate, 1, OPUS_APPLICATION_VOIP, &error); + OpusEncoder *new_encoder = opus_encoder_create(audiobridge->sampling_rate, + audiobridge->spatial_audio ? 2 : 1, OPUS_APPLICATION_VOIP, &error); if(error != OPUS_OK) { if(user_id_allocated) g_free(user_id_str); @@ -5938,7 +5986,8 @@ static void *janus_audiobridge_handler(void *data) { opus_encoder_ctl(new_encoder, OPUS_SET_COMPLEXITY(participant->opus_complexity)); /* Opus decoder */ error = 0; - OpusDecoder *new_decoder = opus_decoder_create(audiobridge->sampling_rate, 1, &error); + OpusDecoder *new_decoder = opus_decoder_create(audiobridge->sampling_rate, + audiobridge->spatial_audio ? 2 : 1, &error); if(error != OPUS_OK) { if(user_id_allocated) g_free(user_id_str); @@ -6020,6 +6069,12 @@ static void *janus_audiobridge_handler(void *data) { participant->audio_dBov_sum = 0; participant->talking = FALSE; participant->volume_gain = volume; + participant->stereo = audiobridge->spatial_audio; + participant->spatial_position = spatial_position; + if(participant->spatial_position < 0) + participant->spatial_position = 0; + else if(participant->spatial_position > 100) + participant->spatial_position = 100; if(quality) { participant->opus_complexity = complexity; if(participant->encoder) @@ -6309,8 +6364,9 @@ static void *janus_audiobridge_handler(void *data) { g_snprintf(s_name, sizeof(s_name), "AudioBridge %s", participant->room->room_id_str); /* Prepare a fmtp string too */ char fmtp[100]; - g_snprintf(fmtp, sizeof(fmtp), "%d maxplaybackrate=%"SCNu32"; stereo=0; sprop-stereo=0; useinbandfec=%d\r\n", - participant->opus_pt, participant->room->sampling_rate, participant->fec ? 1 : 0); + g_snprintf(fmtp, sizeof(fmtp), "%d maxplaybackrate=%"SCNu32"; stereo=%d; sprop-stereo=%d; useinbandfec=%d\r\n", + participant->opus_pt, participant->room->sampling_rate, + participant->stereo ? 1 : 0, participant->stereo ? 1 : 0, participant->fec ? 1 : 0); /* If we got an offer, we need to answer */ janus_sdp *offer = NULL, *answer = NULL; if(got_offer) { @@ -6443,7 +6499,7 @@ static void *janus_audiobridge_mixer_thread(void *data) { {'f', 'm', 't', ' '}, 16, 1, - 1, + audiobridge->spatial_audio ? 2 : 1, audiobridge->sampling_rate, audiobridge->sampling_rate * 2, 2, @@ -6461,12 +6517,16 @@ static void *janus_audiobridge_mixer_thread(void *data) { /* Buffer (we allocate assuming 48kHz, although we'll likely use less than that) */ int samples = audiobridge->sampling_rate/50; - opus_int32 buffer[OPUS_SAMPLES], sumBuffer[OPUS_SAMPLES]; - opus_int16 outBuffer[OPUS_SAMPLES], resampled[OPUS_SAMPLES], *curBuffer = NULL; - memset(buffer, 0, OPUS_SAMPLES*4); - memset(sumBuffer, 0, OPUS_SAMPLES*4); - memset(outBuffer, 0, OPUS_SAMPLES*2); - memset(resampled, 0, OPUS_SAMPLES*2); + if(audiobridge->spatial_audio) + samples = samples*2; + opus_int32 buffer[audiobridge->spatial_audio ? OPUS_SAMPLES*2 : OPUS_SAMPLES], + sumBuffer[audiobridge->spatial_audio ? OPUS_SAMPLES*2 : OPUS_SAMPLES]; + opus_int16 outBuffer[audiobridge->spatial_audio ? OPUS_SAMPLES*2 : OPUS_SAMPLES], + resampled[audiobridge->spatial_audio ? OPUS_SAMPLES*2 : OPUS_SAMPLES], *curBuffer = NULL; + memset(buffer, 0, OPUS_SAMPLES*(audiobridge->spatial_audio ? 8 : 4)); + memset(sumBuffer, 0, OPUS_SAMPLES*(audiobridge->spatial_audio ? 8 : 4)); + memset(outBuffer, 0, OPUS_SAMPLES*(audiobridge->spatial_audio ? 4 : 2)); + memset(resampled, 0, OPUS_SAMPLES*(audiobridge->spatial_audio ? 4 : 2)); /* Base RTP packet, in case there are forwarders involved */ unsigned char *rtpbuffer = g_malloc0(1500); @@ -6491,6 +6551,7 @@ static void *janus_audiobridge_mixer_thread(void *data) { /* Loop */ int i=0; int count = 0, rf_count = 0, pf_count = 0, prev_count = 0; + int lgain = 0, rgain = 0, diff = 0; while(!g_atomic_int_get(&stopping) && !g_atomic_int_get(&audiobridge->destroyed)) { /* See if it's time to prepare a frame */ gettimeofday(&now, NULL); @@ -6568,11 +6629,33 @@ static void *janus_audiobridge_mixer_thread(void *data) { memcpy((opus_int16 *)pkt->data, resampled, pkt->length); } curBuffer = (opus_int16 *)pkt->data; - for(i=0; ivolume_gain == 100) { - buffer[i] += curBuffer[i]; - } else { - buffer[i] += (curBuffer[i]*p->volume_gain)/100; + if(!p->stereo) { + for(i=0; ivolume_gain == 100) { + buffer[i] += curBuffer[i]; + } else { + buffer[i] += (curBuffer[i]*p->volume_gain)/100; + } + } + } else { + /* TODO Volume gain */ + diff = 50 - p->spatial_position; + lgain = 50 + diff; + rgain = 50 - diff; + for(i=0; iqmutex); curBuffer = (opus_int16 *)((pkt && pkt->length && !pkt->silence) ? pkt->data : NULL); - for(i=0; ivolume_gain == 100) - sumBuffer[i] = buffer[i] - (curBuffer ? (curBuffer[i]) : 0); - else - sumBuffer[i] = buffer[i] - (curBuffer ? (curBuffer[i]*p->volume_gain)/100 : 0); + if(!p->stereo) { + for(i=0; ivolume_gain == 100) + sumBuffer[i] = buffer[i] - (curBuffer ? (curBuffer[i]) : 0); + else + sumBuffer[i] = buffer[i] - (curBuffer ? (curBuffer[i]*p->volume_gain)/100 : 0); + } + } else { + /* TODO Volume gain */ + diff = 50 - p->spatial_position; + lgain = 50 + diff; + rgain = 50 - diff; + for(i=0; icodec == JANUS_AUDIOCODEC_OPUS && !have_opus) { /* This is an Opus forwarder and we don't have a version for that yet */ - length = opus_encode(audiobridge->rtp_encoder, outBuffer, samples, rtpbuffer+12, 1500-12); + length = opus_encode(audiobridge->rtp_encoder, outBuffer, + audiobridge->spatial_audio ? samples/2 : samples, rtpbuffer+12, 1500-12); if(length < 0) { JANUS_LOG(LOG_ERR, "[Opus] Ops! got an error encoding the Opus frame: %d (%s)\n", length, opus_strerror(length)); continue; @@ -6963,7 +7069,8 @@ static void *janus_audiobridge_participant_thread(void *data) { g_atomic_int_compare_and_exchange(&participant->encoding, 0, 1)) { /* Encode raw frame to Opus */ opus_int16 *outBuffer = (opus_int16 *)mixedpkt->data; - outpkt->length = opus_encode(participant->encoder, outBuffer, mixedpkt->length, payload+12, 1500-12); + outpkt->length = opus_encode(participant->encoder, outBuffer, + participant->stereo ? mixedpkt->length/2 : mixedpkt->length, payload+12, 1500-12); g_atomic_int_set(&participant->encoding, 0); if(outpkt->length < 0) { JANUS_LOG(LOG_ERR, "[Opus] Ops! got an error encoding the Opus frame: %d (%s)\n", outpkt->length, opus_strerror(outpkt->length)); From 6bee09c4d70f4765a0b8bfb8c729602cc8b5d842 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Tue, 4 May 2021 14:35:41 +0200 Subject: [PATCH 2/7] Save spatial_audio property to config file, when creating permanent rooms --- plugins/janus_audiobridge.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/janus_audiobridge.c b/plugins/janus_audiobridge.c index cc600015e7..ee6e2e68c8 100644 --- a/plugins/janus_audiobridge.c +++ b/plugins/janus_audiobridge.c @@ -2917,6 +2917,8 @@ static json_t *janus_audiobridge_process_synchronous_request(janus_audiobridge_s janus_config_add(config, c, janus_config_item_create("record", "yes")); janus_config_add(config, c, janus_config_item_create("record_file", audiobridge->record_file)); } + if(audiobridge->spatial_audio) + janus_config_add(config, c, janus_config_item_create("spatial_audio", "yes")); /* Save modified configuration */ if(janus_config_save(config, config_folder, JANUS_AUDIOBRIDGE_PACKAGE) < 0) save = FALSE; /* This will notify the user the room is not permanent */ @@ -3059,6 +3061,8 @@ static json_t *janus_audiobridge_process_synchronous_request(janus_audiobridge_s janus_config_add(config, c, janus_config_item_create("record", "yes")); janus_config_add(config, c, janus_config_item_create("record_file", audiobridge->record_file)); } + if(audiobridge->spatial_audio) + janus_config_add(config, c, janus_config_item_create("spatial_audio", "yes")); /* Save modified configuration */ if(janus_config_save(config, config_folder, JANUS_AUDIOBRIDGE_PACKAGE) < 0) save = FALSE; /* This will notify the user the room changes are not permanent */ From e8702ddce9e32df880b09fbb90d76b8b1b2bdde0 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Tue, 4 May 2021 14:40:32 +0200 Subject: [PATCH 3/7] Take global participant volume into account, when mixing stereo --- plugins/janus_audiobridge.c | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/plugins/janus_audiobridge.c b/plugins/janus_audiobridge.c index ee6e2e68c8..ba2dcedd23 100644 --- a/plugins/janus_audiobridge.c +++ b/plugins/janus_audiobridge.c @@ -6922,22 +6922,37 @@ static void *janus_audiobridge_mixer_thread(void *data) { } } } else { - /* TODO Volume gain */ diff = 50 - p->spatial_position; lgain = 50 + diff; rgain = 50 - diff; for(i=0; ivolume_gain == 100) { + buffer[i] += curBuffer[i]; + } else { + buffer[i] += (curBuffer[i]*p->volume_gain)/100; + } } else { - buffer[i] += (curBuffer[i]*lgain)/100; + if(p->volume_gain == 100) { + buffer[i] += (curBuffer[i]*lgain)/100; + } else { + buffer[i] += (((curBuffer[i]*lgain)/100)*p->volume_gain)/100; + } } } else { if(rgain == 100) { - buffer[i] += curBuffer[i]; + if(p->volume_gain == 100) { + buffer[i] += curBuffer[i]; + } else { + buffer[i] += (curBuffer[i]*p->volume_gain)/100; + } } else { - buffer[i] += (curBuffer[i]*rgain)/100; + if(p->volume_gain == 100) { + buffer[i] += (curBuffer[i]*rgain)/100; + } else { + buffer[i] += (((curBuffer[i]*rgain)/100)*p->volume_gain)/100; + } } } } From 30f220f81b75ab52edb344b979d16c32b9c2c590 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Tue, 4 May 2021 14:46:56 +0200 Subject: [PATCH 4/7] Add spatial position of participants to room events --- plugins/janus_audiobridge.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/plugins/janus_audiobridge.c b/plugins/janus_audiobridge.c index ba2dcedd23..86521b78e2 100644 --- a/plugins/janus_audiobridge.c +++ b/plugins/janus_audiobridge.c @@ -3544,6 +3544,8 @@ static json_t *janus_audiobridge_process_synchronous_request(janus_audiobridge_s json_object_set_new(pl, "display", json_string(participant->display)); json_object_set_new(pl, "setup", g_atomic_int_get(&participant->session->started) ? json_true() : json_false()); json_object_set_new(pl, "muted", participant->muted ? json_true() : json_false()); + if(audiobridge->spatial_audio) + json_object_set_new(pl, "spatial_position", json_integer(participant->spatial_position)); json_array_append_new(list, pl); json_t *pub = json_object(); json_object_set_new(pub, "audiobridge", json_string("event")); @@ -3846,6 +3848,8 @@ static json_t *janus_audiobridge_process_synchronous_request(janus_audiobridge_s json_object_set_new(pl, "muted", p->muted ? json_true() : json_false()); if(p->extmap_id > 0) json_object_set_new(pl, "talking", p->talking ? json_true() : json_false()); + if(audiobridge->spatial_audio) + json_object_set_new(pl, "spatial_position", json_integer(p->spatial_position)); json_array_append_new(list, pl); } janus_refcount_decrease(&audiobridge->ref); @@ -4775,6 +4779,8 @@ void janus_audiobridge_setup_media(janus_plugin_session *handle) { json_object_set_new(pl, "display", json_string(participant->display)); json_object_set_new(pl, "setup", json_true()); json_object_set_new(pl, "muted", participant->muted ? json_true() : json_false()); + if(audiobridge->spatial_audio) + json_object_set_new(pl, "spatial_position", json_integer(participant->spatial_position)); json_array_append_new(list, pl); json_t *pub = json_object(); json_object_set_new(pub, "audiobridge", json_string("event")); @@ -5762,6 +5768,8 @@ static void *janus_audiobridge_handler(void *data) { /* Clarify we're still waiting for the user to negotiate a PeerConnection */ json_object_set_new(pl, "setup", json_false()); json_object_set_new(pl, "muted", participant->muted ? json_true() : json_false()); + if(audiobridge->spatial_audio) + json_object_set_new(pl, "spatial_position", json_integer(participant->spatial_position)); json_array_append_new(newuserlist, pl); json_object_set_new(newuser, "participants", newuserlist); GHashTableIter iter; @@ -5793,6 +5801,8 @@ static void *janus_audiobridge_handler(void *data) { json_object_set_new(pl, "muted", p->muted ? json_true() : json_false()); if(p->extmap_id > 0) json_object_set_new(pl, "talking", p->talking ? json_true() : json_false()); + if(audiobridge->spatial_audio) + json_object_set_new(pl, "spatial_position", json_integer(p->spatial_position)); json_array_append_new(list, pl); } janus_mutex_unlock(&audiobridge->mutex); @@ -5816,6 +5826,8 @@ static void *janus_audiobridge_handler(void *data) { json_object_set_new(info, "display", json_string(participant->display)); json_object_set_new(info, "setup", g_atomic_int_get(&participant->session->started) ? json_true() : json_false()); json_object_set_new(info, "muted", participant->muted ? json_true() : json_false()); + if(participant->stereo) + json_object_set_new(info, "spatial_position", json_integer(participant->spatial_position)); gateway->notify_event(&janus_audiobridge_plugin, session->handle, info); } if(user_id_allocated) @@ -5942,6 +5954,8 @@ static void *janus_audiobridge_handler(void *data) { json_object_set_new(pl, "display", json_string(participant->display)); json_object_set_new(pl, "setup", g_atomic_int_get(&participant->session->started) ? json_true() : json_false()); json_object_set_new(pl, "muted", participant->muted ? json_true() : json_false()); + if(audiobridge->spatial_audio) + json_object_set_new(pl, "spatial_position", json_integer(participant->spatial_position)); json_array_append_new(list, pl); json_t *pub = json_object(); json_object_set_new(pub, "audiobridge", json_string("event")); @@ -6025,6 +6039,8 @@ static void *janus_audiobridge_handler(void *data) { json_object_set_new(info, "display", json_string(participant->display)); json_object_set_new(info, "muted", participant->muted ? json_true() : json_false()); json_object_set_new(info, "quality", json_integer(participant->opus_complexity)); + if(participant->stereo) + json_object_set_new(info, "spatial_position", json_integer(participant->spatial_position)); gateway->notify_event(&janus_audiobridge_plugin, session->handle, info); } /* If we need to generate an offer ourselves, do that */ @@ -6367,6 +6383,8 @@ static void *janus_audiobridge_handler(void *data) { json_object_set_new(pl, "display", json_string(participant->display)); json_object_set_new(pl, "setup", g_atomic_int_get(&participant->session->started) ? json_true() : json_false()); json_object_set_new(pl, "muted", participant->muted ? json_true() : json_false()); + if(audiobridge->spatial_audio) + json_object_set_new(pl, "spatial_position", json_integer(participant->spatial_position)); json_array_append_new(newuserlist, pl); json_object_set_new(newuser, "participants", newuserlist); g_hash_table_iter_init(&iter, audiobridge->participants); @@ -6396,6 +6414,8 @@ static void *janus_audiobridge_handler(void *data) { json_object_set_new(pl, "muted", p->muted ? json_true() : json_false()); if(p->extmap_id > 0) json_object_set_new(pl, "talking", p->talking ? json_true() : json_false()); + if(audiobridge->spatial_audio) + json_object_set_new(pl, "spatial_position", json_integer(p->spatial_position)); json_array_append_new(list, pl); } event = json_object(); @@ -6414,6 +6434,8 @@ static void *janus_audiobridge_handler(void *data) { string_ids ? json_string(participant->user_id_str) : json_integer(participant->user_id)); json_object_set_new(info, "display", json_string(participant->display)); json_object_set_new(info, "muted", participant->muted ? json_true() : json_false()); + if(participant->stereo) + json_object_set_new(info, "spatial_position", json_integer(participant->spatial_position)); gateway->notify_event(&janus_audiobridge_plugin, session->handle, info); } if(user_id_allocated) From 9d33de82c87e1ecd1d4f8192841045a60a2cc40c Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Tue, 4 May 2021 16:43:34 +0200 Subject: [PATCH 5/7] Fixed typo on default position value --- plugins/janus_audiobridge.c | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/plugins/janus_audiobridge.c b/plugins/janus_audiobridge.c index 86521b78e2..c65b923c40 100644 --- a/plugins/janus_audiobridge.c +++ b/plugins/janus_audiobridge.c @@ -4347,7 +4347,7 @@ static json_t *janus_audiobridge_process_synchronous_request(janus_audiobridge_s /* Setup the opus decoder */ int opuserror = 0; p->stereo = audiobridge->spatial_audio; - p->spatial_position = 64; + p->spatial_position = 50; p->decoder = opus_decoder_create(audiobridge->sampling_rate, audiobridge->spatial_audio ? 2 : 1, &opuserror); if(opuserror != OPUS_OK) { @@ -5461,7 +5461,7 @@ static void *janus_audiobridge_handler(void *data) { audiobridge->default_prebuffering); } int volume = gain ? json_integer_value(gain) : 100; - int spatial_position = spatial ? json_integer_value(spatial) : 64; + int spatial_position = spatial ? json_integer_value(spatial) : 50; int complexity = quality ? json_integer_value(quality) : DEFAULT_COMPLEXITY; if(complexity < 1 || complexity > 10) { janus_mutex_unlock(&audiobridge->mutex); @@ -5569,11 +5569,11 @@ static void *janus_audiobridge_handler(void *data) { participant->volume_gain = volume; participant->opus_complexity = complexity; participant->stereo = audiobridge->spatial_audio; - participant->spatial_position = spatial_position; - if(participant->spatial_position < 0) - participant->spatial_position = 0; - else if(participant->spatial_position > 100) - participant->spatial_position = 100; + if(participant->stereo) { + if(spatial_position > 100) + spatial_position = 100; + participant->spatial_position = spatial_position; + } participant->user_audio_active_packets = json_integer_value(user_audio_active_packets); participant->user_audio_level_average = json_integer_value(user_audio_level_average); if(participant->outbuf == NULL) @@ -5890,13 +5890,6 @@ static void *janus_audiobridge_handler(void *data) { } if(gain) participant->volume_gain = json_integer_value(gain); - if(spatial) { - participant->spatial_position = json_integer_value(spatial); - if(participant->spatial_position < 0) - participant->spatial_position = 0; - else if(participant->spatial_position > 100) - participant->spatial_position = 100; - } if(quality) { int complexity = json_integer_value(quality); if(complexity < 1 || complexity > 10) { @@ -5909,7 +5902,7 @@ static void *janus_audiobridge_handler(void *data) { if(participant->encoder) opus_encoder_ctl(participant->encoder, OPUS_SET_COMPLEXITY(participant->opus_complexity)); } - if(muted || display) { + if(muted || display || (participant->stereo && spatial)) { if(muted) { participant->muted = json_is_true(muted); JANUS_LOG(LOG_VERB, "Setting muted property: %s (room %s, user %s)\n", @@ -5941,6 +5934,12 @@ static void *janus_audiobridge_handler(void *data) { JANUS_LOG(LOG_VERB, "Setting display property: %s (room %s, user %s)\n", participant->display, participant->room->room_id_str, participant->user_id_str); } + if(participant->stereo && spatial) { + int spatial_position = json_integer_value(spatial); + if(spatial_position > 100) + spatial_position = 100; + participant->spatial_position = spatial_position; + } /* Notify all other participants about the mute/unmute */ janus_mutex_lock(&rooms_mutex); janus_audiobridge_room *audiobridge = participant->room; @@ -6231,7 +6230,7 @@ static void *janus_audiobridge_handler(void *data) { old_audiobridge->spatial_audio != audiobridge->spatial_audio) { /* Create a new one that takes into account the sampling rate we want now */ participant->stereo = audiobridge->spatial_audio; - participant->spatial_position = 64; + participant->spatial_position = 50; int error = 0; OpusEncoder *new_encoder = opus_encoder_create(audiobridge->sampling_rate, audiobridge->spatial_audio ? 2 : 1, OPUS_APPLICATION_VOIP, &error); From 31d2baf7961600b813a0d99540c4acc476e32fbb Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Tue, 4 May 2021 16:43:51 +0200 Subject: [PATCH 6/7] Added stereo position to demo, when available --- html/audiobridgetest.html | 5 ++- html/audiobridgetest.js | 91 +++++++++++++++++++++++++++++---------- 2 files changed, 73 insertions(+), 23 deletions(-) diff --git a/html/audiobridgetest.html b/html/audiobridgetest.html index 6621aca885..c30994b1c8 100644 --- a/html/audiobridgetest.html +++ b/html/audiobridgetest.html @@ -11,6 +11,7 @@ +