From f90b67155981601daad73493d4dc1c67761a4877 Mon Sep 17 00:00:00 2001
From: Lorenzo Miniero <lminiero@gmail.com>
Date: Tue, 20 Sep 2022 10:38:55 +0200
Subject: [PATCH] Update deprecated DTLS functions to OpenSSL v3.0 (see #3048)

---
 dtls.c | 48 +++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 37 insertions(+), 11 deletions(-)

diff --git a/dtls.c b/dtls.c
index 3afda93a53..c5e603e4b4 100644
--- a/dtls.c
+++ b/dtls.c
@@ -187,22 +187,24 @@ static void janus_dtls_cb_openssl_lock(int mode, int type, const char *file, int
 
 static int janus_dtls_generate_keys(X509 **certificate, EVP_PKEY **private_key, gboolean rsa_private_key) {
 	static const int num_bits = 2048;
+#if OPENSSL_VERSION_MAJOR < 3
 	BIGNUM *bne = NULL;
 	RSA *rsa_key = NULL;
-	X509_NAME *cert_name = NULL;
 	EC_KEY *ecc_key = NULL;
+#endif
+	X509_NAME *cert_name = NULL;
 
 	JANUS_LOG(LOG_VERB, "Generating DTLS key / cert\n");
 
-	/* Create a private key object (needed to hold the RSA key). */
-	*private_key = EVP_PKEY_new();
-	if(!*private_key) {
-		JANUS_LOG(LOG_FATAL, "EVP_PKEY_new() failed\n");
-		goto error;
-	}
-
-
 	if(rsa_private_key) {
+#if OPENSSL_VERSION_MAJOR < 3
+		/* Create a private key object (needed to hold the RSA key). */
+		*private_key = EVP_PKEY_new();
+		if(!*private_key) {
+			JANUS_LOG(LOG_FATAL, "EVP_PKEY_new() failed\n");
+			goto error;
+		}
+
 		/* Create a big number object. */
 		bne = BN_new();
 		if(!bne) {
@@ -235,8 +237,16 @@ static int janus_dtls_generate_keys(X509 **certificate, EVP_PKEY **private_key,
 
 		/* The RSA key now belongs to the private key, so don't clean it up separately. */
 		rsa_key = NULL;
+#else
+		*private_key = EVP_RSA_gen(num_bits);
+		if(!*private_key) {
+			JANUS_LOG(LOG_FATAL, "EVP_RSA_gen() failed\n");
+			goto error;
+		}
+#endif
 	} else {
 		/* Create key with curve dictated by DTLS_ELLIPTIC_CURVE */
+#if OPENSSL_VERSION_MAJOR < 3
 		if((ecc_key = EC_KEY_new_by_curve_name(DTLS_ELLIPTIC_CURVE)) == NULL) {
 			JANUS_LOG(LOG_FATAL, "EC_KEY_new_by_curve_name() failed\n");
 			goto error;
@@ -257,6 +267,13 @@ static int janus_dtls_generate_keys(X509 **certificate, EVP_PKEY **private_key,
 
 		/* The EC key now belongs to the private key, so don't clean it up separately. */
 		ecc_key = NULL;
+#else
+		*private_key = EVP_EC_gen("prime256v1");
+		if(!*private_key) {
+			JANUS_LOG(LOG_FATAL, "EVP_EC_gen() failed\n");
+			goto error;
+		}
+#endif
 	}
 
 	/* Create the X509 certificate. */
@@ -304,16 +321,20 @@ static int janus_dtls_generate_keys(X509 **certificate, EVP_PKEY **private_key,
 	}
 
 	/* Free stuff and resurn. */
+#if OPENSSL_VERSION_MAJOR < 3
 	BN_free(bne);
+#endif
 	return 0;
 
 error:
+#if OPENSSL_VERSION_MAJOR < 3
 	if(bne)
 		BN_free(bne);
 	if(rsa_key && !*private_key)
 		RSA_free(rsa_key);
 	if(ecc_key && !*private_key)
 		EC_KEY_free(ecc_key);
+#endif
 	if(*private_key)
 		EVP_PKEY_free(*private_key);  /* This also frees the RSA key. */
 	if(*certificate)
@@ -593,6 +614,7 @@ janus_dtls_srtp *janus_dtls_srtp_create(void *ice_component, janus_dtls_role rol
 	 * negotiated when acting as the server. Use NIST's P-256 which is
 	 * commonly supported.
 	 */
+#if OPENSSL_VERSION_MAJOR < 3
 	EC_KEY *ecdh = EC_KEY_new_by_curve_name(NID_X9_62_prime256v1);
 	if(ecdh == NULL) {
 		JANUS_LOG(LOG_ERR, "[%"SCNu64"]   Error creating ECDH group! (%s)\n",
@@ -600,10 +622,14 @@ janus_dtls_srtp *janus_dtls_srtp_create(void *ice_component, janus_dtls_role rol
 		janus_refcount_decrease(&dtls->ref);
 		return NULL;
 	}
-	const long flags = SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 | SSL_OP_NO_COMPRESSION | SSL_OP_SINGLE_ECDH_USE;
-	SSL_set_options(dtls->ssl, flags);
 	SSL_set_tmp_ecdh(dtls->ssl, ecdh);
 	EC_KEY_free(ecdh);
+#else
+	int grp_list[1] = { NID_X9_62_prime256v1 };
+	SSL_set1_groups(dtls->ssl, grp_list, 1);
+#endif
+	const long flags = SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 | SSL_OP_NO_COMPRESSION | SSL_OP_SINGLE_ECDH_USE;
+	SSL_set_options(dtls->ssl, flags);
 #ifdef HAVE_DTLS_SETTIMEOUT
 	JANUS_LOG(LOG_VERB, "[%"SCNu64"]   Setting DTLS initial timeout: %"SCNu16"ms\n", handle->handle_id, dtls_timeout_base);
 	DTLSv1_set_initial_timeout_duration(dtls->ssl, dtls_timeout_base);