PR #23433 opened by Jack Lau (JackLau) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23433 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23433.patch
This patchset aims to make sure that the DTLS and TLS open function can use common code like hostname handling to fix the potential related bug >From ae1304c116f5f9a6d5756faa9df88f0b90d23625 Mon Sep 17 00:00:00 2001 From: Jack Lau <[email protected]> Date: Wed, 10 Jun 2026 12:53:13 +0800 Subject: [PATCH 1/2] avformat/tls_openssl: copy the dtls code into tls_open() This is first step for moving dtls related code into tls_open() to make sure they can use common code in one function. Next patch should be removing dtls_start() and create a simple dtls_open() wrapper (set is_dtls and then call tls_open()) just like GnuTLS. Signed-off-by: Jack Lau <[email protected]> --- libavformat/tls_openssl.c | 87 ++++++++++++++++++++++++++++++++------- 1 file changed, 72 insertions(+), 15 deletions(-) diff --git a/libavformat/tls_openssl.c b/libavformat/tls_openssl.c index 7c1289c595..06e04b1c0e 100644 --- a/libavformat/tls_openssl.c +++ b/libavformat/tls_openssl.c @@ -874,29 +874,53 @@ static int tls_open(URLContext *h, const char *uri, int flags, AVDictionary **op TLSShared *s = &c->tls_shared; int ret; - if ((ret = ff_tls_open_underlying(s, h, uri, options)) < 0) - goto fail; + if (!c->tls_shared.external_sock) { + if ((ret = ff_tls_open_underlying(&c->tls_shared, h, uri, options)) < 0) { + av_log(c, AV_LOG_ERROR, "Failed to connect %s\n", uri); + goto fail; + } + } // We want to support all versions of TLS >= 1.0, but not the deprecated // and insecure SSLv2 and SSLv3. Despite the name, TLS_*_method() // enables support for all versions of SSL and TLS, and we then disable // support for the old protocols immediately after creating the context. - c->ctx = SSL_CTX_new(s->listen ? TLS_server_method() : TLS_client_method()); + if (s->is_dtls) + c->ctx = SSL_CTX_new(s->listen ? DTLS_server_method() : DTLS_client_method()); + else + c->ctx = SSL_CTX_new(s->listen ? TLS_server_method() : TLS_client_method()); if (!c->ctx) { av_log(h, AV_LOG_ERROR, "%s\n", openssl_get_error(c)); ret = AVERROR(EIO); goto fail; } - if (!SSL_CTX_set_min_proto_version(c->ctx, TLS1_VERSION)) { - av_log(h, AV_LOG_ERROR, "Failed to set minimum TLS version to TLSv1\n"); - ret = AVERROR_EXTERNAL; - goto fail; + if (!s->is_dtls) { + if (!SSL_CTX_set_min_proto_version(c->ctx, TLS1_VERSION)) { + av_log(h, AV_LOG_ERROR, "Failed to set minimum TLS version to TLSv1\n"); + ret = AVERROR_EXTERNAL; + goto fail; + } } ret = openssl_init_ca_key_cert(h); if (ret < 0) goto fail; if (s->verify) SSL_CTX_set_verify(c->ctx, SSL_VERIFY_PEER|SSL_VERIFY_FAIL_IF_NO_PEER_CERT, NULL); + + if (s->is_dtls && s->use_srtp) { + /** + * The profile for OpenSSL's SRTP is SRTP_AES128_CM_SHA1_80, see ssl/d1_srtp.c. + * The profile for FFmpeg's SRTP is SRTP_AES128_CM_HMAC_SHA1_80, see libavformat/srtp.c. + */ + const char *profiles = "SRTP_AES128_CM_SHA1_80"; + if (SSL_CTX_set_tlsext_use_srtp(c->ctx, profiles)) { + av_log(c, AV_LOG_ERROR, "Init SSL_CTX_set_tlsext_use_srtp failed, profiles=%s, %s\n", + profiles, openssl_get_error(c)); + ret = AVERROR(EINVAL); + goto fail; + } + } + c->ssl = SSL_new(c->ctx); if (!c->ssl) { av_log(h, AV_LOG_ERROR, "%s\n", openssl_get_error(c)); @@ -905,6 +929,19 @@ static int tls_open(URLContext *h, const char *uri, int flags, AVDictionary **op } SSL_set_ex_data(c->ssl, 0, c); SSL_CTX_set_info_callback(c->ctx, openssl_info_callback); + + if (s->is_dtls) { + /** + * We have set the MTU to fragment the DTLS packet. It is important to note that the + * packet is split to ensure that each handshake packet is smaller than the MTU. + */ + if (s->mtu <= 0) + s->mtu = 1096; + SSL_set_options(c->ssl, SSL_OP_NO_QUERY_MTU); + SSL_set_mtu(c->ssl, s->mtu); + DTLS_set_link_mtu(c->ssl, s->mtu); + } + init_bio_method(h); if (!s->listen && !s->numerichost) { // By default OpenSSL does too lax wildcard matching @@ -921,14 +958,34 @@ static int tls_open(URLContext *h, const char *uri, int flags, AVDictionary **op goto fail; } } - ret = s->listen ? SSL_accept(c->ssl) : SSL_connect(c->ssl); - if (ret == 0) { - av_log(h, AV_LOG_ERROR, "Unable to negotiate TLS/SSL session\n"); - ret = AVERROR(EIO); - goto fail; - } else if (ret < 0) { - ret = print_ssl_error(h, ret); - goto fail; + + if (s->is_dtls) { + /* This seems to be necessary despite explicitly setting client/server method above. */ + if (s->listen) + SSL_set_accept_state(c->ssl); + else + SSL_set_connect_state(c->ssl); + + /* The SSL_do_handshake can't be called if DTLS hasn't prepared for udp. */ + if (!c->tls_shared.external_sock) { + ret = dtls_handshake(h); + // Fatal SSL error, for example, no available suite when peer is DTLS 1.0 while we are DTLS 1.2. + if (ret < 0) { + av_log(c, AV_LOG_ERROR, "Failed to drive SSL context, ret=%d\n", ret); + return AVERROR(EIO); + } + } + av_log(c, AV_LOG_VERBOSE, "Setup ok, MTU=%d\n", c->tls_shared.mtu); + } else { + ret = s->listen ? SSL_accept(c->ssl) : SSL_connect(c->ssl); + if (ret == 0) { + av_log(h, AV_LOG_ERROR, "Unable to negotiate TLS/SSL session\n"); + ret = AVERROR(EIO); + goto fail; + } else if (ret < 0) { + ret = print_ssl_error(h, ret); + goto fail; + } } return 0; -- 2.52.0 >From fc3cb756a901f4b92c932e5f038230c8df3ac285 Mon Sep 17 00:00:00 2001 From: Jack Lau <[email protected]> Date: Wed, 10 Jun 2026 13:14:54 +0800 Subject: [PATCH 2/2] avformat/tls_openssl: refactor dtls_start() to use common code Removing the dtls_start() and create a simple dtls_open() wrapper that just set s->is_dtls and call tls_open() to use common code. Signed-off-by: Jack Lau <[email protected]> --- libavformat/tls_openssl.c | 104 ++++---------------------------------- 1 file changed, 9 insertions(+), 95 deletions(-) diff --git a/libavformat/tls_openssl.c b/libavformat/tls_openssl.c index 06e04b1c0e..27ebc003b8 100644 --- a/libavformat/tls_openssl.c +++ b/libavformat/tls_openssl.c @@ -774,100 +774,6 @@ fail: return ret; } -/** - * Once the DTLS role has been negotiated - active for the DTLS client or passive for the - * DTLS server - we proceed to set up the DTLS state and initiate the handshake. - */ -static int dtls_start(URLContext *h, const char *url, int flags, AVDictionary **options) -{ - TLSContext *c = h->priv_data; - TLSShared *s = &c->tls_shared; - int ret = 0; - s->is_dtls = 1; - - if (!c->tls_shared.external_sock) { - if ((ret = ff_tls_open_underlying(&c->tls_shared, h, url, options)) < 0) { - av_log(c, AV_LOG_ERROR, "Failed to connect %s\n", url); - return ret; - } - } - - c->ctx = SSL_CTX_new(s->listen ? DTLS_server_method() : DTLS_client_method()); - if (!c->ctx) { - ret = AVERROR(ENOMEM); - goto fail; - } - - ret = openssl_init_ca_key_cert(h); - if (ret < 0) goto fail; - - /* Note, this doesn't check that the peer certificate actually matches the requested hostname. */ - if (s->verify) - SSL_CTX_set_verify(c->ctx, SSL_VERIFY_PEER|SSL_VERIFY_FAIL_IF_NO_PEER_CERT, NULL); - - if (s->use_srtp) { - /** - * The profile for OpenSSL's SRTP is SRTP_AES128_CM_SHA1_80, see ssl/d1_srtp.c. - * The profile for FFmpeg's SRTP is SRTP_AES128_CM_HMAC_SHA1_80, see libavformat/srtp.c. - */ - const char* profiles = "SRTP_AES128_CM_SHA1_80"; - if (SSL_CTX_set_tlsext_use_srtp(c->ctx, profiles)) { - av_log(c, AV_LOG_ERROR, "Init SSL_CTX_set_tlsext_use_srtp failed, profiles=%s, %s\n", - profiles, openssl_get_error(c)); - ret = AVERROR(EINVAL); - goto fail; - } - } - - /* The ssl should not be created unless the ctx has been initialized. */ - c->ssl = SSL_new(c->ctx); - if (!c->ssl) { - ret = AVERROR(ENOMEM); - goto fail; - } - - if (!s->listen && !s->numerichost) - SSL_set_tlsext_host_name(c->ssl, s->host); - - /* Setup the callback for logging. */ - SSL_set_ex_data(c->ssl, 0, c); - SSL_CTX_set_info_callback(c->ctx, openssl_info_callback); - - /** - * We have set the MTU to fragment the DTLS packet. It is important to note that the - * packet is split to ensure that each handshake packet is smaller than the MTU. - */ - if (s->mtu <= 0) - s->mtu = 1096; - SSL_set_options(c->ssl, SSL_OP_NO_QUERY_MTU); - SSL_set_mtu(c->ssl, s->mtu); - DTLS_set_link_mtu(c->ssl, s->mtu); - init_bio_method(h); - - /* This seems to be necessary despite explicitly setting client/server method above. */ - if (s->listen) - SSL_set_accept_state(c->ssl); - else - SSL_set_connect_state(c->ssl); - - /* The SSL_do_handshake can't be called if DTLS hasn't prepare for udp. */ - if (!c->tls_shared.external_sock) { - ret = dtls_handshake(h); - // Fatal SSL error, for example, no available suite when peer is DTLS 1.0 while we are DTLS 1.2. - if (ret < 0) { - av_log(c, AV_LOG_ERROR, "Failed to drive SSL context, ret=%d\n", ret); - return AVERROR(EIO); - } - } - - av_log(c, AV_LOG_VERBOSE, "Setup ok, MTU=%d\n", c->tls_shared.mtu); - - return 0; -fail: - tls_close(h); - return ret; -} - static int tls_open(URLContext *h, const char *uri, int flags, AVDictionary **options) { TLSContext *c = h->priv_data; @@ -994,6 +900,14 @@ fail: return ret; } +static int dtls_open(URLContext *h, const char *uri, int flags, AVDictionary **options) +{ + TLSContext *c = h->priv_data; + TLSShared *s = &c->tls_shared; + s->is_dtls = 1; + return tls_open(h, uri, flags, options); +} + static int tls_read(URLContext *h, uint8_t *buf, int size) { TLSContext *c = h->priv_data; @@ -1083,7 +997,7 @@ static const AVClass dtls_class = { const URLProtocol ff_dtls_protocol = { .name = "dtls", - .url_open2 = dtls_start, + .url_open2 = dtls_open, .url_handshake = dtls_handshake, .url_close = tls_close, .url_read = tls_read, -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
