The rtmpdh patch was kinda (completly) broken, fixed patch is attached. Concerning LibreSSL: They use OPENSSL_VERSION_NUMBER = 0x2050200fL which breaks the other openssl code in tls_openssl.c anyway, so LibreSSL support should probably be worried about in a different patch.
From 0edf1d179824fa6400031c6d05b3f464f01abf36 Mon Sep 17 00:00:00 2001 From: sfan5 <sf...@live.de> Date: Wed, 22 Feb 2017 15:38:16 +0100 Subject: [PATCH 2/2] rtmpdh: Stop using OpenSSL-provided DH functions to support 1.1.0
DH (struct dh_st) was made private in the 1.1 series, instead DH is now done the same way as with gcrypt / libgmp. --- libavformat/rtmpdh.c | 94 ++++++++++++++++++++++++---------------------------- libavformat/rtmpdh.h | 13 +++----- 2 files changed, 49 insertions(+), 58 deletions(-) diff --git a/libavformat/rtmpdh.c b/libavformat/rtmpdh.c index 1876fd44f9..1ec1286d23 100644 --- a/libavformat/rtmpdh.c +++ b/libavformat/rtmpdh.c @@ -54,7 +54,6 @@ "F71C35FDAD44CFD2D74F9208BE258FF324943328F67329C0" \ "FFFFFFFFFFFFFFFF" -#if CONFIG_GMP || CONFIG_GCRYPT #if CONFIG_GMP #define bn_new(bn) \ do { \ @@ -93,7 +92,11 @@ else \ ret = 1; \ } while (0) -#define bn_modexp(bn, y, q, p) mpz_powm(bn, y, q, p) +#define bn_modexp(bn, y, q, p, ret) \ + do { \ + mpz_powm(bn, y, q, p); \ + ret = 1; \ + } while(0) #define bn_random(bn, num_bits) \ do { \ int bits = num_bits; \ @@ -125,8 +128,34 @@ #define bn_bn2bin(bn, buf, len) gcry_mpi_print(GCRYMPI_FMT_USG, buf, len, NULL, bn) #define bn_bin2bn(bn, buf, len) gcry_mpi_scan(&bn, GCRYMPI_FMT_USG, buf, len, NULL) #define bn_hex2bn(bn, buf, ret) ret = (gcry_mpi_scan(&bn, GCRYMPI_FMT_HEX, buf, 0, 0) == 0) -#define bn_modexp(bn, y, q, p) gcry_mpi_powm(bn, y, q, p) +#define bn_modexp(bn, y, q, p, ret) \ + do { \ + cry_mpi_powm(bn, y, q, p); \ + ret = 1; \ + } while (0) #define bn_random(bn, num_bits) gcry_mpi_randomize(bn, num_bits, GCRY_WEAK_RANDOM) +#elif CONFIG_OPENSSL +#define bn_new(bn) bn = BN_new() +#define bn_free(bn) BN_free(bn) +#define bn_set_word(bn, w) BN_set_word(bn, w) +#define bn_cmp(a, b) BN_cmp(a, b) +#define bn_copy(to, from) BN_copy(to, from) +#define bn_sub_word(bn, w) BN_sub_word(bn, w) +#define bn_cmp_1(bn) BN_cmp(bn, BN_value_one()) +#define bn_num_bytes(bn) BN_num_bytes(bn) +#define bn_bn2bin(bn, buf, len) BN_bn2bin(bn, buf) +#define bn_bin2bn(bn, buf, len) bn = BN_bin2bn(buf, len, 0) +#define bn_hex2bn(bn, buf, ret) ret = BN_hex2bn(&bn, buf) +#define bn_modexp(bn, y, q, p, ret) \ + do { \ + BN_CTX *ctx = BN_CTX_new(); \ + if (!ctx) \ + ret = 0; \ + else \ + ret = BN_mod_exp(bn, y, q, p, ctx); \ + BN_CTX_free(ctx); \ + } while (0) +#define bn_random(bn, num_bits) BN_rand(bn, num_bits, 0, 0) #endif #define MAX_BYTES 18000 @@ -135,7 +164,7 @@ static FFBigNum dh_generate_key(FF_DH *dh) { - int num_bytes; + int num_bytes, ret; num_bytes = bn_num_bytes(dh->p) - 1; if (num_bytes <= 0 || num_bytes > MAX_BYTES) @@ -152,7 +181,9 @@ static FFBigNum dh_generate_key(FF_DH *dh) return NULL; } - bn_modexp(dh->pub_key, dh->g, dh->priv_key, dh->p); + bn_modexp(dh->pub_key, dh->g, dh->priv_key, dh->p, ret); + if (!ret) + return NULL; return dh->pub_key; } @@ -161,12 +192,15 @@ static int dh_compute_key(FF_DH *dh, FFBigNum pub_key_bn, uint32_t secret_key_len, uint8_t *secret_key) { FFBigNum k; + int ret; bn_new(k); if (!k) return -1; - bn_modexp(k, pub_key_bn, dh->priv_key, dh->p); + bn_modexp(k, pub_key_bn, dh->priv_key, dh->p, ret); + if (!ret) + return -1; bn_bn2bin(k, secret_key, secret_key_len); bn_free(k); @@ -184,53 +218,11 @@ void ff_dh_free(FF_DH *dh) bn_free(dh->priv_key); av_free(dh); } -#elif CONFIG_OPENSSL -#define bn_new(bn) bn = BN_new() -#define bn_free(bn) BN_free(bn) -#define bn_set_word(bn, w) BN_set_word(bn, w) -#define bn_cmp(a, b) BN_cmp(a, b) -#define bn_copy(to, from) BN_copy(to, from) -#define bn_sub_word(bn, w) BN_sub_word(bn, w) -#define bn_cmp_1(bn) BN_cmp(bn, BN_value_one()) -#define bn_num_bytes(bn) BN_num_bytes(bn) -#define bn_bn2bin(bn, buf, len) BN_bn2bin(bn, buf) -#define bn_bin2bn(bn, buf, len) bn = BN_bin2bn(buf, len, 0) -#define bn_hex2bn(bn, buf, ret) ret = BN_hex2bn(&bn, buf) -#define bn_modexp(bn, y, q, p) \ - do { \ - BN_CTX *ctx = BN_CTX_new(); \ - if (!ctx) \ - return AVERROR(ENOMEM); \ - if (!BN_mod_exp(bn, y, q, p, ctx)) { \ - BN_CTX_free(ctx); \ - return AVERROR(EINVAL); \ - } \ - BN_CTX_free(ctx); \ - } while (0) - -#define dh_new() DH_new() -#define dh_generate_key(dh) DH_generate_key(dh) - -static int dh_compute_key(FF_DH *dh, FFBigNum pub_key_bn, - uint32_t secret_key_len, uint8_t *secret_key) -{ - if (secret_key_len < DH_size(dh)) - return AVERROR(EINVAL); - return DH_compute_key(secret_key, pub_key_bn, dh); -} - -void ff_dh_free(FF_DH *dh) -{ - if (!dh) - return; - DH_free(dh); -} -#endif static int dh_is_valid_public_key(FFBigNum y, FFBigNum p, FFBigNum q) { FFBigNum bn = NULL; - int ret = AVERROR(EINVAL); + int ret = AVERROR(EINVAL), ret2; bn_new(bn); if (!bn) @@ -254,7 +246,9 @@ static int dh_is_valid_public_key(FFBigNum y, FFBigNum p, FFBigNum q) * random data. */ /* y must fulfill y^q mod p = 1 */ - bn_modexp(bn, y, q, p); + bn_modexp(bn, y, q, p, ret2); + if (!ret2) + goto fail; if (bn_cmp_1(bn)) goto fail; diff --git a/libavformat/rtmpdh.h b/libavformat/rtmpdh.h index 2b250f595d..3f01e6b17b 100644 --- a/libavformat/rtmpdh.h +++ b/libavformat/rtmpdh.h @@ -26,7 +26,6 @@ #include "config.h" -#if CONFIG_GMP || CONFIG_GCRYPT #if CONFIG_GMP #include <gmp.h> @@ -35,6 +34,11 @@ typedef mpz_ptr FFBigNum; #include <gcrypt.h> typedef gcry_mpi_t FFBigNum; + +#elif CONFIG_OPENSSL +#include <openssl/bn.h> + +typedef BIGNUM *FFBigNum; #endif typedef struct FF_DH { @@ -45,13 +49,6 @@ typedef struct FF_DH { long length; } FF_DH; -#elif CONFIG_OPENSSL -#include <openssl/bn.h> -#include <openssl/dh.h> - -typedef BIGNUM *FFBigNum; -typedef DH FF_DH; -#endif /** * Initialize a Diffie-Hellmann context. -- 2.11.1
_______________________________________________ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org http://ffmpeg.org/mailman/listinfo/ffmpeg-devel