This is an automated email from the ASF dual-hosted git repository. oflebbe pushed a commit to branch branch-1.4 in repository https://gitbox.apache.org/repos/asf/bigtop.git
commit 7a5e779ba7437f7b427a50ba724120a455447bb3 Author: Luca Toscano <[email protected]> AuthorDate: Sat Feb 29 11:03:23 2020 +0100 BIGTOP-3308: add support for openssl 1.1.0 on Debian In BIGTOP-2932 a patch was backported from Hadoop upstream to better support openssl 1.1.x. Some time later a new commit was filed to address some corner cases, but it was never backported to BigTop. On Debian 9 openssl 1.1.0 is the default, and BigTop builds against it. When executing the command "hadoop checknative -a" though an error shows up: openssl: false EVP_CIPHER_CTX_cleanup The issue is that EVP_CIPHER_CTX_cleanup was removed from openssl 1.1.0, but OpensslCipher.c kept using it until a proper if guard was added by the patch provided by this pull request. In my case this issue caused Spark 2.4's RPC encryption, based on Openssl native lib, to fail when establishing new connections between workers and AM on Yarn. I rebuilt the hadoop deb package with the patch and tested in the Debian 9 Docker image, the checknative util now returns correctly a positive "true" for openssl. I also tested it with Spark 2.4 on a test Hadoop cluster and it seems working fine. --- .../src/common/hadoop/patch3-HADOOP-15062.diff | 105 +++++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/bigtop-packages/src/common/hadoop/patch3-HADOOP-15062.diff b/bigtop-packages/src/common/hadoop/patch3-HADOOP-15062.diff new file mode 100644 index 0000000..e6b5456 --- /dev/null +++ b/bigtop-packages/src/common/hadoop/patch3-HADOOP-15062.diff @@ -0,0 +1,105 @@ +From 138c1ed5660f713d24bfebc44ea1846f76c00cb9 Mon Sep 17 00:00:00 2001 +From: Yufei Gu <[email protected]> +Date: Tue, 20 Mar 2018 15:19:18 -0700 +Subject: [PATCH] HADOOP-15062. TestCryptoStreamsWithOpensslAesCtrCryptoCodec + fails on Debian 9. Contributed by Miklos Szegedi. + +--- + .../org/apache/hadoop/crypto/OpensslCipher.c | 33 +++++++++++++++++++ + 1 file changed, 33 insertions(+) + +diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c +index c7984a33475f..abff7ea5f17f 100644 +--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c ++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c +@@ -27,8 +27,12 @@ + #ifdef UNIX + static EVP_CIPHER_CTX * (*dlsym_EVP_CIPHER_CTX_new)(void); + static void (*dlsym_EVP_CIPHER_CTX_free)(EVP_CIPHER_CTX *); ++#if OPENSSL_API_COMPAT < 0x10100000L && OPENSSL_VERSION_NUMBER >= 0x10100000L ++static int (*dlsym_EVP_CIPHER_CTX_reset)(EVP_CIPHER_CTX *); ++#else + static int (*dlsym_EVP_CIPHER_CTX_cleanup)(EVP_CIPHER_CTX *); + static void (*dlsym_EVP_CIPHER_CTX_init)(EVP_CIPHER_CTX *); ++#endif + static int (*dlsym_EVP_CIPHER_CTX_set_padding)(EVP_CIPHER_CTX *, int); + static int (*dlsym_EVP_CIPHER_CTX_test_flags)(const EVP_CIPHER_CTX *, int); + static int (*dlsym_EVP_CIPHER_CTX_block_size)(const EVP_CIPHER_CTX *); +@@ -123,10 +127,16 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs + "EVP_CIPHER_CTX_new"); + LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_free, env, openssl, \ + "EVP_CIPHER_CTX_free"); ++#if OPENSSL_API_COMPAT < 0x10100000L && OPENSSL_VERSION_NUMBER >= 0x10100000L ++ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_reset, env, openssl, \ ++ "EVP_CIPHER_CTX_reset"); ++#else + LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_cleanup, env, openssl, \ + "EVP_CIPHER_CTX_cleanup"); + LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_init, env, openssl, \ + "EVP_CIPHER_CTX_init"); ++#endif ++ + LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_set_padding, env, openssl, \ + "EVP_CIPHER_CTX_set_padding"); + LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_test_flags, env, openssl, \ +@@ -271,7 +281,11 @@ JNIEXPORT jlong JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_init + (*env)->ReleaseByteArrayElements(env, key, jKey, 0); + (*env)->ReleaseByteArrayElements(env, iv, jIv, 0); + if (rc == 0) { ++#if OPENSSL_API_COMPAT < 0x10100000L && OPENSSL_VERSION_NUMBER >= 0x10100000L ++ dlsym_EVP_CIPHER_CTX_reset(context); ++#else + dlsym_EVP_CIPHER_CTX_cleanup(context); ++#endif + THROW(env, "java/lang/InternalError", "Error in EVP_CipherInit_ex."); + return (jlong)0; + } +@@ -334,7 +348,11 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_update + int output_len = 0; + if (!dlsym_EVP_CipherUpdate(context, output_bytes, &output_len, \ + input_bytes, input_len)) { ++#if OPENSSL_API_COMPAT < 0x10100000L && OPENSSL_VERSION_NUMBER >= 0x10100000L ++ dlsym_EVP_CIPHER_CTX_reset(context); ++#else + dlsym_EVP_CIPHER_CTX_cleanup(context); ++#endif + THROW(env, "java/lang/InternalError", "Error in EVP_CipherUpdate."); + return 0; + } +@@ -376,7 +394,11 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_doFinal + + int output_len = 0; + if (!dlsym_EVP_CipherFinal_ex(context, output_bytes, &output_len)) { ++#if OPENSSL_API_COMPAT < 0x10100000L && OPENSSL_VERSION_NUMBER >= 0x10100000L ++ dlsym_EVP_CIPHER_CTX_reset(context); ++#else + dlsym_EVP_CIPHER_CTX_cleanup(context); ++#endif + THROW(env, "java/lang/InternalError", "Error in EVP_CipherFinal_ex."); + return 0; + } +@@ -396,6 +418,16 @@ JNIEXPORT jstring JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_getLibrary + (JNIEnv *env, jclass clazz) + { + #ifdef UNIX ++#if OPENSSL_API_COMPAT < 0x10100000L && OPENSSL_VERSION_NUMBER >= 0x10100000L ++ if (dlsym_EVP_CIPHER_CTX_reset) { ++ Dl_info dl_info; ++ if(dladdr( ++ dlsym_EVP_CIPHER_CTX_reset, ++ &dl_info)) { ++ return (*env)->NewStringUTF(env, dl_info.dli_fname); ++ } ++ } ++#else + if (dlsym_EVP_CIPHER_CTX_init) { + Dl_info dl_info; + if(dladdr( +@@ -404,6 +436,7 @@ JNIEXPORT jstring JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_getLibrary + return (*env)->NewStringUTF(env, dl_info.dli_fname); + } + } ++#endif + + return (*env)->NewStringUTF(env, HADOOP_OPENSSL_LIBRARY); + #endif
