This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch branch-0.15 in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 259cc7b2ad35b885b3b79fe6ff4575fdab8f6442 Author: Mingyu Chen <[email protected]> AuthorDate: Tue Oct 19 13:26:02 2021 +0800 [ARM64] Fix some problem when compiling on ARM64 platform (#6836) (#6872) With thirdparties 1.4.0 to 1.4.1 1. Add patch for aws-c-cal-0.4.5 2. Add some solutions for `undefined reference libpsl` 3. Move libgsasl to fix link problme of libcurl. 4. Downgrade openssl to 1.0.2k to fix problem of low version glibc --- be/CMakeLists.txt | 8 ++++---- be/src/util/aes_util.cpp | 16 ++++++++-------- docs/en/installing/compilation-arm.md | 8 ++++++++ docs/en/installing/compilation.md | 8 +++++--- docs/zh-CN/installing/compilation-arm.md | 13 +++++++++++-- docs/zh-CN/installing/compilation.md | 8 +++++--- thirdparty/CHANGELOG.md | 14 ++++++++++++-- thirdparty/download-thirdparty.sh | 12 ++++++++++++ thirdparty/patches/aws-c-cal-0.4.5.patch | 11 +++++++++++ thirdparty/vars.sh | 8 ++++---- 10 files changed, 80 insertions(+), 26 deletions(-) diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index b9edcda..d8ac46d 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -294,14 +294,14 @@ set_target_properties(aws-s2n PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib add_library(minzip STATIC IMPORTED) set_target_properties(minzip PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libminizip.a) +add_library(gsasl STATIC IMPORTED) +set_target_properties(gsasl PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libgsasl.a) + if (ARCH_AMD64) # libhdfs3 only support x86 or amd64 add_library(hdfs3 STATIC IMPORTED) set_target_properties(hdfs3 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libhdfs3.a) - add_library(gsasl STATIC IMPORTED) - set_target_properties(gsasl PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libgsasl.a) - add_library(xml2 STATIC IMPORTED) set_target_properties(xml2 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libxml2.a) @@ -494,6 +494,7 @@ set(COMMON_THIRDPARTY lz4 libevent libevent_pthreads + gsasl curl ${LIBZ} ${LIBBZ2} @@ -531,7 +532,6 @@ endif() set(X86_DEPENDENCIES ${COMMON_THIRDPARTY} hdfs3 - gsasl xml2 lzma ) diff --git a/be/src/util/aes_util.cpp b/be/src/util/aes_util.cpp index 67d9fe1..1b4c7d1 100644 --- a/be/src/util/aes_util.cpp +++ b/be/src/util/aes_util.cpp @@ -102,6 +102,7 @@ static int do_encrypt(EVP_CIPHER_CTX* aes_ctx, const EVP_CIPHER* cipher, int AesUtil::encrypt(AesMode mode, const unsigned char* source, uint32_t source_length, const unsigned char* key, uint32_t key_length, const unsigned char* iv, bool padding, unsigned char* encrypt) { + EVP_CIPHER_CTX aes_ctx; const EVP_CIPHER* cipher = get_evp_type(mode); /* The encrypt key to be used for encryption */ unsigned char encrypt_key[AES_MAX_KEY_LENGTH / 8]; @@ -110,12 +111,11 @@ int AesUtil::encrypt(AesMode mode, const unsigned char* source, uint32_t source_ if (cipher == nullptr || (EVP_CIPHER_iv_length(cipher) > 0 && !iv)) { return AES_BAD_DATA; } - EVP_CIPHER_CTX* aes_ctx = EVP_CIPHER_CTX_new(); - EVP_CIPHER_CTX_reset(aes_ctx); + EVP_CIPHER_CTX_init(&aes_ctx); int length = 0; - int ret = do_encrypt(aes_ctx, cipher, source, source_length, encrypt_key, iv, padding, encrypt, + int ret = do_encrypt(&aes_ctx, cipher, source, source_length, encrypt_key, iv, padding, encrypt, &length); - EVP_CIPHER_CTX_free(aes_ctx); + EVP_CIPHER_CTX_cleanup(&aes_ctx); if (ret == 0) { ERR_clear_error(); return AES_BAD_DATA; @@ -150,6 +150,7 @@ static int do_decrypt(EVP_CIPHER_CTX* aes_ctx, const EVP_CIPHER* cipher, int AesUtil::decrypt(AesMode mode, const unsigned char* encrypt, uint32_t encrypt_length, const unsigned char* key, uint32_t key_length, const unsigned char* iv, bool padding, unsigned char* decrypt_content) { + EVP_CIPHER_CTX aes_ctx; const EVP_CIPHER* cipher = get_evp_type(mode); /* The encrypt key to be used for decryption */ @@ -160,12 +161,11 @@ int AesUtil::decrypt(AesMode mode, const unsigned char* encrypt, uint32_t encryp return AES_BAD_DATA; } - EVP_CIPHER_CTX* aes_ctx = EVP_CIPHER_CTX_new(); - EVP_CIPHER_CTX_reset(aes_ctx); + EVP_CIPHER_CTX_init(&aes_ctx); int length = 0; - int ret = do_decrypt(aes_ctx, cipher, encrypt, encrypt_length, encrypt_key, iv, padding, + int ret = do_decrypt(&aes_ctx, cipher, encrypt, encrypt_length, encrypt_key, iv, padding, decrypt_content, &length); - EVP_CIPHER_CTX_free(aes_ctx); + EVP_CIPHER_CTX_cleanup(&aes_ctx); if (ret > 0) { return length; } else { diff --git a/docs/en/installing/compilation-arm.md b/docs/en/installing/compilation-arm.md index 19276d8..a013979 100644 --- a/docs/en/installing/compilation-arm.md +++ b/docs/en/installing/compilation-arm.md @@ -222,3 +222,11 @@ Suppose Doris source code is under `/home/doris/doris-src/`. Execute `sh build.sh`. +### 5. FAQ + +1. `undefined reference to psl_free` appears when compiling Doris + + libcurl will call libpsl functions, but libpsl is not linked for an unknown reason. Solutions (choose one of the two): + + 1. Add `--without-libpsl` to the `build_curl` method in `thirdparty/build-thirdparty.sh`, recompile libcurl, and then recompile Doris. + 2. About line 603 in `be/CMakeLists.txt`, add `-lpsl` after `-pthread`, and then recompile Doris. diff --git a/docs/en/installing/compilation.md b/docs/en/installing/compilation.md index f48340d..4e18c77 100644 --- a/docs/en/installing/compilation.md +++ b/docs/en/installing/compilation.md @@ -53,13 +53,15 @@ Note: For different versions of Doris, you need to download the corresponding mi | apache/incubator-doris:build-env-1.1 | [ff0dd0d](https://github.com/apache/incubator-doris/commit/ff0dd0d2daa588f18b6db56f947e813a56d8ec81) or later | 0.10.x or later | | apache/incubator-doris:build-env-1.2 | [4ef5a8c](https://github.com/apache/incubator-doris/commit/4ef5a8c8560351d7fff7ff8fd51c4c7a75e006a8) or later | 0.12.x - 0.14.0 | | apache/incubator-doris:build-env-1.3.1 | [ad67dd3](https://github.com/apache/incubator-doris/commit/ad67dd34a04c1ca960cff38e5b335b30fc7d559f) or later | 0.14.x | -| apache/incubator-doris:build-env-1.4.0 | [24d3861](https://github.com/apache/incubator-doris/commit/24d38614a0f21ed606462816a262c2e1d8273ace) or later | 1.15.0 or later | +| apache/incubator-doris:build-env-1.4.1 | [24d3861](https://github.com/apache/incubator-doris/commit/24d38614a0f21ed606462816a262c2e1d8273ace) or later | 1.15.0 or later | **note**: -> 1. Doris version 0.14.0 still uses apache/incubator-doris:build-env-1.2 to compile, and the subsequent code will use apache/incubator-doris:build-env-1.3.1. +> 1. Dev docker image [ChangeLog](https://github.com/apache/incubator-doris/blob/master/thirdparty/CHANGELOG.md) -> 2. In the docker image of build-env-1.3.1, both OpenJDK 8 and OpenJDK 11 are included, and OpenJDK 11 is used for compilation by default. Please make sure that the JDK version used for compiling is the same as the JDK version used at runtime, otherwise it may cause unexpected operation errors. You can use the following command to switch the default JDK version in container: +> 2. Doris version 0.14.0 still uses apache/incubator-doris:build-env-1.2 to compile, and the subsequent code will use apache/incubator-doris:build-env-1.3.1. + +> 3. In the docker image of build-env-1.3.1, both OpenJDK 8 and OpenJDK 11 are included, and OpenJDK 11 is used for compilation by default. Please make sure that the JDK version used for compiling is the same as the JDK version used at runtime, otherwise it may cause unexpected operation errors. You can use the following command to switch the default JDK version in container: > > Switch to JDK 8: > diff --git a/docs/zh-CN/installing/compilation-arm.md b/docs/zh-CN/installing/compilation-arm.md index fbf750f..772a474 100644 --- a/docs/zh-CN/installing/compilation-arm.md +++ b/docs/zh-CN/installing/compilation-arm.md @@ -177,7 +177,7 @@ make -j && make install 1. 关闭 `build_mysql` 和 `build_libhdfs3` - mysql 不在需要。而 libhdfs3 暂不支持 arm 架构，所以在arm中运行Doris，暂不支持通过 libhdfs3 直接访问 hdfs，需要通过broker。 + mysql 不再需要。而 libhdfs3 暂不支持 arm 架构，所以在arm中运行Doris，暂不支持通过 libhdfs3 直接访问 hdfs，需要通过broker。 2. 在 `build_curl` 中增加 configure 参数：`--without-libpsl`。如果不添加，则在最终编译Doris BE的链接阶段，可能报错：`undefined reference to ‘psl_is_cookie_domain_acceptable'` @@ -217,7 +217,16 @@ make -j && make install * 编译到某个阶段卡住不动。 不确定原因。解决方案：重跑 `build-thirdparty.sh`。`build-thirdparty.sh` 是可以重复执行的。 - + ### 4. 编译Doris源码 执行 `sh build.sh` 即可。 + +### 5. 常见错误 + +1. 编译 Doris 时出现 `undefined reference to psl_free` + + libcurl 会调用 libpsl 的函数，但 libpsl 未连接，原因未知。解决方法（二选一）： + + 1. 在 `thirdparty/build-thirdparty.sh` 中的 `build_curl` 方法中添加 `--without-libpsl` 后重新编译 libcurl，然后再重新编译 Doris。 + 2. `be/CMakeLists.txt` 中 603 行左右，`-pthread` 后添加 `-lpsl`，然后重新编译 Doris。 diff --git a/docs/zh-CN/installing/compilation.md b/docs/zh-CN/installing/compilation.md index 17e2ddd..e8213c1 100644 --- a/docs/zh-CN/installing/compilation.md +++ b/docs/zh-CN/installing/compilation.md @@ -52,13 +52,15 @@ under the License. | apache/incubator-doris:build-env-1.1 | [ff0dd0d](https://github.com/apache/incubator-doris/commit/ff0dd0d2daa588f18b6db56f947e813a56d8ec81) | 0.10.x, 0.11.x | | apache/incubator-doris:build-env-1.2 | [4ef5a8c](https://github.com/apache/incubator-doris/commit/4ef5a8c8560351d7fff7ff8fd51c4c7a75e006a8) | 0.12.x - 0.14.0 | | apache/incubator-doris:build-env-1.3.1 | [ad67dd3](https://github.com/apache/incubator-doris/commit/ad67dd34a04c1ca960cff38e5b335b30fc7d559f) | 0.14.x | -| apache/incubator-doris:build-env-1.4.0 | [24d3861](https://github.com/apache/incubator-doris/commit/24d38614a0f21ed606462816a262c2e1d8273ace) or later | 后续的发布版本 | +| apache/incubator-doris:build-env-1.4.1 | [24d3861](https://github.com/apache/incubator-doris/commit/24d38614a0f21ed606462816a262c2e1d8273ace) or later | 后续的发布版本 | **注意**： -> 1. doris 0.14.0 版本仍然使用apache/incubator-doris:build-env-1.2 编译，之后的代码将使用apache/incubator-doris:build-env-1.3.1。 +> 1. 编译镜像 [ChangeLog](https://github.com/apache/incubator-doris/blob/master/thirdparty/CHANGELOG.md)。 -> 2. 在 build-env-1.3.1 的docker镜像中，同时包含了 OpenJDK 8 和 OpenJDK 11，并且默认使用 OpenJDK 11 编译。请确保编译使用的 JDK 版本和运行时使用的 JDK 版本一致，否则会导致非预期的运行错误。你可以使用在进入编译镜像的容器后，使用以下命令切换默认 JDK 版本： +> 2. doris 0.14.0 版本仍然使用apache/incubator-doris:build-env-1.2 编译，之后的代码将使用apache/incubator-doris:build-env-1.3.1。 + +> 3. 在 build-env-1.3.1 的docker镜像中，同时包含了 OpenJDK 8 和 OpenJDK 11，并且默认使用 OpenJDK 11 编译。请确保编译使用的 JDK 版本和运行时使用的 JDK 版本一致，否则会导致非预期的运行错误。你可以使用在进入编译镜像的容器后，使用以下命令切换默认 JDK 版本： > > 切换到 JDK 8： > diff --git a/thirdparty/CHANGELOG.md b/thirdparty/CHANGELOG.md index 81cde3c..597a78b 100644 --- a/thirdparty/CHANGELOG.md +++ b/thirdparty/CHANGELOG.md @@ -2,7 +2,17 @@ This file contains version of the third-party dependency libraries in the build-env image. The docker build-env image is apache/incubator-doris, and the tag is `build-env-${version}` -## build-env-1.4.0 [2021-10-13] +## build-env-1.4.1 + +### Changes + +Openssl 1.1.1l introduces the getentropy() method, which requires glibc 2.25 or higher, +which will cause Doris BE to fail to run in the low-version glibc environment. +Temporarily roll back the openssl version. + +- OpenSSL 1.1.1l -> 1.0.2k + +## build-env-1.4.0 ### Changes @@ -135,4 +145,4 @@ This file contains version of the third-party dependency libraries in the build- - libhdfs3: master - libdivide: 4.0.0 - pdqsort: git20180419 -- benchmark: 1.5.6 \ No newline at end of file +- benchmark: 1.5.6 diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index 8f118f0..bfe20ac 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -267,5 +267,17 @@ if [ ! -f $PATCHED_MARK ]; then touch $PATCHED_MARK fi cd - + +# aws-c-cal patch to fix compile error +# This bug has been fixed in new version of aws-c-cal +if [ $AWS_C_CAL_SOURCE == "aws-c-cal-0.4.5" ]; then + cd $TP_SOURCE_DIR/$AWS_C_CAL_SOURCE + if [ ! -f $PATCHED_MARK ]; then + patch -p1 < $TP_PATCH_DIR/aws-c-cal-0.4.5.patch + touch $PATCHED_MARK + fi + cd - +fi + echo "Finished patching $HDFS3_SOURCE" diff --git a/thirdparty/patches/aws-c-cal-0.4.5.patch b/thirdparty/patches/aws-c-cal-0.4.5.patch new file mode 100644 index 0000000..aca87f0 --- /dev/null +++ b/thirdparty/patches/aws-c-cal-0.4.5.patch @@ -0,0 +1,11 @@ +--- a/source/unix/openssl_platform_init.c 2021-10-18 19:33:56.524537840 +0800 ++++ b/source/unix/openssl_platform_init.c 2021-10-18 19:35:06.534537840 +0800 +@@ -367,7 +367,7 @@ static int s_resolve_libcrypto(void) { + } + + /* Ignore warnings about how CRYPTO_get_locking_callback() always returns NULL on 1.1.1 */ +-#if !defined(__GNUC__) || (__GNUC__ >= 4 && __GNUC_MINOR__ > 1) ++#if !defined(__GNUC__) || (__GNUC__ * 100 + __GNUC_MINOR__ * 10 > 410) + # pragma GCC diagnostic push + # pragma GCC diagnostic ignored "-Waddress" + #endif diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index 5edd167..734a656 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -58,10 +58,10 @@ LIBEVENT_SOURCE=libevent-release-2.1.12-stable LIBEVENT_MD5SUM="0d5a27436bf7ff8253420c8cf09f47ca" # openssl -OPENSSL_DOWNLOAD="https://github.com/openssl/openssl/archive/OpenSSL_1_1_1l.tar.gz"; -OPENSSL_NAME=openssl-OpenSSL_1_1_1l.tar.gz -OPENSSL_SOURCE=openssl-OpenSSL_1_1_1l -OPENSSL_MD5SUM="2ff02cd10f96671162df3efd60470cfe" +OPENSSL_DOWNLOAD="https://www.openssl.org/source/old/1.0.2/openssl-1.0.2k.tar.gz"; +OPENSSL_NAME=openssl-1.0.2k.tar.gz +OPENSSL_SOURCE=openssl-1.0.2k +OPENSSL_MD5SUM="f965fc0bf01bf882b31314b61391ae65" # thrift THRIFT_DOWNLOAD="http://archive.apache.org/dist/thrift/0.13.0/thrift-0.13.0.tar.gz"; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
