This is an automated email from the ASF dual-hosted git repository. laszlog pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 408c119f7da022e2f6de2df2b5af4546eefdb5bc Author: Laszlo Gaal <[email protected]> AuthorDate: Tue Mar 19 23:02:32 2024 +0100 IMPALA-12564: Prevent Hive loading libfesupport.so in the minicluster during TSAN runs During TSAN runs all Impala binaries (including libfesupport.so) are built with TSAN options, which include a reference to the external symbol __tsan_init. This causes a problem for libfesupport.so when it is loaded into Hive during minicluster startup, because the Java VM running Hive's code cannot supply this symbol (the stock JVM is obviously not built with TSAN). Unfortunately this symbol resolution failure causes Hive's JVM simply to abort on Red Hat 8 (or later) and on Ubuntu 20.04 (or later). On earlier versions of the same platforms the JVM turned the same failure into an UnsatisfiedLinkError exception, which is actually handled by Hive. This patch prevents libfesupport.so from being loaded into Hive for TSAN runs so that the minicluster can actually be started. This is achieved by not adding the directory containing libfesupport.so to JAVA_LIBRARY_PATH, preventing the JVM from finding it. Change-Id: Ie030d9876c297d6e9dae80eba37e525ee2bccb20 Reviewed-on: http://gerrit.cloudera.org:8080/21191 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- testdata/bin/run-hive-server.sh | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/testdata/bin/run-hive-server.sh b/testdata/bin/run-hive-server.sh index 245d4ef09..b824be7e6 100755 --- a/testdata/bin/run-hive-server.sh +++ b/testdata/bin/run-hive-server.sh @@ -166,13 +166,26 @@ if [[ ${START_METASTORE} -eq 1 && -z $HMS_PID ]]; then ${CLUSTER_BIN}/wait-for-metastore.py --transport=${METASTORE_TRANSPORT} fi -# Include the latest libfesupport.so in the JAVA_LIBRARY_PATH -export JAVA_LIBRARY_PATH="${JAVA_LIBRARY_PATH-}:${IMPALA_HOME}/be/build/latest/service/" - -# Add the toolchain's libstdc++ to the LD_LIBRARY_PATH, because libfesupport.so may -# need the newer version. -GCC_HOME="${IMPALA_TOOLCHAIN_PACKAGES_HOME}/gcc-${IMPALA_GCC_VERSION}" -export LD_LIBRARY_PATH="${LD_LIBRARY_PATH-}:${GCC_HOME}/lib64" +# In TSAN builds libfesupport.so refers to an external symbol __tsan_init, which cannot +# be supplied by Hive's JVM when it loads libfesupport.so. On RedHat 8 and Ubuntu 20.04 +# (or later versions) the symbol resolution failure aborts the JVM itself (instead of +# just throwing an UnsatisfiedLinkError exception), breaking the minicluster by not +# letting Hive run at all. +# Avoid this by artifically blocking libfesupport.so from being loaded during TSAN runs. +# TSAN and FULL_TSAN runs are detected by reading the build type from +# the .cmake_build_type file generated by CMake. +# As this script is only run in a minicluster context, it is reasonable to assume that +# a CMake-controlled build was run before the minicluster start attempt. If not, the +# condition fails "safely", not blocking the load. +if ! grep -q "TSAN" ${IMPALA_HOME}/.cmake_build_type ; then + # Include the latest libfesupport.so in the JAVA_LIBRARY_PATH + export JAVA_LIBRARY_PATH="${JAVA_LIBRARY_PATH-}:${IMPALA_HOME}/be/build/latest/service/" + + # Add the toolchain's libstdc++ to the LD_LIBRARY_PATH, because libfesupport.so may + # need the newer version. + GCC_HOME="${IMPALA_TOOLCHAIN_PACKAGES_HOME}/gcc-${IMPALA_GCC_VERSION}" + export LD_LIBRARY_PATH="${LD_LIBRARY_PATH-}:${GCC_HOME}/lib64" +fi export HIVESERVER2_HADOOP_OPTS="-Xdebug -Xrunjdwp:transport=dt_socket,server=y,\ suspend=n,address=30020"
