This is an automated email from the ASF dual-hosted git repository. abukor pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kudu.git
commit dfb3ce9efe51ced39e3c83e229725e26abeb0136 Author: Yingchun Lai <[email protected]> AuthorDate: Fri Oct 29 14:46:14 2021 +0800 [build] Introduce an env variable to indicate the thirdparty path Kudu thirdparty will cost dozens of minutes to build, now introduce an env variable THIRDPARTY_DIR to indicate the thirdparty path, it can be a well built thirdparty libraries path. With this feature, we can - Share the same thirdparty path by different code paths, these code paths are not needed to build thirdparty libraries repeatly - Build an docker image with thirdparty libraries well built, then we can use this image for building Kudu, CI, and etc. Change-Id: I5e0bdf4faa44322622c48cacadf8e1165eccd38a Reviewed-on: http://gerrit.cloudera.org:8080/18018 Tested-by: Kudu Jenkins Reviewed-by: Attila Bukor <[email protected]> Reviewed-by: Andrew Wong <[email protected]> --- CMakeLists.txt | 8 ++++++-- build-support/ccache-clang/clang | 3 ++- build-support/ccache-clang/clang++ | 3 ++- build-support/clang_tidy_gerrit.py | 6 +++--- build-support/iwyu.py | 7 ++++--- build-support/iwyu/iwyu_tool.py | 4 +++- build-support/jenkins/build-and-test.sh | 20 ++++++++++++-------- build-support/kudu_util.py | 5 +++++ build-support/lint.sh | 3 ++- build-support/llvm-gcov-wrapper | 3 ++- .../mini-cluster/build_mini_cluster_binaries.sh | 7 ++++--- docs/support/scripts/make_site.sh | 3 ++- src/kudu/clock/CMakeLists.txt | 4 ++-- src/kudu/hms/CMakeLists.txt | 4 ++-- src/kudu/postgres/CMakeLists.txt | 8 ++++---- src/kudu/ranger/CMakeLists.txt | 4 ++-- src/kudu/scripts/dump_breakpad_symbols.py | 11 +++++++++-- 17 files changed, 66 insertions(+), 37 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e276cf1..f3f657a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -90,8 +90,12 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules") include(CMakeParseArguments) set(BUILD_SUPPORT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/build-support) -set(THIRDPARTY_DIR ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty) -set(THIRDPARTY_TOOLCHAIN_DIR ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/clang-toolchain) +if ("$ENV{THIRDPARTY_DIR}" STREQUAL "") + set(THIRDPARTY_DIR ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty) +else() + set(THIRDPARTY_DIR $ENV{THIRDPARTY_DIR}) +endif() +set(THIRDPARTY_TOOLCHAIN_DIR ${THIRDPARTY_DIR}/clang-toolchain) set(THIRDPARTY_INSTALL_DIR ${THIRDPARTY_DIR}/installed) set(THIRDPARTY_INSTALL_COMMON_DIR ${THIRDPARTY_INSTALL_DIR}/common) set(THIRDPARTY_INSTALL_UNINSTRUMENTED_DIR ${THIRDPARTY_INSTALL_DIR}/uninstrumented) diff --git a/build-support/ccache-clang/clang b/build-support/ccache-clang/clang index a31b1ea..a56b27e 100755 --- a/build-support/ccache-clang/clang +++ b/build-support/ccache-clang/clang @@ -17,5 +17,6 @@ # under the License. ROOT=$(dirname $BASH_SOURCE)/../.. -CLANG=$ROOT/thirdparty/clang-toolchain/bin/clang +THIRDPARTY_DIR=${THIRDPARTY_DIR:-$ROOT/thirdparty} +CLANG=${THIRDPARTY_DIR}/clang-toolchain/bin/clang CCACHE_CPP2=yes exec ccache $CLANG -Qunused-arguments `test -t 2 && echo -fcolor-diagnostics` "$@" diff --git a/build-support/ccache-clang/clang++ b/build-support/ccache-clang/clang++ index 9cf7f90..fe1b7d8 100755 --- a/build-support/ccache-clang/clang++ +++ b/build-support/ccache-clang/clang++ @@ -17,5 +17,6 @@ # under the License. ROOT=$(dirname $BASH_SOURCE)/../.. -CLANG=$ROOT/thirdparty/clang-toolchain/bin/clang++ +THIRDPARTY_DIR=${THIRDPARTY_DIR:-$ROOT/thirdparty} +CLANG=${THIRDPARTY_DIR}/clang-toolchain/bin/clang++ CCACHE_CPP2=yes exec ccache $CLANG -Qunused-arguments `test -t 2 && echo -fcolor-diagnostics` "$@" diff --git a/build-support/clang_tidy_gerrit.py b/build-support/clang_tidy_gerrit.py index 35f32f6..aba74ae 100755 --- a/build-support/clang_tidy_gerrit.py +++ b/build-support/clang_tidy_gerrit.py @@ -33,16 +33,16 @@ import sys import unittest import tempfile -from kudu_util import init_logging +from kudu_util import init_logging, get_thirdparty_dir ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) BUILD_PATH = os.path.join(ROOT, "build", "latest") CLANG_TIDY_DIFF = os.path.join( - ROOT, "thirdparty/installed/uninstrumented/share/clang/clang-tidy-diff.py") + get_thirdparty_dir(), "installed/uninstrumented/share/clang/clang-tidy-diff.py") CLANG_TIDY = os.path.join( - ROOT, "thirdparty/clang-toolchain/bin/clang-tidy") + get_thirdparty_dir(), "clang-toolchain/bin/clang-tidy") GERRIT_USER = 'tidybot' GERRIT_PASSWORD = os.getenv('TIDYBOT_PASSWORD') diff --git a/build-support/iwyu.py b/build-support/iwyu.py index ce965d5..4bb8f28 100755 --- a/build-support/iwyu.py +++ b/build-support/iwyu.py @@ -30,7 +30,8 @@ import re import subprocess import sys -from kudu_util import get_upstream_commit, check_output, ROOT, Colors, init_logging +from kudu_util import get_upstream_commit, check_output, ROOT, Colors, \ + init_logging, get_thirdparty_dir import iwyu.fix_includes from iwyu.fix_includes import ParseAndMergeIWYUOutput @@ -44,7 +45,7 @@ script in order to fix any reported issues. """ _MAPPINGS_DIR = os.path.join(ROOT, "build-support/iwyu/mappings/") -_TOOLCHAIN_DIR = os.path.join(ROOT, "thirdparty/clang-toolchain/bin") +_TOOLCHAIN_DIR = os.path.join(get_thirdparty_dir(), "clang-toolchain/bin") _IWYU_TOOL = os.path.join(ROOT, "build-support/iwyu/iwyu_tool.py") # Matches source files that we should run on. @@ -129,7 +130,7 @@ def _relativize_paths(paths): def _get_thirdparty_include_dirs(): - return glob.glob(os.path.join(ROOT, "thirdparty", "installed", "*", "include")) + return glob.glob(os.path.join(get_thirdparty_dir(), "installed", "*", "include")) def _get_fixer_flags(flags): diff --git a/build-support/iwyu/iwyu_tool.py b/build-support/iwyu/iwyu_tool.py index d2a15ea..8eb7d44 100755 --- a/build-support/iwyu/iwyu_tool.py +++ b/build-support/iwyu/iwyu_tool.py @@ -85,6 +85,8 @@ import re import subprocess import sys import unittest +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) +from kudu_util import get_thirdparty_dir def iwyu_formatter(output): @@ -232,7 +234,7 @@ def workaround_add_libcpp(build_dir, command): # # Some sanitizer builds (like TSAN) already include libc++; if the command # already has a flag including libc++, we don't want to add it again. - path_to_libcpp_prefix = os.path.join(build_dir, "..", "..", "thirdparty", "installed") + path_to_libcpp_prefix = os.path.join(get_thirdparty_dir(), "installed") path_to_libcpp_suffix = os.path.join("include", "c++", "v1") tp_pattern = os.path.join(path_to_libcpp_prefix, "*", path_to_libcpp_suffix) found = False diff --git a/build-support/jenkins/build-and-test.sh b/build-support/jenkins/build-and-test.sh index b2ae527..a76476a 100755 --- a/build-support/jenkins/build-and-test.sh +++ b/build-support/jenkins/build-and-test.sh @@ -149,12 +149,17 @@ if [ "$BUILD_TYPE" = "TSAN" ]; then export KUDU_USE_TSAN=1 fi +SOURCE_ROOT=$(cd $(dirname "$BASH_SOURCE")/../..; pwd) +BUILD_ROOT=$SOURCE_ROOT/build/$BUILD_TYPE_LOWER + export KUDU_FLAKY_TEST_ATTEMPTS=${KUDU_FLAKY_TEST_ATTEMPTS:-1} export KUDU_ALLOW_SLOW_TESTS=${KUDU_ALLOW_SLOW_TESTS:-$DEFAULT_ALLOW_SLOW_TESTS} export KUDU_COMPRESS_TEST_OUTPUT=${KUDU_COMPRESS_TEST_OUTPUT:-1} export TEST_TMPDIR=${TEST_TMPDIR:-/tmp/kudutest-$UID} export PARALLEL=${PARALLEL:-$(getconf _NPROCESSORS_ONLN)} export PARALLEL_TESTS=${PARALLEL_TESTS:-$PARALLEL} +export THIRDPARTY_DIR=${THIRDPARTY_DIR:-$SOURCE_ROOT/thirdparty} + BUILD_JAVA=${BUILD_JAVA:-1} BUILD_GRADLE=${BUILD_GRADLE:-1} BUILD_PYTHON=${BUILD_PYTHON:-1} @@ -168,9 +173,6 @@ if [ ! -w "$TEST_TMPDIR" ]; then exit 1 fi -SOURCE_ROOT=$(cd $(dirname "$BASH_SOURCE")/../..; pwd) -BUILD_ROOT=$SOURCE_ROOT/build/$BUILD_TYPE_LOWER - # Remove testing artifacts from the previous run before we do anything # else. Otherwise, if we fail during the "build" step, Jenkins will # archive the test logs from the previous run, thinking they came from @@ -312,16 +314,18 @@ if [ "$BUILD_TYPE" = "TSAN" ]; then THIRDPARTY_TYPE=tsan fi -# The settings for PARALLEL (see above) also affects the thirdparty build. -$SOURCE_ROOT/build-support/enable_devtoolset.sh thirdparty/build-if-necessary.sh $THIRDPARTY_TYPE +if [ ! -n "${NO_REBUILD_THIRDPARTY}" ]; then + # The settings for PARALLEL (see above) also affects the thirdparty build. + $SOURCE_ROOT/build-support/enable_devtoolset.sh $THIRDPARTY_DIR/build-if-necessary.sh $THIRDPARTY_TYPE +fi -THIRDPARTY_BIN=$(pwd)/thirdparty/installed/common/bin +THIRDPARTY_BIN=$THIRDPARTY_DIR/installed/common/bin export PPROF_PATH=$THIRDPARTY_BIN/pprof if which ccache >/dev/null ; then CLANG=$(pwd)/build-support/ccache-clang/clang else - CLANG=$(pwd)/thirdparty/clang-toolchain/bin/clang + CLANG=$THIRDPARTY_DIR/clang-toolchain/bin/clang fi # Make sure we use JDK8 @@ -517,7 +521,7 @@ if [ "$DO_COVERAGE" == "1" ]; then echo echo Generating coverage report... echo ------------------------------------------------------------ - if ! $SOURCE_ROOT/thirdparty/installed/common/bin/gcovr \ + if ! $THIRDPARTY_DIR/installed/common/bin/gcovr \ -r $SOURCE_ROOT \ --gcov-filter='.*src#kudu.*' \ --gcov-executable=$SOURCE_ROOT/build-support/llvm-gcov-wrapper \ diff --git a/build-support/kudu_util.py b/build-support/kudu_util.py index 15d1059..df079f2 100644 --- a/build-support/kudu_util.py +++ b/build-support/kudu_util.py @@ -114,3 +114,8 @@ def get_my_email(): """ Return the email address in the user's git config. """ return check_output(['git', 'config', '--get', 'user.email']).strip().decode('utf-8') + + +def get_thirdparty_dir(): + env = os.environ.copy() + return env['THIRDPARTY_DIR'] if env.has_key('THIRDPARTY_DIR') else os.path.join(ROOT, "thirdparty") diff --git a/build-support/lint.sh b/build-support/lint.sh index f93168d..7f10f83 100755 --- a/build-support/lint.sh +++ b/build-support/lint.sh @@ -50,7 +50,8 @@ fi cpplint_filter="+runtime/broken_libstdcpp_regex,-whitespace/comments,-readability/todo,-readability/inheritance,-build/header_guard,-build/include_order,-legal/copyright,-build/c++11,-readability/nolint" cd $ROOT -$ROOT/thirdparty/installed/common/bin/cpplint.py \ +THIRDPARTY_DIR=${THIRDPARTY_DIR:-$ROOT/thirdparty} +${THIRDPARTY_DIR}/installed/common/bin/cpplint.py \ --verbose=4 \ --filter=$cpplint_filter \ $FILES 2>&1 | grep -v 'Done processing' | tee $TMP diff --git a/build-support/llvm-gcov-wrapper b/build-support/llvm-gcov-wrapper index 0b24685..42b4ead 100755 --- a/build-support/llvm-gcov-wrapper +++ b/build-support/llvm-gcov-wrapper @@ -20,4 +20,5 @@ # gcovr can call it. SOURCE_ROOT=$(cd $(dirname "$BASH_SOURCE")/..; pwd) -exec $SOURCE_ROOT/thirdparty/clang-toolchain/bin/llvm-cov gcov "$@" +THIRDPARTY_DIR=${THIRDPARTY_DIR:-$SOURCE_ROOT/thirdparty} +exec $THIRDPARTY_DIR/clang-toolchain/bin/llvm-cov gcov "$@" diff --git a/build-support/mini-cluster/build_mini_cluster_binaries.sh b/build-support/mini-cluster/build_mini_cluster_binaries.sh index da8d630..fe966f5 100755 --- a/build-support/mini-cluster/build_mini_cluster_binaries.sh +++ b/build-support/mini-cluster/build_mini_cluster_binaries.sh @@ -84,13 +84,14 @@ if [ -d "$BUILD_ROOT" ]; then fi fi +THIRDPARTY_DIR=${THIRDPARTY_DIR:-$BASE_DIR/thirdparty} cd $SOURCE_ROOT if [ -n "$NO_REBUILD_THIRDPARTY" ]; then echo Skipping thirdparty because NO_REBUILD_THIRDPARTY is not empty else echo Building thirdparty... >&2 $SOURCE_ROOT/build-support/enable_devtoolset.sh \ - ./thirdparty/build-if-necessary.sh + $THIRDPARTY_DIR/build-if-necessary.sh fi mkdir -p $BUILD_ROOT @@ -121,7 +122,7 @@ rm -rf CMakeCache.txt CMakeFiles # with the mini-cluster. echo Configuring Kudu... >&2 $SOURCE_ROOT/build-support/enable_devtoolset.sh \ - $SOURCE_ROOT/thirdparty/installed/common/bin/cmake ../.. \ + $THIRDPARTY_DIR/installed/common/bin/cmake ../.. \ -DNO_TESTS=1 -DNO_CHRONY=1 \ -DCMAKE_BUILD_TYPE=RELEASE -DKUDU_LINK=dynamic $EXTRA_CMAKE_FLAGS @@ -191,7 +192,7 @@ if [ $MACOS ]; then JAR_LICENSE=$MINI_CLUSTER_SRCDIR/LICENSE-BINARY-JAR-OSX.txt fi cat $SOURCE_ROOT/LICENSE.txt \ - $SOURCE_ROOT/thirdparty/LICENSE.txt \ + $THIRDPARTY_DIR/LICENSE.txt \ $JAR_LICENSE \ > $ARTIFACT_NAME/LICENSE.txt diff --git a/docs/support/scripts/make_site.sh b/docs/support/scripts/make_site.sh index b01e642..3adbe6f 100755 --- a/docs/support/scripts/make_site.sh +++ b/docs/support/scripts/make_site.sh @@ -73,8 +73,9 @@ if [ -z "$GIT_REMOTE" ]; then usage fi +THIRDPARTY_DIR=${THIRDPARTY_DIR:-$SOURCE_ROOT/thirdparty} # Build Kudu thirdparty -$SOURCE_ROOT/build-support/enable_devtoolset.sh $SOURCE_ROOT/thirdparty/build-if-necessary.sh +$SOURCE_ROOT/build-support/enable_devtoolset.sh $THIRDPARTY_DIR/build-if-necessary.sh echo "Successfully built third-party dependencies." # Build the binaries so we can auto-generate the command-line references diff --git a/src/kudu/clock/CMakeLists.txt b/src/kudu/clock/CMakeLists.txt index b57d30f..0c20fea 100644 --- a/src/kudu/clock/CMakeLists.txt +++ b/src/kudu/clock/CMakeLists.txt @@ -45,10 +45,10 @@ if (NOT NO_CHRONY) # Link the chrony binaries so that they can be found via # MiniChronyd::GetPath in MiniChronyd::Start. execute_process(COMMAND ln -nsf - "${CMAKE_SOURCE_DIR}/thirdparty/installed/common/bin/chronyc" + "${THIRDPARTY_DIR}/installed/common/bin/chronyc" "${EXECUTABLE_OUTPUT_PATH}/chronyc") execute_process(COMMAND ln -nsf - "${CMAKE_SOURCE_DIR}/thirdparty/installed/common/sbin/chronyd" + "${THIRDPARTY_DIR}/installed/common/sbin/chronyd" "${EXECUTABLE_OUTPUT_PATH}/chronyd") set(MINI_CHRONYD_SRCS test/mini_chronyd.cc) diff --git a/src/kudu/hms/CMakeLists.txt b/src/kudu/hms/CMakeLists.txt index ef1171b..e016d84 100644 --- a/src/kudu/hms/CMakeLists.txt +++ b/src/kudu/hms/CMakeLists.txt @@ -54,10 +54,10 @@ target_link_libraries(kudu_hms ${HMS_DEPS}) # Link the home directories, so that they can be found via # test_util::FindHomeDir in MiniHms::Start. execute_process(COMMAND ln -nsf - "${CMAKE_SOURCE_DIR}/thirdparty/installed/common/opt/hive" + "${THIRDPARTY_DIR}/installed/common/opt/hive" "${EXECUTABLE_OUTPUT_PATH}/hive-home") execute_process(COMMAND ln -nsf - "${CMAKE_SOURCE_DIR}/thirdparty/installed/common/opt/hadoop" + "${THIRDPARTY_DIR}/installed/common/opt/hadoop" "${EXECUTABLE_OUTPUT_PATH}/hadoop-home") execute_process(COMMAND ln -nsf "${JAVA_HOME}" diff --git a/src/kudu/postgres/CMakeLists.txt b/src/kudu/postgres/CMakeLists.txt index e9c97a5..6daa30f 100644 --- a/src/kudu/postgres/CMakeLists.txt +++ b/src/kudu/postgres/CMakeLists.txt @@ -20,16 +20,16 @@ ####################################### execute_process(COMMAND ln -nsf - "${CMAKE_SOURCE_DIR}/thirdparty/installed/common/bin" + "${THIRDPARTY_DIR}/installed/common/bin" "${EXECUTABLE_OUTPUT_PATH}/postgres") execute_process(COMMAND ln -nsf - "${CMAKE_SOURCE_DIR}/thirdparty/installed/common/lib" + "${THIRDPARTY_DIR}/installed/common/lib" "${EXECUTABLE_OUTPUT_PATH}/postgres-lib") execute_process(COMMAND ln -nsf - "${CMAKE_SOURCE_DIR}/thirdparty/installed/common/share/postgresql" + "${THIRDPARTY_DIR}/installed/common/share/postgresql" "${EXECUTABLE_OUTPUT_PATH}/postgres-share") execute_process(COMMAND ln -nsf - "${CMAKE_SOURCE_DIR}/thirdparty/installed/common/opt/jdbc/postgresql.jar" + "${THIRDPARTY_DIR}/installed/common/opt/jdbc/postgresql.jar" "${EXECUTABLE_OUTPUT_PATH}/postgresql.jar") add_library(mini_postgres diff --git a/src/kudu/ranger/CMakeLists.txt b/src/kudu/ranger/CMakeLists.txt index a52040b..0c6241f 100644 --- a/src/kudu/ranger/CMakeLists.txt +++ b/src/kudu/ranger/CMakeLists.txt @@ -52,10 +52,10 @@ target_link_libraries(kudu_ranger ${RANGER_DEPS}) ####################################### execute_process(COMMAND ln -nsf - "${CMAKE_SOURCE_DIR}/thirdparty/installed/common/opt/ranger" + "${THIRDPARTY_DIR}/installed/common/opt/ranger" "${EXECUTABLE_OUTPUT_PATH}/ranger-home") execute_process(COMMAND ln -nsf - "${CMAKE_SOURCE_DIR}/thirdparty/installed/common/opt/hadoop" + "${THIRDPARTY_DIR}/installed/common/opt/hadoop" "${EXECUTABLE_OUTPUT_PATH}/hadoop-home") execute_process(COMMAND ln -nsf "${JAVA_HOME}" diff --git a/src/kudu/scripts/dump_breakpad_symbols.py b/src/kudu/scripts/dump_breakpad_symbols.py index bdb2921..f18246c 100644 --- a/src/kudu/scripts/dump_breakpad_symbols.py +++ b/src/kudu/scripts/dump_breakpad_symbols.py @@ -56,7 +56,7 @@ # python dump_breakpad_symbols.py -d /tmp/syms -b kudu/build/debug # # * After, to process a minidump file, use the 'minidump_stackwalk' tool: -# $KUDU_HOME/thirdparty/installed/uninstrumented/bin/minidump_stackwalk \ +# $THIRDPARTY_DIR/installed/uninstrumented/bin/minidump_stackwalk \ # /tmp/kudu-minidumps/kudu-tserver/03c0ee26-bfd1-cf3e-43fa49ca-1a6aae25.dmp /tmp/syms # # For more information, see the getting started docs at @@ -86,6 +86,12 @@ def die(msg=''): sys.exit(1) +def get_thirdparty_dir(): + env = os.environ.copy() + return env['THIRDPARTY_DIR'] if env.has_key('THIRDPARTY_DIR') else \ + os.path.join(ROOT, "thirdparty") + + def find_dump_syms_binary(): """Locate the 'dump_syms' binary from Breakpad. @@ -98,7 +104,8 @@ def find_dump_syms_binary(): if not os.path.isdir(kudu_home): die('Could not find KUDU_HOME directory') # TODO: Use dump_syms_mac if on macOS. - dump_syms = os.path.join(kudu_home, 'thirdparty', 'installed', 'uninstrumented', 'bin', 'dump_syms') + dump_syms = os.path.join(get_thirdparty_dir(), + 'installed', 'uninstrumented', 'bin', 'dump_syms') if not os.path.isfile(dump_syms): die('Could not find dump_syms executable at %s' % dump_syms) return dump_syms
