This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 206ad72d9e4 branch-3.1: [fix](build): Fix build.sh (#57684)
206ad72d9e4 is described below
commit 206ad72d9e453fafa4c00e3c4fca3fdfc3ad26bd
Author: Calvin Kirs <[email protected]>
AuthorDate: Tue Nov 4 20:29:01 2025 +0800
branch-3.1: [fix](build): Fix build.sh (#57684)
---
build.sh | 132 ++++++++++++++++++++++++++++++---------------------------------
1 file changed, 63 insertions(+), 69 deletions(-)
diff --git a/build.sh b/build.sh
index 90621ae7461..809d29d0e81 100755
--- a/build.sh
+++ b/build.sh
@@ -35,12 +35,11 @@ if [[ -z "${DORIS_THIRDPARTY}" ]]; then
fi
export TP_INCLUDE_DIR="${DORIS_THIRDPARTY}/installed/include"
export TP_LIB_DIR="${DORIS_THIRDPARTY}/installed/lib"
-
+HADOOP_DEPS_NAME="hadoop-deps"
TARGET_SYSTEM="$(uname -s)"
TARGET_ARCH="$(uname -m)"
echo "Target system: ${TARGET_SYSTEM}; Target arch: ${TARGET_ARCH}"
-HADOOP_DEPS_NAME="hadoop-deps"
. "${DORIS_HOME}/env.sh"
# Check args
@@ -55,8 +54,6 @@ Usage: $0 <options>
--file-cache-microbench build Backend file cache microbench tool.
Default OFF.
--cloud build Cloud. Default OFF.
--index-tool build Backend inverted index tool. Default OFF.
- --benchmark build Google Benchmark. Default OFF.
- --task-executor-simulator build Backend task executor simulator. Default
OFF.
--broker build Broker. Default ON.
--hive-udf build Hive UDF library for Ingestion Load.
Default ON.
--be-java-extensions build Backend java extensions. Default ON.
@@ -72,7 +69,6 @@ Usage: $0 <options>
DISABLE_BE_JAVA_EXTENSIONS If set DISABLE_BE_JAVA_EXTENSIONS=ON, we will
do not build binary with java-udf,hadoop-hudi-scanner,jdbc-scanner and so on
Default is OFF.
DISABLE_JAVA_CHECK_STYLE If set DISABLE_JAVA_CHECK_STYLE=ON, it will
skip style check of java code in FE.
DISABLE_BUILD_AZURE If set DISABLE_BUILD_AZURE=ON, it will not
build azure into BE.
-
Eg.
$0 build all
$0 --be build Backend
@@ -80,10 +76,9 @@ Usage: $0 <options>
$0 --file-cache-microbench build Backend file cache
microbench tool
$0 --cloud build Cloud
$0 --index-tool build Backend inverted index tool
- $0 --benchmark build Google Benchmark of Backend
- $0 --fe --clean clean and build Frontend.
+ $0 --fe --clean clean and build Frontend and Spark
Dpp application
$0 --fe --be --clean clean and build Frontend and
Backend
- $0 --task-executor-simulator build task executor simulator
+ $0 --spark-dpp build Spark DPP application alone
$0 --broker build Broker
$0 --be --fe build Backend, Frontend, and Java
UDF library
$0 --be --coverage build Backend with coverage enabled
@@ -142,8 +137,6 @@ if ! OPTS="$(getopt \
-l 'meta-tool' \
-l 'file-cache-microbench' \
-l 'index-tool' \
- -l 'benchmark' \
- -l 'task-executor-simulator' \
-l 'spark-dpp' \
-l 'hive-udf' \
-l 'be-java-extensions' \
@@ -167,8 +160,6 @@ BUILD_BROKER=0
BUILD_META_TOOL='OFF'
BUILD_FILE_CACHE_MICROBENCH_TOOL='OFF'
BUILD_INDEX_TOOL='OFF'
-BUILD_BENCHMARK='OFF'
-BUILD_TASK_EXECUTOR_SIMULATOR='OFF'
BUILD_BE_JAVA_EXTENSIONS=0
BUILD_HIVE_UDF=0
CLEAN=0
@@ -187,9 +178,7 @@ if [[ "$#" == 1 ]]; then
BUILD_BROKER=1
BUILD_META_TOOL='OFF'
BUILD_FILE_CACHE_MICROBENCH_TOOL='OFF'
- BUILD_TASK_EXECUTOR_SIMULATOR='OFF'
BUILD_INDEX_TOOL='OFF'
- BUILD_BENCHMARK='OFF'
BUILD_HIVE_UDF=1
BUILD_BE_JAVA_EXTENSIONS=1
CLEAN=0
@@ -228,16 +217,6 @@ else
BUILD_INDEX_TOOL='ON'
shift
;;
- --benchmark)
- BUILD_BENCHMARK='ON'
- BUILD_BE=1 # go into BE cmake building, but benchmark instead of
doris_be
- shift
- ;;
- --task-executor-simulator)
- BUILD_TASK_EXECUTOR_SIMULATOR='ON'
- BUILD_BE=1
- shift
- ;;
--spark-dpp)
BUILD_SPARK_DPP=1
shift
@@ -298,7 +277,6 @@ else
BUILD_META_TOOL='ON'
BUILD_FILE_CACHE_MICROBENCH_TOOL='OFF'
BUILD_INDEX_TOOL='ON'
- BUILD_TASK_EXECUTOR_SIMULATOR='OFF'
BUILD_HIVE_UDF=1
BUILD_BE_JAVA_EXTENSIONS=1
CLEAN=0
@@ -308,13 +286,8 @@ fi
if [[ "${HELP}" -eq 1 ]]; then
usage
fi
-# build thirdparty libraries if necessary. check last thirdparty lib
installation
-if [[ "${TARGET_SYSTEM}" == 'Darwin' ]]; then
- LAST_THIRDPARTY_LIB='libbrotlienc.a'
-else
- LAST_THIRDPARTY_LIB='hadoop_hdfs/native/libhdfs.a'
-fi
-if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" ]]; then
+# build thirdparty libraries if necessary
+if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/libbacktrace.a" ]]; then
echo "Thirdparty libraries need to be build ..."
# need remove all installed pkgs because some lib like lz4 will throw
error if its lib alreay exists
rm -rf "${DORIS_THIRDPARTY}/installed"
@@ -369,6 +342,9 @@ if [[ "${CLEAN}" -eq 1 && "${BUILD_BE}" -eq 0 &&
"${BUILD_FE}" -eq 0 && ${BUILD_
exit 0
fi
+if [[ -z "${WITH_MYSQL}" ]]; then
+ WITH_MYSQL='OFF'
+fi
if [[ -z "${GLIBC_COMPATIBILITY}" ]]; then
if [[ "${TARGET_SYSTEM}" != 'Darwin' ]]; then
GLIBC_COMPATIBILITY='ON'
@@ -392,21 +368,41 @@ fi
if [[ -z "${STRIP_DEBUG_INFO}" ]]; then
STRIP_DEBUG_INFO='OFF'
fi
+if [[ -z "${USE_MEM_TRACKER}" ]]; then
+ if [[ "${TARGET_SYSTEM}" != 'Darwin' ]]; then
+ USE_MEM_TRACKER='ON'
+ else
+ USE_MEM_TRACKER='OFF'
+ fi
+fi
BUILD_TYPE_LOWWER=$(echo "${BUILD_TYPE}" | tr '[:upper:]' '[:lower:]')
if [[ "${BUILD_TYPE_LOWWER}" == "asan" ]]; then
USE_JEMALLOC='OFF'
elif [[ -z "${USE_JEMALLOC}" ]]; then
- if [[ "${TARGET_SYSTEM}" != 'Darwin' ]]; then
- USE_JEMALLOC='ON'
- else
- USE_JEMALLOC='OFF'
- fi
+ USE_JEMALLOC='ON'
+fi
+if [[ -f "${TP_INCLUDE_DIR}/jemalloc/jemalloc_doris_with_prefix.h" ]]; then
+ # compatible with old thirdparty
+ rm -rf "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h"
+ rm -rf "${TP_LIB_DIR}/libjemalloc_doris.a"
+ rm -rf "${TP_LIB_DIR}/libjemalloc_doris_pic.a"
+ rm -rf "${TP_INCLUDE_DIR}/rocksdb"
+ rm -rf "${TP_LIB_DIR}/librocksdb.a"
+
+ mv "${TP_INCLUDE_DIR}/jemalloc/jemalloc_doris_with_prefix.h"
"${TP_INCLUDE_DIR}/jemalloc/jemalloc.h"
+ mv "${TP_LIB_DIR}/libjemalloc_doris_with_prefix.a"
"${TP_LIB_DIR}/libjemalloc_doris.a"
+ mv "${TP_LIB_DIR}/libjemalloc_doris_with_prefix_pic.a"
"${TP_LIB_DIR}/libjemalloc_doris_pic.a"
+ mv "${TP_LIB_DIR}/librocksdb_jemalloc_with_prefix.a"
"${TP_LIB_DIR}/librocksdb.a"
+ mv -f "${TP_INCLUDE_DIR}/rocksdb_jemalloc_with_prefix"
"${TP_INCLUDE_DIR}/rocksdb"
fi
-
if [[ -z "${USE_BTHREAD_SCANNER}" ]]; then
USE_BTHREAD_SCANNER='OFF'
fi
+if [[ -z "${USE_DWARF}" ]]; then
+ USE_DWARF='OFF'
+fi
+
if [[ -z "${USE_UNWIND}" ]]; then
if [[ "${TARGET_SYSTEM}" != 'Darwin' ]]; then
USE_UNWIND='ON'
@@ -447,7 +443,7 @@ if [[ -z "${DISABLE_JAVA_CHECK_STYLE}" ]]; then
DISABLE_JAVA_CHECK_STYLE='OFF'
fi
-if [[ "$(echo "${DISABLE_BUILD_AZURE}" | tr '[:lower:]' '[:upper:]')" == "ON"
]]; then
+if [[ -n "${DISABLE_BUILD_AZURE}" ]]; then
BUILD_AZURE='OFF'
fi
@@ -455,8 +451,8 @@ if [[ -z "${ENABLE_INJECTION_POINT}" ]]; then
ENABLE_INJECTION_POINT='OFF'
fi
-if [[ -z "${BUILD_BENCHMARK}" ]]; then
- BUILD_BENCHMARK='OFF'
+if [[ -z "${ENABLE_CACHE_LOCK_DEBUG}" ]]; then
+ ENABLE_CACHE_LOCK_DEBUG='ON'
fi
if [[ -z "${RECORD_COMPILER_SWITCHES}" ]]; then
@@ -494,20 +490,22 @@ echo "Get params:
BUILD_META_TOOL -- ${BUILD_META_TOOL}
BUILD_FILE_CACHE_MICROBENCH_TOOL -- ${BUILD_FILE_CACHE_MICROBENCH_TOOL}
BUILD_INDEX_TOOL -- ${BUILD_INDEX_TOOL}
- BUILD_BENCHMARK -- ${BUILD_BENCHMARK}
- BUILD_TASK_EXECUTOR_SIMULATOR -- ${BUILD_TASK_EXECUTOR_SIMULATOR}
BUILD_BE_JAVA_EXTENSIONS -- ${BUILD_BE_JAVA_EXTENSIONS}
BUILD_HIVE_UDF -- ${BUILD_HIVE_UDF}
PARALLEL -- ${PARALLEL}
CLEAN -- ${CLEAN}
+ WITH_MYSQL -- ${WITH_MYSQL}
GLIBC_COMPATIBILITY -- ${GLIBC_COMPATIBILITY}
USE_AVX2 -- ${USE_AVX2}
USE_LIBCPP -- ${USE_LIBCPP}
+ USE_DWARF -- ${USE_DWARF}
USE_UNWIND -- ${USE_UNWIND}
STRIP_DEBUG_INFO -- ${STRIP_DEBUG_INFO}
+ USE_MEM_TRACKER -- ${USE_MEM_TRACKER}
USE_JEMALLOC -- ${USE_JEMALLOC}
USE_BTHREAD_SCANNER -- ${USE_BTHREAD_SCANNER}
ENABLE_INJECTION_POINT -- ${ENABLE_INJECTION_POINT}
+ ENABLE_CACHE_LOCK_DEBUG -- ${ENABLE_CACHE_LOCK_DEBUG}
DENABLE_CLANG_COVERAGE -- ${DENABLE_CLANG_COVERAGE}
DISPLAY_BUILD_TIME -- ${DISPLAY_BUILD_TIME}
ENABLE_PCH -- ${ENABLE_PCH}
@@ -533,6 +531,9 @@ fi
# Assesmble FE modules
FE_MODULES=''
+# TODO: docs are temporarily removed, so this var is always OFF
+# Fix it later
+BUILD_DOCS='OFF'
modules=("")
if [[ "${BUILD_FE}" -eq 1 ]]; then
modules+=("fe-common")
@@ -576,10 +577,8 @@ FE_MODULES="$(
# Clean and build Backend
if [[ "${BUILD_BE}" -eq 1 ]]; then
- update_submodule "contrib/apache-orc" "apache-orc"
"https://github.com/apache/doris-thirdparty/archive/refs/heads/orc.tar.gz";
- update_submodule "contrib/clucene" "clucene"
"https://github.com/apache/doris-thirdparty/archive/refs/heads/clucene.tar.gz";
- update_submodule "contrib/openblas" "openblas"
"https://github.com/apache/doris-thirdparty/archive/refs/heads/openblas.tar.gz";
- update_submodule "contrib/faiss" "faiss"
"https://github.com/apache/doris-thirdparty/archive/refs/heads/faiss.tar.gz";
+ update_submodule "be/src/apache-orc" "apache-orc"
"https://github.com/apache/doris-thirdparty/archive/refs/heads/orc.tar.gz";
+ update_submodule "be/src/clucene" "clucene"
"https://github.com/apache/doris-thirdparty/archive/refs/heads/clucene-3.0.tar.gz";
if [[ -e "${DORIS_HOME}/gensrc/build/gen_cpp/version.h" ]]; then
rm -f "${DORIS_HOME}/gensrc/build/gen_cpp/version.h"
fi
@@ -595,20 +594,10 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then
BUILD_FS_BENCHMARK=OFF
fi
- if [[ -z "${BUILD_TASK_EXECUTOR_SIMULATOR}" ]]; then
- BUILD_TASK_EXECUTOR_SIMULATOR=OFF
- fi
-
- if [[ -z "${BUILD_FILE_CACHE_LRU_TOOL}" ]]; then
- BUILD_FILE_CACHE_LRU_TOOL=OFF
- fi
-
echo "-- Make program: ${MAKE_PROGRAM}"
echo "-- Use ccache: ${CMAKE_USE_CCACHE}"
echo "-- Extra cxx flags: ${EXTRA_CXX_FLAGS:-}"
echo "-- Build fs benchmark tool: ${BUILD_FS_BENCHMARK}"
- echo "-- Build task executor simulator: ${BUILD_TASK_EXECUTOR_SIMULATOR}"
- echo "-- Build file cache lru tool: ${BUILD_FILE_CACHE_LRU_TOOL}"
mkdir -p "${CMAKE_BUILD_DIR}"
cd "${CMAKE_BUILD_DIR}"
@@ -617,20 +606,21 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
-DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" \
-DENABLE_INJECTION_POINT="${ENABLE_INJECTION_POINT}" \
+ -DENABLE_CACHE_LOCK_DEBUG="${ENABLE_CACHE_LOCK_DEBUG}" \
-DMAKE_TEST=OFF \
- -DBUILD_BENCHMARK="${BUILD_BENCHMARK}" \
-DBUILD_FS_BENCHMARK="${BUILD_FS_BENCHMARK}" \
- -DBUILD_TASK_EXECUTOR_SIMULATOR="${BUILD_TASK_EXECUTOR_SIMULATOR}" \
- -DBUILD_FILE_CACHE_LRU_TOOL="${BUILD_FILE_CACHE_LRU_TOOL}" \
${CMAKE_USE_CCACHE:+${CMAKE_USE_CCACHE}} \
+ -DWITH_MYSQL="${WITH_MYSQL}" \
-DUSE_LIBCPP="${USE_LIBCPP}" \
-DBUILD_META_TOOL="${BUILD_META_TOOL}" \
-DBUILD_FILE_CACHE_MICROBENCH_TOOL="${BUILD_FILE_CACHE_MICROBENCH_TOOL}" \
-DBUILD_INDEX_TOOL="${BUILD_INDEX_TOOL}" \
-DSTRIP_DEBUG_INFO="${STRIP_DEBUG_INFO}" \
+ -DUSE_DWARF="${USE_DWARF}" \
-DUSE_UNWIND="${USE_UNWIND}" \
-DDISPLAY_BUILD_TIME="${DISPLAY_BUILD_TIME}" \
-DENABLE_PCH="${ENABLE_PCH}" \
+ -DUSE_MEM_TRACKER="${USE_MEM_TRACKER}" \
-DUSE_JEMALLOC="${USE_JEMALLOC}" \
-DUSE_AVX2="${USE_AVX2}" \
-DARM_MARCH="${ARM_MARCH}" \
@@ -677,6 +667,7 @@ if [[ "${BUILD_CLOUD}" -eq 1 ]]; then
-DUSE_LIBCPP="${USE_LIBCPP}" \
-DENABLE_HDFS_STORAGE_VAULT=${ENABLE_HDFS_STORAGE_VAULT:-ON} \
-DSTRIP_DEBUG_INFO="${STRIP_DEBUG_INFO}" \
+ -DUSE_DWARF="${USE_DWARF}" \
-DUSE_JEMALLOC="${USE_JEMALLOC}" \
-DEXTRA_CXX_FLAGS="${EXTRA_CXX_FLAGS}" \
-DBUILD_AZURE="${BUILD_AZURE}" \
@@ -688,6 +679,14 @@ if [[ "${BUILD_CLOUD}" -eq 1 ]]; then
echo "Build cloud done"
fi
+if [[ "${BUILD_DOCS}" = "ON" ]]; then
+ # Build docs, should be built before Frontend
+ echo "Build docs"
+ cd "${DORIS_HOME}/docs"
+ ./build_help_zip.sh
+ cd "${DORIS_HOME}"
+fi
+
function build_ui() {
NPM='npm'
if ! ${NPM} --version; then
@@ -818,6 +817,10 @@ if [[ "${OUTPUT_BE_BINARY}" -eq 1 ]]; then
cp -r -p "${DORIS_HOME}/be/output/conf"/* "${DORIS_OUTPUT}/be/conf"/
cp -r -p "${DORIS_HOME}/be/output/dict" "${DORIS_OUTPUT}/be/"
+ if [[ -d "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs/" ]]; then
+ cp -r -p "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs/"
"${DORIS_OUTPUT}/be/lib/"
+ fi
+
if [[ -f "${DORIS_THIRDPARTY}/installed/lib/libz.so" ]]; then
cp -r -p "${DORIS_THIRDPARTY}/installed/lib/libz.so"*
"${DORIS_OUTPUT}/be/lib/"
fi
@@ -863,19 +866,10 @@ EOF
cp -r -p "${DORIS_HOME}/be/output/lib/debug_info"
"${DORIS_OUTPUT}/be/lib"/
fi
- if [[ "${BUILD_BENCHMARK}" = "ON" ]]; then
- cp -r -p "${DORIS_HOME}/be/output/lib/benchmark_test"
"${DORIS_OUTPUT}/be/lib/"/
- fi
-
if [[ "${BUILD_FS_BENCHMARK}" = "ON" ]]; then
cp -r -p "${DORIS_HOME}/bin/run-fs-benchmark.sh"
"${DORIS_OUTPUT}/be/bin/"/
fi
- if [[ "${BUILD_TASK_EXECUTOR_SIMULATOR}" = "ON" ]]; then
- cp -r -p "${DORIS_HOME}/bin/run-task-executor-simulator.sh"
"${DORIS_OUTPUT}/be/bin/"/
- cp -r -p "${DORIS_HOME}/be/output/lib/task_executor_simulator"
"${DORIS_OUTPUT}/be/lib/"/
- fi
-
extensions_modules=("java-udf")
extensions_modules+=("jdbc-scanner")
extensions_modules+=("hadoop-hudi-scanner")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
