This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new a9a269d02cb branch-4.0: [fix](chore) fix no space issue when building
thirdparties (#61415)
a9a269d02cb is described below
commit a9a269d02cb4e2ec12d0c2b6ee4e496edfeea619
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Tue Mar 17 18:26:49 2026 -0700
branch-4.0: [fix](chore) fix no space issue when building thirdparties
(#61415)
part of #60730 #60296
---
.github/workflows/build-thirdparty.yml | 47 +++++++++----
thirdparty/build-thirdparty.sh | 118 +++++++++++++++++++++++++++++++++
2 files changed, 152 insertions(+), 13 deletions(-)
diff --git a/.github/workflows/build-thirdparty.yml
b/.github/workflows/build-thirdparty.yml
index 6e7a888f416..73d2be34a32 100644
--- a/.github/workflows/build-thirdparty.yml
+++ b/.github/workflows/build-thirdparty.yml
@@ -65,16 +65,17 @@ jobs:
run: |
git clone -b v7 https://github.com/easimon/maximize-build-space
- - name: Maximize build space
- uses: ./maximize-build-space
- with:
- root-reserve-mb: 4096
- swap-size-mb: 8192
- remove-dotnet: 'true'
- remove-android: 'true'
- remove-haskell: 'true'
- remove-codeql: 'true'
- remove-docker-images: 'true'
+ #- name: Maximize build space
+ # uses: ./maximize-build-space
+ # with:
+ # root-reserve-mb: 4096
+ # temp-reserve-mb: 4096
+ # swap-size-mb: 8192
+ # remove-dotnet: 'true'
+ # remove-android: 'true'
+ # remove-haskell: 'true'
+ # remove-codeql: 'true'
+ # remove-docker-images: 'true'
- name: Checkout ${{ github.ref }}
uses: actions/checkout@v4
@@ -135,14 +136,24 @@ jobs:
export CMAKE_POLICY_VERSION_MINIMUM=3.10
export CUSTOM_CMAKE="/usr/local/bin/cmake"
+ #export TMPDIR=/home/runner/work/doris/doris/.tmp
+ #export TMP=$TMPDIR
+ #export TEMP=$TMPDIR
+ #mkdir -p $TMPDIR
+
+ df -h
+ echo $TMPDIR
+ echo $RUNNER_TEMP
+
cd thirdparty
- ./build-thirdparty.sh -j "$(nproc)"
+ #./build-thirdparty.sh -j "$(nproc)"
+ ./build-thirdparty.sh -j 2
build_macos:
name: Build Third Party Libraries (macOS)
needs: changes
if: ${{ needs.changes.outputs.thirdparty_changes == 'true' }}
- runs-on: macos-13
+ runs-on: macos-15
steps:
- name: Checkout ${{ github.ref }}
uses: actions/checkout@v4
@@ -195,7 +206,12 @@ jobs:
export CMAKE_POLICY_VERSION_MINIMUM=3.10
export CUSTOM_CMAKE="/usr/local/bin/cmake"
+ df -h
+ echo $TMPDIR
+ echo $RUNNER_TEMP
+
cd thirdparty
+ #./build-thirdparty.sh -j "$(nproc)"
./build-thirdparty.sh -j "$(nproc)"
build_macos_arm64:
@@ -255,6 +271,11 @@ jobs:
export CMAKE_POLICY_VERSION_MINIMUM=3.10
export CUSTOM_CMAKE="/usr/local/bin/cmake"
+ df -h
+ echo $TMPDIR
+ echo $RUNNER_TEMP
+
cd thirdparty
- ./build-thirdparty.sh -j "$(nproc)"
+ #./build-thirdparty.sh -j "$(nproc)"
+ ./build-thirdparty.sh -j 2
diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index 8bf68daa2c1..ef0b5897402 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -2052,6 +2052,122 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
fi
fi
+# Map a package name to its source directory variable(s) and remove them to
free disk space.
+# This is called after each package is built and installed successfully.
+cleanup_package_source() {
+ local pkg="$1"
+ local src_var
+ local src_dir
+
+ # Map package name to the uppercase *_SOURCE variable name
+ case "${pkg}" in
+ libevent) src_var="LIBEVENT_SOURCE" ;;
+ openssl) src_var="OPENSSL_SOURCE" ;;
+ thrift) src_var="THRIFT_SOURCE" ;;
+ protobuf) src_var="PROTOBUF_SOURCE" ;;
+ gflags) src_var="GFLAGS_SOURCE" ;;
+ glog) src_var="GLOG_SOURCE" ;;
+ gtest) src_var="GTEST_SOURCE" ;;
+ rapidjson) src_var="RAPIDJSON_SOURCE" ;;
+ snappy) src_var="SNAPPY_SOURCE" ;;
+ gperftools) src_var="GPERFTOOLS_SOURCE" ;;
+ zlib) src_var="ZLIB_SOURCE" ;;
+ crc32c) src_var="CRC32C_SOURCE" ;;
+ lz4) src_var="LZ4_SOURCE" ;;
+ bzip) src_var="BZIP_SOURCE" ;;
+ lzo2) src_var="LZO2_SOURCE" ;;
+ zstd) src_var="ZSTD_SOURCE" ;;
+ #boost) src_var="BOOST_SOURCE" ;; // boost is used for mysql
later
+ abseil) src_var="ABSEIL_SOURCE" ;;
+ curl) src_var="CURL_SOURCE" ;;
+ re2) src_var="RE2_SOURCE" ;;
+ hyperscan)
+ # hyperscan also builds ragel, clean both
+ if [[ -n "${RAGEL_SOURCE}" && -d
"${TP_SOURCE_DIR}/${RAGEL_SOURCE}" ]]; then
+ echo "Cleaning up source: ${RAGEL_SOURCE}"
+ rm -rf "${TP_SOURCE_DIR}/${RAGEL_SOURCE}"
+ fi
+ src_var="HYPERSCAN_SOURCE"
+ ;;
+ mysql) src_var="MYSQL_SOURCE" ;;
+ odbc) src_var="ODBC_SOURCE" ;;
+ leveldb) src_var="LEVELDB_SOURCE" ;;
+ brpc) src_var="BRPC_SOURCE" ;;
+ rocksdb) src_var="ROCKSDB_SOURCE" ;;
+ cyrus_sasl) src_var="CYRUS_SASL_SOURCE" ;;
+ librdkafka) src_var="LIBRDKAFKA_SOURCE" ;;
+ flatbuffers) src_var="FLATBUFFERS_SOURCE" ;;
+ arrow) src_var="ARROW_SOURCE" ;;
+ brotli) src_var="BROTLI_SOURCE" ;;
+ cares) src_var="CARES_SOURCE" ;;
+ grpc) src_var="GRPC_SOURCE" ;;
+ s2) src_var="S2_SOURCE" ;;
+ bitshuffle) src_var="BITSHUFFLE_SOURCE" ;;
+ croaringbitmap) src_var="CROARINGBITMAP_SOURCE" ;;
+ fmt) src_var="FMT_SOURCE" ;;
+ parallel_hashmap) src_var="PARALLEL_HASHMAP_SOURCE" ;;
+ orc) src_var="ORC_SOURCE" ;;
+ cctz) src_var="CCTZ_SOURCE" ;;
+ jemalloc_doris) src_var="JEMALLOC_DORIS_SOURCE" ;;
+ libunwind) src_var="LIBUNWIND_SOURCE" ;;
+ benchmark) src_var="BENCHMARK_SOURCE" ;;
+ simdjson) src_var="SIMDJSON_SOURCE" ;;
+ nlohmann_json) src_var="NLOHMANN_JSON_SOURCE" ;;
+ libbacktrace) src_var="LIBBACKTRACE_SOURCE" ;;
+ sse2neon) src_var="SSE2NEON_SOURCE" ;;
+ xxhash) src_var="XXHASH_SOURCE" ;;
+ concurrentqueue) src_var="CONCURRENTQUEUE_SOURCE" ;;
+ fast_float) src_var="FAST_FLOAT_SOURCE" ;;
+ hadoop_libs) src_var="HADOOP_LIBS_SOURCE" ;;
+ hadoop_libs_3_4) src_var="HADOOP_LIBS_3_4_SOURCE" ;;
+ avx2neon) src_var="AVX2NEON_SOURCE" ;;
+ libdeflate) src_var="LIBDEFLATE_SOURCE" ;;
+ streamvbyte) src_var="STREAMVBYTE_SOURCE" ;;
+ ali_sdk)
+ # ali_sdk internally builds jsoncpp and libuuid, clean all three
+ for dep_var in JSONCPP_SOURCE LIBUUID_SOURCE ALI_SDK_SOURCE; do
+ dep_dir="${!dep_var}"
+ if [[ -n "${dep_dir}" && -d "${TP_SOURCE_DIR}/${dep_dir}" ]];
then
+ echo "Cleaning up source: ${dep_dir}"
+ rm -rf "${TP_SOURCE_DIR}/${dep_dir}"
+ fi
+ done
+ return
+ ;;
+ base64) src_var="BASE64_SOURCE" ;;
+ azure) src_var="AZURE_SOURCE" ;;
+ dragonbox) src_var="DRAGONBOX_SOURCE" ;;
+ icu) src_var="ICU_SOURCE" ;;
+ jindofs) src_var="JINDOFS_SOURCE" ;;
+ pugixml) src_var="PUGIXML_SOURCE" ;;
+ aws_sdk) src_var="AWS_SDK_SOURCE" ;;
+ lzma) src_var="LZMA_SOURCE" ;;
+ xml2) src_var="XML2_SOURCE" ;;
+ idn) src_var="IDN_SOURCE" ;;
+ gsasl) src_var="GSASL_SOURCE" ;;
+ krb5) src_var="KRB5_SOURCE" ;;
+ hdfs3) src_var="HDFS3_SOURCE" ;;
+ libdivide) src_var="LIBDIVIDE_SOURCE" ;;
+ binutils) src_var="BINUTILS_SOURCE" ;;
+ gettext) src_var="GETTEXT_SOURCE" ;;
+ # Header-only files, skip cleanup
+ pdqsort|timsort|tsan_header|js_and_css)
+ return
+ ;;
+ *)
+ echo "Warning: no source mapping for package '${pkg}', skipping
cleanup"
+ return
+ ;;
+ esac
+
+ src_dir="${!src_var}"
+ if [[ -n "${src_dir}" && -d "${TP_SOURCE_DIR}/${src_dir}" ]]; then
+ echo "Cleaning up source: ${src_dir}"
+ rm -rf "${TP_SOURCE_DIR}/${src_dir}"
+ fi
+}
+
+
for package in "${packages[@]}"; do
if [[ "${package}" == "${start_package}" ]]; then
PACKAGE_FOUND=1
@@ -2059,6 +2175,8 @@ for package in "${packages[@]}"; do
if [[ "${CONTINUE}" -eq 0 ]] || [[ "${PACKAGE_FOUND}" -eq 1 ]]; then
command="build_${package}"
${command}
+ cd "${TP_DIR}"
+ cleanup_package_source "${package}"
fi
done
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
