This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new cc2be36734 [VL] Customize VCPKG build features according to user's
build options (#7052)
cc2be36734 is described below
commit cc2be367342a68f730c28d92d777e3a1282d3a20
Author: PHILO-HE <[email protected]>
AuthorDate: Mon Sep 23 09:21:55 2024 +0800
[VL] Customize VCPKG build features according to user's build options
(#7052)
---
dev/builddeps-veloxbe.sh | 9 ++---
dev/vcpkg/env.sh | 22 +++++++++++-
dev/vcpkg/init.sh | 86 ++++++++++++++++++++++++++++++++++-------------
dev/vcpkg/toolchain.cmake | 13 +++++--
dev/vcpkg/vcpkg.json | 2 +-
5 files changed, 100 insertions(+), 32 deletions(-)
diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh
index 7f1ce41a11..0a07a568e7 100755
--- a/dev/builddeps-veloxbe.sh
+++ b/dev/builddeps-veloxbe.sh
@@ -171,8 +171,9 @@ function concat_velox_param {
if [ "$ENABLE_VCPKG" = "ON" ]; then
# vcpkg will install static depends and init build environment
- envs="$("$GLUTEN_DIR/dev/vcpkg/init.sh")"
- eval "$envs"
+ BUILD_OPTIONS="--build_tests=$BUILD_TESTS --enable_s3=$ENABLE_S3
--enable_gcs=$ENABLE_GCS \
+ --enable_hdfs=$ENABLE_HDFS --enable_abfs=$ENABLE_ABFS"
+ source ./dev/vcpkg/env.sh ${BUILD_OPTIONS}
fi
if [ "$SPARK_VERSION" = "3.2" ] || [ "$SPARK_VERSION" = "3.3" ] \
@@ -196,8 +197,8 @@ function build_velox {
cd $GLUTEN_DIR/ep/build-velox/src
# When BUILD_TESTS is on for gluten cpp, we need turn on
VELOX_BUILD_TEST_UTILS via build_test_utils.
./build_velox.sh --enable_s3=$ENABLE_S3 --enable_gcs=$ENABLE_GCS
--build_type=$BUILD_TYPE --enable_hdfs=$ENABLE_HDFS \
- --enable_abfs=$ENABLE_ABFS
--enable_ep_cache=$ENABLE_EP_CACHE --build_test_utils=$BUILD_TESTS
--build_tests=$BUILD_VELOX_TESTS --build_benchmarks=$BUILD_VELOX_BENCHMARKS \
- --num_threads=$NUM_THREADS
+ --enable_abfs=$ENABLE_ABFS
--enable_ep_cache=$ENABLE_EP_CACHE --build_test_utils=$BUILD_TESTS \
+ --build_tests=$BUILD_VELOX_TESTS
--build_benchmarks=$BUILD_VELOX_BENCHMARKS --num_threads=$NUM_THREADS
}
function build_gluten_cpp {
diff --git a/dev/vcpkg/env.sh b/dev/vcpkg/env.sh
index 8b247a907a..91aaa6a7d2 100755
--- a/dev/vcpkg/env.sh
+++ b/dev/vcpkg/env.sh
@@ -7,5 +7,25 @@ if [ -z "${BASH_SOURCE[0]}" ] || [ "$0" == "${BASH_SOURCE[0]}"
]; then
fi
SCRIPT_ROOT="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"
-init_vcpkg_env=$("${SCRIPT_ROOT}/init.sh")
+init_vcpkg_env=$("${SCRIPT_ROOT}/init.sh" $@)
eval "$init_vcpkg_env"
+
+export VCPKG_ROOT="$SCRIPT_ROOT/.vcpkg"
+export VCPKG="$SCRIPT_ROOT/.vcpkg/vcpkg"
+export VCPKG_TRIPLET=x64-linux-avx
+export
VCPKG_TRIPLET_INSTALL_DIR=${SCRIPT_ROOT}/vcpkg_installed/${VCPKG_TRIPLET}
+export EXPORT_TOOLS_PATH="${VCPKG_TRIPLET_INSTALL_DIR}/tools/protobuf"
+
+if [ "\${GLUTEN_VCPKG_ENABLED:-}" != "${VCPKG_ROOT}" ]; then
+ export VCPKG_ROOT=${VCPKG_ROOT}
+ export VCPKG_MANIFEST_DIR=${SCRIPT_ROOT}
+ export VCPKG_TRIPLET=${VCPKG_TRIPLET}
+
+ export CMAKE_TOOLCHAIN_FILE=${SCRIPT_ROOT}/toolchain.cmake
+ export
PKG_CONFIG_PATH=${VCPKG_TRIPLET_INSTALL_DIR}/lib/pkgconfig:${VCPKG_TRIPLET_INSTALL_DIR}/share/pkgconfig:\${PKG_CONFIG_PATH:-}
+ export PATH="${EXPORT_TOOLS_PATH}:$PATH"
+
+ export GLUTEN_VCPKG_ENABLED=${VCPKG_ROOT}
+else
+ echo "Gluten's vcpkg environment is enabled" >&2
+fi
diff --git a/dev/vcpkg/init.sh b/dev/vcpkg/init.sh
index e69aec94ab..9beaa19526 100755
--- a/dev/vcpkg/init.sh
+++ b/dev/vcpkg/init.sh
@@ -4,9 +4,44 @@ set -e
exec 3>&1 >&2
-SCRIPT_ROOT="$(realpath "$(dirname "$0")")"
-VCPKG_ROOT="$SCRIPT_ROOT/.vcpkg"
-VCPKG="$SCRIPT_ROOT/.vcpkg/vcpkg"
+BUILD_TESTS=OFF
+ENABLE_S3=OFF
+ENABLE_GCS=OFF
+ENABLE_HDFS=OFF
+ENABLE_ABFS=OFF
+
+for arg in "$@"; do
+ case $arg in
+ --build_tests=*)
+ BUILD_TESTS=("${arg#*=}")
+ shift # Remove argument name from processing
+ ;;
+ --enable_s3=*)
+ ENABLE_S3=("${arg#*=}")
+ shift # Remove argument name from processing
+ ;;
+ --enable_gcs=*)
+ ENABLE_GCS=("${arg#*=}")
+ shift # Remove argument name from processing
+ ;;
+ --enable_hdfs=*)
+ ENABLE_HDFS=("${arg#*=}")
+ shift # Remove argument name from processing
+ ;;
+ --enable_abfs=*)
+ ENABLE_ABFS=("${arg#*=}")
+ shift # Remove argument name from processing
+ ;;
+ *)
+ echo "Unrecognized argument: $arg"
+ exit 1
+ ;;
+ esac
+done
+
+export SCRIPT_ROOT="$(realpath "$(dirname "$0")")"
+export VCPKG_ROOT="$SCRIPT_ROOT/.vcpkg"
+export VCPKG="$SCRIPT_ROOT/.vcpkg/vcpkg"
VCPKG_TRIPLET=x64-linux-avx
cd "$SCRIPT_ROOT"
@@ -19,12 +54,29 @@ fi
sed -i "s/3.27.1/3.28.3/g" $VCPKG_ROOT/scripts/vcpkgTools.xml
sed -i
"s/192374a68e2971f04974a194645726196d9b8ee7abd650d1e6f65f7aa2ccc9b186c3edb473bb4958c764532edcdd42f4182ee1fcb86b17d78b0bcd6305ce3df1/bd311ca835ef0914952f21d70d1753564d58de2ede02e80ede96e78cd2f40b4189e006007643ebb37792e13edd97eb4a33810bc8aca1eab6dd428eaffe1d2e38/g"
$VCPKG_ROOT/scripts/vcpkgTools.xml
+EXTRA_FEATURES=""
+if [ "$BUILD_TESTS" = "ON" ]; then
+ EXTRA_FEATURES+="--x-feature=duckdb "
+fi
+if [ "$ENABLE_S3" = "ON" ]; then
+ EXTRA_FEATURES+="--x-feature=velox-s3 "
+fi
+if [ "$ENABLE_GCS" = "ON" ]; then
+ EXTRA_FEATURES+="--x-feature=velox-gcs "
+fi
+if [ "$ENABLE_HDFS" = "ON" ]; then
+ EXTRA_FEATURES+="--x-feature=velox-hdfs "
+fi
+if [ "$ENABLE_ABFS" = "ON" ]; then
+ EXTRA_FEATURES+="--x-feature=velox-abfs"
+fi
+
+
$VCPKG install --no-print-usage \
- --triplet="${VCPKG_TRIPLET}" --host-triplet="${VCPKG_TRIPLET}"
+ --triplet="${VCPKG_TRIPLET}" --host-triplet="${VCPKG_TRIPLET}"
${EXTRA_FEATURES}
-VCPKG_TRIPLET_INSTALL_DIR=${SCRIPT_ROOT}/vcpkg_installed/${VCPKG_TRIPLET}
-EXPORT_TOOLS_PATH=
-EXPORT_TOOLS_PATH="${VCPKG_TRIPLET_INSTALL_DIR}/tools/protobuf:${EXPORT_TOOLS_PATH}"
+export
VCPKG_TRIPLET_INSTALL_DIR=${SCRIPT_ROOT}/vcpkg_installed/${VCPKG_TRIPLET}
+EXPORT_TOOLS_PATH="${VCPKG_TRIPLET_INSTALL_DIR}/tools/protobuf"
# This scripts depends on environment $CMAKE_TOOLCHAIN_FILE, which requires
# cmake >= 3.21. If system cmake < 3.25, vcpkg will download latest cmake. We
@@ -34,7 +86,7 @@ if [ -f "$VCPKG_CMAKE_BIN_DIR/cmake" ]; then
EXPORT_TOOLS_PATH="${VCPKG_CMAKE_BIN_DIR}:${EXPORT_TOOLS_PATH}"
fi
-EXPORT_TOOLS_PATH=${EXPORT_TOOLS_PATH/%:/}
+export EXPORT_TOOLS_PATH=${EXPORT_TOOLS_PATH/%:/}
# For fixing a build error like below when gluten's build type is Debug:
# No rule to make target
'/root/gluten/dev/vcpkg/vcpkg_installed/x64-linux-avx/debug/lib/libz.a',
@@ -45,20 +97,6 @@ cp $VCPKG_TRIPLET_INSTALL_DIR/lib/libssl.a
$VCPKG_TRIPLET_INSTALL_DIR/debug/lib
cp $VCPKG_TRIPLET_INSTALL_DIR/lib/libcrypto.a
$VCPKG_TRIPLET_INSTALL_DIR/debug/lib
cp $VCPKG_TRIPLET_INSTALL_DIR/lib/liblzma.a
$VCPKG_TRIPLET_INSTALL_DIR/debug/lib
cp $VCPKG_TRIPLET_INSTALL_DIR/lib/libdwarf.a
$VCPKG_TRIPLET_INSTALL_DIR/debug/lib
-cp $VCPKG_TRIPLET_INSTALL_DIR/lib/libhdfs3.a
$VCPKG_TRIPLET_INSTALL_DIR/debug/lib
+# Allow libhdfs3.a is not installed as build option may not enable hdfs.
+cp $VCPKG_TRIPLET_INSTALL_DIR/lib/libhdfs3.a
$VCPKG_TRIPLET_INSTALL_DIR/debug/lib || true
-cat <<EOF >&3
-if [ "\${GLUTEN_VCPKG_ENABLED:-}" != "${VCPKG_ROOT}" ]; then
- export VCPKG_ROOT=${VCPKG_ROOT}
- export VCPKG_MANIFEST_DIR=${SCRIPT_ROOT}
- export VCPKG_TRIPLET=${VCPKG_TRIPLET}
-
- export CMAKE_TOOLCHAIN_FILE=${SCRIPT_ROOT}/toolchain.cmake
- export
PKG_CONFIG_PATH=${VCPKG_TRIPLET_INSTALL_DIR}/lib/pkgconfig:${VCPKG_TRIPLET_INSTALL_DIR}/share/pkgconfig:\${PKG_CONFIG_PATH:-}
- export PATH="${EXPORT_TOOLS_PATH}:\$PATH"
-
- export GLUTEN_VCPKG_ENABLED=${VCPKG_ROOT}
-else
- echo "Gluten's vcpkg environment is enabled" >&2
-fi
-EOF
diff --git a/dev/vcpkg/toolchain.cmake b/dev/vcpkg/toolchain.cmake
index 13f5560070..21ff9090fb 100644
--- a/dev/vcpkg/toolchain.cmake
+++ b/dev/vcpkg/toolchain.cmake
@@ -1,5 +1,14 @@
+# This file will be used by cmake before cmake function `project(xxx)`
+# is executed, even though it's an external cmake project.
+
set(ENABLE_GLUTEN_VCPKG ON)
-set(VCPKG_MANIFEST_DIR $ENV{VCPKG_MANIFEST_DIR})
+
+# If this arg is set, `vcpkg install` will be executed according
+# to the manifest file exists in this given path, i.e., vcpkg.json,
+# which will not respect our setting for extra features through
+# `--x-feature`.
+#set(VCPKG_MANIFEST_DIR $ENV{VCPKG_MANIFEST_DIR})
+
set(VCPKG_TARGET_TRIPLET $ENV{VCPKG_TRIPLET})
set(VCPKG_HOST_TRIPLET $ENV{VCPKG_TRIPLET})
set(VCPKG_INSTALLED_DIR $ENV{VCPKG_MANIFEST_DIR}/vcpkg_installed)
@@ -14,4 +23,4 @@ set(CMAKE_EXE_LINKER_FLAGS "-static-libstdc++ -static-libgcc")
set(CMAKE_SHARED_LINKER_FLAGS "-static-libstdc++ -static-libgcc")
# Disable boost new version warning for FindBoost module
-set(Boost_NO_WARN_NEW_VERSIONS ON)
\ No newline at end of file
+set(Boost_NO_WARN_NEW_VERSIONS ON)
diff --git a/dev/vcpkg/vcpkg.json b/dev/vcpkg/vcpkg.json
index 618293ff63..b3d8dc2fcc 100644
--- a/dev/vcpkg/vcpkg.json
+++ b/dev/vcpkg/vcpkg.json
@@ -2,7 +2,7 @@
"$schema":
"https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg.schema.json";,
"builtin-baseline": "a7b6122f6b6504d16d96117336a0562693579933",
"dependencies": ["jemalloc"],
- "default-features": ["velox", "velox-s3", "velox-gcs", "velox-hdfs",
"velox-abfs", "duckdb"],
+ "default-features": ["velox"],
"features": {
"velox": {
"description": "Velox backend",
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
