This is an automated email from the ASF dual-hosted git repository.
zhouyuan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new ca7a0a4e49 [GLUTEN-7139][VL] Manage Velox Arrow through vcpkg (#12625)
ca7a0a4e49 is described below
commit ca7a0a4e49df173949d2ea1bd7d681089783f48f
Author: Chang Chen <[email protected]>
AuthorDate: Mon Oct 5 14:29:00 2026 +0800
[GLUTEN-7139][VL] Manage Velox Arrow through vcpkg (#12625)
---
.github/workflows/velox_weekly.yml | 8 +-
cpp/CMake/ConfigArrow.cmake | 8 ++
cpp/CMake/Findglog.cmake | 71 +++++-----
cpp/CMakeLists.txt | 3 +-
cpp/core/CMakeLists.txt | 14 +-
cpp/velox/CMakeLists.txt | 50 +++----
cpp/velox/benchmarks/CMakeLists.txt | 3 +-
.../operators/writer/VeloxParquetDataSource.h | 1 -
dev/builddep-veloxbe-inc.sh | 14 +-
dev/builddeps-veloxbe.sh | 22 ++-
.../Dockerfile.almalinux8-gcc13-static-build | 3 +-
dev/docker/Dockerfile.centos7-gcc13-static-build | 5 +-
dev/docker/Dockerfile.centos7-static-build | 5 +-
dev/docker/Dockerfile.centos8-gcc13-static-build | 5 +-
dev/docker/Dockerfile.centos8-static-build | 5 +-
dev/docker/Dockerfile.centos9-static-build | 5 +-
dev/vcpkg/env.sh | 9 +-
dev/vcpkg/populate-static-image-cache.sh | 22 +++
dev/vcpkg/ports/arrow/android.patch | 27 ++++
dev/vcpkg/ports/arrow/arrow-testing-boost.patch | 64 +++++++++
.../ports/arrow/arrow-testing-static-boost.patch | 38 +++++
dev/vcpkg/ports/arrow/msvc-static-name.patch | 13 ++
dev/vcpkg/ports/arrow/portfile.cmake | 154 +++++++++++++++++++++
dev/vcpkg/ports/arrow/thrift.patch | 22 +++
dev/vcpkg/ports/arrow/usage | 4 +
dev/vcpkg/ports/arrow/usage-acero | 3 +
dev/vcpkg/ports/arrow/usage-dataset | 3 +
dev/vcpkg/ports/arrow/usage-flight | 3 +
dev/vcpkg/ports/arrow/usage-flightsql | 3 +
dev/vcpkg/ports/arrow/usage-parquet | 3 +
dev/vcpkg/ports/arrow/utf8proc.patch | 13 ++
dev/vcpkg/ports/arrow/vcpkg.json | 149 ++++++++++++++++++++
dev/vcpkg/toolchain.cmake | 25 +++-
dev/vcpkg/vcpkg-configuration.json | 6 +
dev/vcpkg/vcpkg.json | 5 +
ep/build-velox/src/build-velox.sh | 18 ++-
ep/build-velox/src/get-velox.sh | 11 --
ep/build-velox/src/setup-openeuler24.sh | 6 +-
tools/workload/benchmark_velox/initialize.ipynb | 12 --
39 files changed, 697 insertions(+), 138 deletions(-)
diff --git a/.github/workflows/velox_weekly.yml
b/.github/workflows/velox_weekly.yml
index 47e08a9c13..c4968a8c3c 100644
--- a/.github/workflows/velox_weekly.yml
+++ b/.github/workflows/velox_weekly.yml
@@ -212,7 +212,8 @@ jobs:
- uses: actions/checkout@v7
- name: Install dependencies
run: |
- dnf update -y && dnf install -y java-1.8.0-openjdk-devel sudo patch
git perl ccache curl python3
+ dnf install -y --disablerepo='update*' --disablerepo='*source*'
--disablerepo='*debuginfo*' \
+ java-1.8.0-openjdk-devel sudo patch git perl ccache curl python3
echo "JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk" >> $GITHUB_ENV
- name: Install Stash restore dependencies # the stash action needs gh
and jq, absent from this container
run: |
@@ -270,7 +271,8 @@ jobs:
- uses: actions/checkout@v7
- name: Install dependencies
run: |
- dnf update -y && dnf install -y java-1.8.0-openjdk-devel sudo patch
git perl curl python3
+ dnf install -y --disablerepo='update*' --disablerepo='*source*'
--disablerepo='*debuginfo*' \
+ java-1.8.0-openjdk-devel sudo patch git perl curl python3
echo "JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk" >> $GITHUB_ENV
- name: Install Stash restore dependencies # the stash action needs gh
and jq, absent from this container
run: |
@@ -295,7 +297,7 @@ jobs:
mkdir -p $CCACHE_DIR
cd $GITHUB_WORKSPACE/
./dev/vcpkg/setup-build-depends.sh
- ./dev/package-vcpkg.sh --build_arrow=ON --spark_version=3.5
+ ./dev/package-vcpkg.sh --spark_version=3.5
cd $GITHUB_WORKSPACE/tools/gluten-it
$GITHUB_WORKSPACE/build/mvn -ntp clean install -Pspark-3.5
- name: Clean Ccache tmp before save
diff --git a/cpp/CMake/ConfigArrow.cmake b/cpp/CMake/ConfigArrow.cmake
index 0d604a91e6..4cf061a769 100644
--- a/cpp/CMake/ConfigArrow.cmake
+++ b/cpp/CMake/ConfigArrow.cmake
@@ -20,6 +20,14 @@ set(ARROW_STATIC_LIBRARY_SUFFIX ".a")
set(ARROW_LIB_NAME "arrow")
set(ARROW_BUNDLED_DEPS "arrow_bundled_dependencies")
+if(ENABLE_GLUTEN_VCPKG)
+ find_package(Arrow CONFIG REQUIRED)
+ if(NOT TARGET Arrow::arrow)
+ add_library(Arrow::arrow INTERFACE IMPORTED)
+ target_link_libraries(Arrow::arrow INTERFACE Arrow::arrow_static)
+ endif()
+endif()
+
set(ARROW_INSTALL_DIR "${ARROW_HOME}/install")
set(ARROW_LIB_DIR "${ARROW_INSTALL_DIR}/lib")
set(ARROW_LIB64_DIR "${ARROW_INSTALL_DIR}/lib64")
diff --git a/cpp/CMake/Findglog.cmake b/cpp/CMake/Findglog.cmake
index 6d9dbdacf1..6549adc9a1 100644
--- a/cpp/CMake/Findglog.cmake
+++ b/cpp/CMake/Findglog.cmake
@@ -39,51 +39,52 @@ if(NOT glog_FOUND)
include(BuildGlog)
endif()
-get_filename_component(libglog_ext ${GLOG_LIBRARY} EXT)
-if(libglog_ext STREQUAL ".a")
- set(libglog_type STATIC)
- set(libgflags_component static)
-else()
- set(libglog_type SHARED)
- set(libgflags_component shared)
-endif()
+if(NOT TARGET glog::glog)
+ get_filename_component(libglog_ext ${GLOG_LIBRARY} EXT)
+ if(libglog_ext STREQUAL ".a")
+ set(libglog_type STATIC)
+ set(libgflags_component static)
+ else()
+ set(libglog_type SHARED)
+ set(libgflags_component shared)
+ endif()
-# glog::glog may already exist. Use google::glog to avoid conflicts.
-add_library(google::glog ${libglog_type} IMPORTED)
-set_target_properties(google::glog PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
+ add_library(glog::glog ${libglog_type} IMPORTED)
+ set_target_properties(glog::glog PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
"${GLOG_INCLUDE_DIR}")
-set_target_properties(
- google::glog PROPERTIES IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+ set_target_properties(
+ glog::glog PROPERTIES IMPORTED_LINK_INTERFACE_LANGUAGES "C"
IMPORTED_LOCATION "${GLOG_LIBRARY}")
-set(GLUTEN_GFLAGS_VERSION 2.2.2)
-find_package(gflags ${GLUTEN_GFLAGS_VERSION} CONFIG
- COMPONENTS ${libgflags_component})
-
-if(NOT gflags_FOUND AND glog_FOUND)
- message(
- FATAL_ERROR
- "Glog found but Gflags not found. Set BUILD_GLOG=ON and reload cmake.")
-endif()
+ set(GLUTEN_GFLAGS_VERSION 2.2.2)
+ find_package(gflags ${GLUTEN_GFLAGS_VERSION} CONFIG
+ COMPONENTS ${libgflags_component})
-if(gflags_FOUND)
- if(NOT TARGET gflags::gflags_${libgflags_component}
- AND NOT TARGET gflags_${libgflags_component})
+ if(NOT gflags_FOUND AND glog_FOUND)
message(
FATAL_ERROR
- "Found Gflags but missing component gflags_${libgflags_component}")
+ "Glog found but Gflags not found. Set BUILD_GLOG=ON and reload cmake.")
endif()
- if(TARGET gflags::gflags_${libgflags_component})
- set_target_properties(
- google::glog PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES
+
+ if(gflags_FOUND)
+ if(NOT TARGET gflags::gflags_${libgflags_component}
+ AND NOT TARGET gflags_${libgflags_component})
+ message(
+ FATAL_ERROR
+ "Found Gflags but missing component gflags_${libgflags_component}")
+ endif()
+ if(TARGET gflags::gflags_${libgflags_component})
+ set_target_properties(
+ glog::glog PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES
gflags::gflags_${libgflags_component})
+ else()
+ set_target_properties(
+ glog::glog PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES
+ gflags_${libgflags_component})
+ endif()
else()
+ include(BuildGflags)
set_target_properties(
- google::glog PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES
- gflags_${libgflags_component})
+ glog::glog PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES gflags_static)
endif()
-else()
- include(BuildGflags)
- set_target_properties(
- google::glog PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES gflags_static)
endif()
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index a441b0e56c..f732d9da44 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -80,6 +80,7 @@ endif()
set(GLUTEN_PREFIX_INCLUDE_DIRS)
if(CMAKE_INSTALL_PREFIX
+ AND NOT ENABLE_GLUTEN_VCPKG
AND NOT CMAKE_INSTALL_PREFIX STREQUAL "/usr/local"
AND IS_DIRECTORY "${CMAKE_INSTALL_PREFIX}/include")
list(APPEND GLUTEN_PREFIX_INCLUDE_DIRS "${CMAKE_INSTALL_PREFIX}/include")
@@ -247,7 +248,7 @@ function(ADD_TEST_CASE TEST_NAME)
endif()
add_executable(${TEST_NAME} ${SOURCES})
- target_link_libraries(${TEST_NAME} gluten google::glog GTest::gtest
+ target_link_libraries(${TEST_NAME} gluten glog::glog GTest::gtest
GTest::gtest_main Threads::Threads)
target_include_directories(${TEST_NAME} PRIVATE ${CMAKE_SOURCE_DIR}/core)
diff --git a/cpp/core/CMakeLists.txt b/cpp/core/CMakeLists.txt
index e0d82d4204..66c800595c 100644
--- a/cpp/core/CMakeLists.txt
+++ b/cpp/core/CMakeLists.txt
@@ -199,8 +199,10 @@ if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0)
endif()
endif()
-find_arrow_lib(${ARROW_LIB_NAME})
-find_arrow_lib(${ARROW_BUNDLED_DEPS})
+if(NOT ENABLE_GLUTEN_VCPKG)
+ find_arrow_lib(${ARROW_LIB_NAME})
+ find_arrow_lib(${ARROW_BUNDLED_DEPS})
+endif()
if(ENABLE_QAT)
include(BuildQATzip)
@@ -249,9 +251,11 @@ if(BUILD_TESTS)
add_subdirectory(tests)
endif()
-target_link_libraries(gluten PUBLIC Arrow::arrow
- Arrow::arrow_bundled_dependencies)
-target_link_libraries(gluten PRIVATE google::glog)
+target_link_libraries(gluten PUBLIC Arrow::arrow)
+if(NOT ENABLE_GLUTEN_VCPKG)
+ target_link_libraries(gluten PUBLIC Arrow::arrow_bundled_dependencies)
+endif()
+target_link_libraries(gluten PRIVATE glog::glog)
install(TARGETS gluten DESTINATION ${CMAKE_INSTALL_LIBDIR})
diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt
index 3926b0f252..550b1c92bf 100644
--- a/cpp/velox/CMakeLists.txt
+++ b/cpp/velox/CMakeLists.txt
@@ -247,6 +247,11 @@ endif()
add_library(velox SHARED ${VELOX_SRCS})
+if(ENABLE_GLUTEN_VCPKG)
+ find_package(xxHash CONFIG REQUIRED)
+ target_link_libraries(velox PUBLIC xxHash::xxhash)
+endif()
+
# Keep ABI sensitive Velox feature macros in sync
set(GLUTEN_VELOX_CMAKE_CACHE "${VELOX_BUILD_PATH}/CMakeCache.txt")
set(GLUTEN_VELOX_ENABLE_PARQUET ${VELOX_ENABLE_PARQUET})
@@ -336,6 +341,7 @@ if(BUILD_TESTS)
import_library(
facebook::velox::vector_test_lib
${VELOX_BUILD_PATH}/velox/vector/tests/utils/libvelox_vector_test_lib.a)
+ target_link_libraries(facebook::velox::vector_test_lib INTERFACE
GTest::gtest)
import_library(
facebook::velox::dwio_common_test
${VELOX_BUILD_PATH}/velox/dwio/common/tests/utils/libvelox_dwio_common_test_utils.a
@@ -343,12 +349,28 @@ if(BUILD_TESTS)
import_library(
facebook::velox::file_test_utils
${VELOX_BUILD_PATH}/velox/common/file/tests/libvelox_file_test_utils.a)
+ import_library(
+ facebook::velox::vector_fuzzer_util
+ ${VELOX_BUILD_PATH}/velox/vector/fuzzer/libvelox_vector_fuzzer_util.a)
+ import_library(
+ facebook::velox::constrained_vector_generator
+
${VELOX_BUILD_PATH}/velox/vector/fuzzer/libvelox_constrained_vector_generator.a
+ )
+ target_link_libraries(facebook::velox::constrained_vector_generator
+ INTERFACE facebook::velox::vector_fuzzer_util)
+ import_library(
+ facebook::velox::vector_fuzzer
+ ${VELOX_BUILD_PATH}/velox/vector/fuzzer/libvelox_vector_fuzzer.a)
+ target_link_libraries(
+ facebook::velox::vector_fuzzer
+ INTERFACE facebook::velox::vector_fuzzer_util
+ facebook::velox::constrained_vector_generator)
import_library(
facebook::velox::exec_test_lib
${VELOX_BUILD_PATH}/velox/exec/tests/utils/libvelox_exec_test_lib.a)
target_link_libraries(
facebook::velox::exec_test_lib
- INTERFACE facebook::velox::vector_test_lib
+ INTERFACE facebook::velox::vector_test_lib facebook::velox::vector_fuzzer
facebook::velox::dwio_common_test
facebook::velox::file_test_utils)
target_link_libraries(velox PUBLIC facebook::velox::exec_test_lib)
@@ -423,29 +445,9 @@ endif()
target_link_libraries(velox PUBLIC Folly::folly)
# Link thrift libraries - check vcpkg first, then fall back to system libraries
-# Determine vcpkg triplet directory based on architecture
-if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
- set(VCPKG_TRIPLET_DIR "x64-linux-avx")
-elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch64)|(arm64)")
- set(VCPKG_TRIPLET_DIR "arm64-linux-neon")
-else()
- message(
- WARNING
- "Unknown processor type for vcpkg: ${CMAKE_SYSTEM_PROCESSOR}, defaulting
to x64-linux-avx"
- )
- set(VCPKG_TRIPLET_DIR "x64-linux-avx")
-endif()
-
-if(DEFINED VCPKG_INSTALLED_DIR
- AND EXISTS
"${VCPKG_INSTALLED_DIR}/${VCPKG_TRIPLET_DIR}/lib/libthriftcpp2.a")
- message(
- STATUS
- "Using vcpkg thrift libraries from
${VCPKG_INSTALLED_DIR}/${VCPKG_TRIPLET_DIR}"
- )
- target_link_libraries(
- velox
- PRIVATE ${VCPKG_INSTALLED_DIR}/${VCPKG_TRIPLET_DIR}/lib/libthriftcpp2.a
-
${VCPKG_INSTALLED_DIR}/${VCPKG_TRIPLET_DIR}/lib/libthriftprotocol.a)
+if(ENABLE_GLUTEN_VCPKG)
+ find_package(FBThrift CONFIG REQUIRED)
+ target_link_libraries(velox PUBLIC FBThrift::thriftcpp2)
else()
set(THRIFT_LIBRARY_DIRS)
if(CMAKE_INSTALL_PREFIX)
diff --git a/cpp/velox/benchmarks/CMakeLists.txt
b/cpp/velox/benchmarks/CMakeLists.txt
index 617ada6171..bb1c9d2d90 100644
--- a/cpp/velox/benchmarks/CMakeLists.txt
+++ b/cpp/velox/benchmarks/CMakeLists.txt
@@ -19,8 +19,7 @@ target_include_directories(
velox_benchmark_common PUBLIC ${CMAKE_SOURCE_DIR}/velox
${CMAKE_SOURCE_DIR}/core)
target_link_libraries(velox_benchmark_common PUBLIC velox benchmark::benchmark
- google::glog)
-
+ glog::glog)
function(add_velox_benchmark BM_EXEC BM_FILE)
add_executable(${BM_EXEC} ${BM_FILE})
target_link_libraries(${BM_EXEC} PRIVATE velox_benchmark_common)
diff --git a/cpp/velox/operators/writer/VeloxParquetDataSource.h
b/cpp/velox/operators/writer/VeloxParquetDataSource.h
index dac97168d5..cf51df1d5d 100644
--- a/cpp/velox/operators/writer/VeloxParquetDataSource.h
+++ b/cpp/velox/operators/writer/VeloxParquetDataSource.h
@@ -17,7 +17,6 @@
#pragma once
-#include <arrow/filesystem/filesystem.h>
#include <arrow/memory_pool.h>
#include <arrow/record_batch.h>
#include <arrow/type.h>
diff --git a/dev/builddep-veloxbe-inc.sh b/dev/builddep-veloxbe-inc.sh
index 9869690ca9..b40a82e7af 100755
--- a/dev/builddep-veloxbe-inc.sh
+++ b/dev/builddep-veloxbe-inc.sh
@@ -130,19 +130,7 @@ if [ "$UPDATE_VCPKG" = "ON" ]; then
echo "[Step 1/4] vcpkg dependencies updated."
else
step 1 "Skipping vcpkg check (use --update_vcpkg to update)"
- VCPKG_ROOT="$GLUTEN_DIR/dev/vcpkg/.vcpkg"
- export VCPKG_ROOT
-
- if [ "${CPU_TARGET:-}" = "aarch64" ]; then
- export VCPKG_TRIPLET="arm64-linux-neon"
- else
- export VCPKG_TRIPLET="x64-linux-avx"
- fi
-
- export
VCPKG_TRIPLET_INSTALL_DIR="$GLUTEN_DIR/dev/vcpkg/vcpkg_installed/${VCPKG_TRIPLET}"
- export CMAKE_TOOLCHAIN_FILE="$GLUTEN_DIR/dev/vcpkg/toolchain.cmake"
- export
PKG_CONFIG_PATH="${VCPKG_TRIPLET_INSTALL_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH:-}"
- export GLUTEN_VCPKG_ENABLED="${VCPKG_ROOT}"
+ source "$GLUTEN_DIR/dev/vcpkg/env.sh" --skip-install
fi
# Step 2: Build Velox
diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh
index 91ecc343bc..11f351b787 100755
--- a/dev/builddeps-veloxbe.sh
+++ b/dev/builddeps-veloxbe.sh
@@ -46,6 +46,7 @@ VELOX_BRANCH=""
VELOX_HOME="$GLUTEN_DIR/ep/build-velox/build/velox_ep"
VELOX_PARAMETER=""
BUILD_ARROW=ON
+BUILD_ARROW_EXPLICIT=OFF
SPARK_VERSION=ALL
# set default number of threads as cpu cores minus 2
@@ -146,6 +147,7 @@ do
;;
--build_arrow=*)
BUILD_ARROW="${arg#*=}"
+ BUILD_ARROW_EXPLICIT=ON
shift # Remove argument name from processing
;;
--num_threads=*)
@@ -163,6 +165,18 @@ do
esac
done
+function vcpkg_is_active {
+ [ "$ENABLE_VCPKG" = "ON" ] || [ -n "${GLUTEN_VCPKG_ENABLED:-}" ]
+}
+
+if vcpkg_is_active; then
+ if [ "$BUILD_ARROW_EXPLICIT" = "ON" ] && [ "$BUILD_ARROW" = "ON" ]; then
+ echo "ERROR: --build_arrow=ON is deprecated with --enable_vcpkg=ON;
Arrow is managed by Gluten vcpkg." >&2
+ exit 1
+ fi
+ BUILD_ARROW=OFF
+fi
+
if [[ "$(uname)" == "Darwin" ]]; then
export INSTALL_PREFIX=${INSTALL_PREFIX:-${VELOX_HOME}/deps-install}
if [[ "$INSTALL_PREFIX" == "/usr/local" || "$INSTALL_PREFIX" ==
/usr/local/* ]]; then
@@ -228,6 +242,10 @@ concat_velox_param
export VELOX_HOME
function build_arrow {
+ if vcpkg_is_active; then
+ echo "ERROR: build_arrow is deprecated with --enable_vcpkg=ON; Arrow is
managed by Gluten vcpkg." >&2
+ return 1
+ fi
local GLUTEN_BUILD_TYPE="$BUILD_TYPE"
if [ ! -d "$VELOX_HOME" ]; then
get_velox
@@ -279,8 +297,10 @@ function build_gluten_cpp {
)
if [ -n "${INSTALL_PREFIX:-}" ]; then
- GLUTEN_CMAKE_OPTIONS+=("-DCMAKE_PREFIX_PATH=$INSTALL_PREFIX")
GLUTEN_CMAKE_OPTIONS+=("-DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX")
+ if [ -z "${GLUTEN_VCPKG_ENABLED:-}" ]; then
+ GLUTEN_CMAKE_OPTIONS+=("-DCMAKE_PREFIX_PATH=$INSTALL_PREFIX")
+ fi
fi
if [ $OS == 'Darwin' ]; then
if [[ -n "${INSTALL_PREFIX:-}" && "${INSTALL_PREFIX:-}" != "/usr/local" &&
"${INSTALL_PREFIX:-}" != /usr/local/* ]]; then
diff --git a/dev/docker/Dockerfile.almalinux8-gcc13-static-build
b/dev/docker/Dockerfile.almalinux8-gcc13-static-build
index cee3d40e94..f41c7ecdd1 100644
--- a/dev/docker/Dockerfile.almalinux8-gcc13-static-build
+++ b/dev/docker/Dockerfile.almalinux8-gcc13-static-build
@@ -67,8 +67,7 @@ RUN set -ex; \
source /opt/rh/gcc-toolset-13/enable; \
cd /opt/gluten; \
export SSL_VERIFY=false; \
- bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON
--enable_s3=ON --enable_gcs=ON \
- --enable_hdfs=ON --enable_abfs=ON
build_arrow; \
+ bash ./dev/vcpkg/populate-static-image-cache.sh; \
./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta
-Pspark-3.5 -DskipTests; \
rm -rf /opt/gluten; \
rm -rf /root/.cache/ccache;
diff --git a/dev/docker/Dockerfile.centos7-gcc13-static-build
b/dev/docker/Dockerfile.centos7-gcc13-static-build
index 3083b96536..f3292c1c81 100644
--- a/dev/docker/Dockerfile.centos7-gcc13-static-build
+++ b/dev/docker/Dockerfile.centos7-gcc13-static-build
@@ -46,10 +46,9 @@ RUN set -ex; \
yum remove gcc -y && yum clean all; \
yes | cp -rf /usr/share/aclocal/* /usr/local/share/aclocal/; \
mkdir -p ${VCPKG_PATH}; \
- echo "Build arrow, then install the native libs to system paths and jar
package to .m2/ directory."; \
+ echo "vcpkg installs the native dependencies and Maven resolves Java
dependencies."; \
cd /opt/gluten; \
- bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON
--enable_s3=ON --enable_gcs=ON \
- --enable_hdfs=ON --enable_abfs=ON
build_arrow; \
+ bash ./dev/vcpkg/populate-static-image-cache.sh; \
./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta
-Pspark-3.5 -DskipTests; \
dnf clean all; \
rm -rf /opt/gluten; \
diff --git a/dev/docker/Dockerfile.centos7-static-build
b/dev/docker/Dockerfile.centos7-static-build
index 923636961c..25b7d798e8 100644
--- a/dev/docker/Dockerfile.centos7-static-build
+++ b/dev/docker/Dockerfile.centos7-static-build
@@ -42,12 +42,11 @@ RUN set -ex; \
echo "check_certificate = off" >> ~/.wgetrc; \
cd /opt/gluten && bash ./dev/vcpkg/setup-build-depends.sh; \
mkdir -p ${VCPKG_PATH}; \
- echo "Build arrow, then install the native libs to system paths and jar
package to .m2/ directory."; \
+ echo "vcpkg installs the native dependencies and Maven resolves Java
dependencies."; \
cd /opt/gluten; \
source /opt/rh/devtoolset-11/enable; \
source /opt/rh/rh-git227/enable; \
- bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON
--enable_s3=ON --enable_gcs=ON \
- --enable_hdfs=ON --enable_abfs=ON
build_arrow; \
+ bash ./dev/vcpkg/populate-static-image-cache.sh; \
./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta
-Pspark-3.5 -DskipTests; \
dnf clean all; \
rm -rf /opt/gluten; \
diff --git a/dev/docker/Dockerfile.centos8-gcc13-static-build
b/dev/docker/Dockerfile.centos8-gcc13-static-build
index 3d608dbe0c..47ee6e3d39 100644
--- a/dev/docker/Dockerfile.centos8-gcc13-static-build
+++ b/dev/docker/Dockerfile.centos8-gcc13-static-build
@@ -39,14 +39,13 @@ RUN set -ex; \
dnf clean all; \
git clone --depth=1 --branch "${GLUTEN_BRANCH}" "${GLUTEN_REPO}"
/opt/gluten; \
mkdir -p ${VCPKG_PATH}; \
- echo "Build arrow, then install the native libs to system paths and jar
package to .m2/ directory."; \
+ echo "vcpkg installs the native dependencies and Maven resolves Java
dependencies."; \
if [ "$(uname -m)" = "aarch64" ]; then \
export CPU_TARGET="aarch64"; \
export VCPKG_FORCE_SYSTEM_BINARIES=1; \
fi; \
cd /opt/gluten; \
- bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON
--enable_s3=ON --enable_gcs=ON \
- --enable_hdfs=ON --enable_abfs=ON
build_arrow; \
+ bash ./dev/vcpkg/populate-static-image-cache.sh; \
./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta
-Pspark-3.5 -DskipTests; \
rm -rf /opt/gluten; \
rm -rf /root/.cache/ccache;
diff --git a/dev/docker/Dockerfile.centos8-static-build
b/dev/docker/Dockerfile.centos8-static-build
index f689af54a9..c4e77d3c1f 100644
--- a/dev/docker/Dockerfile.centos8-static-build
+++ b/dev/docker/Dockerfile.centos8-static-build
@@ -39,7 +39,7 @@ RUN set -ex; \
git clone --depth=1 --branch "${GLUTEN_BRANCH}" "${GLUTEN_REPO}"
/opt/gluten; \
cd /opt/gluten && bash ./dev/vcpkg/setup-build-depends.sh; \
mkdir -p ${VCPKG_PATH}; \
- echo "Build arrow, then install the native libs to system paths and jar
package to .m2/ directory."; \
+ echo "vcpkg installs the native dependencies and Maven resolves Java
dependencies."; \
if [ "$(uname -m)" = "aarch64" ]; then \
export CPU_TARGET="aarch64"; \
export VCPKG_FORCE_SYSTEM_BINARIES=1; \
@@ -47,8 +47,7 @@ RUN set -ex; \
source /opt/rh/gcc-toolset-11/enable; \
cd /opt/gluten; \
export SSL_VERIFY=false; \
- bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON
--enable_s3=ON --enable_gcs=ON \
- --enable_hdfs=ON --enable_abfs=ON
build_arrow; \
+ bash ./dev/vcpkg/populate-static-image-cache.sh; \
./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta
-Pspark-3.5 -DskipTests; \
rm -rf /opt/gluten; \
rm -rf /root/.cache/ccache;
diff --git a/dev/docker/Dockerfile.centos9-static-build
b/dev/docker/Dockerfile.centos9-static-build
index b8581cbe65..438e333d80 100644
--- a/dev/docker/Dockerfile.centos9-static-build
+++ b/dev/docker/Dockerfile.centos9-static-build
@@ -35,15 +35,14 @@ RUN set -ex; \
git clone --depth=1 --branch "${GLUTEN_BRANCH}" "${GLUTEN_REPO}"
/opt/gluten; \
cd /opt/gluten && bash ./dev/vcpkg/setup-build-depends.sh; \
mkdir -p ${VCPKG_PATH}; \
- echo "Build arrow, then install the native libs to system paths and jar
package to .m2/ directory."; \
+ echo "vcpkg installs the native dependencies and Maven resolves Java
dependencies."; \
if [ "$(uname -m)" = "aarch64" ]; then \
export CPU_TARGET="aarch64"; \
export VCPKG_FORCE_SYSTEM_BINARIES=1; \
fi; \
source /opt/rh/gcc-toolset-12/enable; \
cd /opt/gluten; \
- bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON
--enable_s3=ON --enable_gcs=ON \
- --enable_hdfs=ON --enable_abfs=ON
build_arrow; \
+ bash ./dev/vcpkg/populate-static-image-cache.sh; \
./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta
-Pspark-3.5 -DskipTests; \
rm -rf /opt/gluten; \
rm -rf /root/.cache/ccache;
diff --git a/dev/vcpkg/env.sh b/dev/vcpkg/env.sh
index a86a953e2b..d2cfc3739d 100755
--- a/dev/vcpkg/env.sh
+++ b/dev/vcpkg/env.sh
@@ -29,7 +29,12 @@ export VCPKG="$SCRIPT_ROOT/.vcpkg/vcpkg"
export VCPKG_TRIPLET=$([ "${CPU_TARGET:-}" = "aarch64" ] && echo
"arm64-linux-neon" || echo "x64-linux-avx")
export
VCPKG_TRIPLET_INSTALL_DIR=${SCRIPT_ROOT}/vcpkg_installed/${VCPKG_TRIPLET}
-${SCRIPT_ROOT}/init.sh "$@"
+unset PKG_CONFIG_PATH
+export
PKG_CONFIG_LIBDIR=${VCPKG_TRIPLET_INSTALL_DIR}/lib/pkgconfig:${VCPKG_TRIPLET_INSTALL_DIR}/share/pkgconfig
+
+if [ "${1:-}" != "--skip-install" ]; then
+ ${SCRIPT_ROOT}/init.sh "$@"
+fi
if [ "${GLUTEN_VCPKG_ENABLED:-}" != "${VCPKG_ROOT}" ]; then
EXPORT_TOOLS_PATH="${VCPKG_TRIPLET_INSTALL_DIR}/tools/protobuf"
@@ -45,9 +50,7 @@ if [ "${GLUTEN_VCPKG_ENABLED:-}" != "${VCPKG_ROOT}" ]; then
export VCPKG_ROOT=${VCPKG_ROOT}
export VCPKG_MANIFEST_DIR=${SCRIPT_ROOT}
export VCPKG_TRIPLET=${VCPKG_TRIPLET}
-
export CMAKE_TOOLCHAIN_FILE=${SCRIPT_ROOT}/toolchain.cmake
- export
PKG_CONFIG_PATH=${VCPKG_TRIPLET_INSTALL_DIR}/lib/pkgconfig:${VCPKG_TRIPLET_INSTALL_DIR}/share/pkgconfig:${PKG_CONFIG_PATH:-}
export PATH="${EXPORT_TOOLS_PATH}:$PATH"
export GLUTEN_VCPKG_ENABLED=${VCPKG_ROOT}
diff --git a/dev/vcpkg/populate-static-image-cache.sh
b/dev/vcpkg/populate-static-image-cache.sh
new file mode 100755
index 0000000000..4b2061c205
--- /dev/null
+++ b/dev/vcpkg/populate-static-image-cache.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+SCRIPT_ROOT="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"
+source "$SCRIPT_ROOT/env.sh" --build_tests=ON --enable_s3=ON --enable_gcs=ON \
+ --enable_hdfs=ON --enable_abfs=ON
diff --git a/dev/vcpkg/ports/arrow/android.patch
b/dev/vcpkg/ports/arrow/android.patch
new file mode 100644
index 0000000000..4eb412d1a5
--- /dev/null
+++ b/dev/vcpkg/ports/arrow/android.patch
@@ -0,0 +1,27 @@
+diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
+index 6dc8358..2b91efa 100644
+--- a/cpp/src/arrow/CMakeLists.txt
++++ b/cpp/src/arrow/CMakeLists.txt
+@@ -166,7 +166,7 @@ if(WIN32)
+ list(APPEND ARROW_SYSTEM_LINK_LIBS "ws2_32")
+ endif()
+
+-if(NOT WIN32 AND NOT APPLE)
++if(NOT WIN32 AND NOT APPLE AND NOT ANDROID)
+ # Pass -lrt on Linux only
+ list(APPEND ARROW_SYSTEM_LINK_LIBS rt)
+ endif()
+diff --git a/cpp/src/arrow/vendored/musl/strptime.c
b/cpp/src/arrow/vendored/musl/strptime.c
+index 41912fd..0ea36e9 100644
+--- a/cpp/src/arrow/vendored/musl/strptime.c
++++ b/cpp/src/arrow/vendored/musl/strptime.c
+@@ -18,7 +18,9 @@
+ #undef HAVE_LANGINFO
+
+ #ifndef _WIN32
++# if !(defined(__ANDROID__) && __ANDROID_API__ < 26)
+ #define HAVE_LANGINFO 1
++# endif
+ #endif
+
+ #ifdef HAVE_LANGINFO
diff --git a/dev/vcpkg/ports/arrow/arrow-testing-boost.patch
b/dev/vcpkg/ports/arrow/arrow-testing-boost.patch
new file mode 100644
index 0000000000..a3ef926756
--- /dev/null
+++ b/dev/vcpkg/ports/arrow/arrow-testing-boost.patch
@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This patch replaces Boost::process (header-only, no CMake target) with
+# Boost::filesystem and Boost::system (the actual libraries it would have
linked)
+# Based on https://github.com/apache/arrow/pull/45424
+
+diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake
b/cpp/cmake_modules/ThirdpartyToolchain.cmake
+index db151b4..1da99f1 100644
+--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
++++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
+@@ -1266,13 +1266,19 @@ endif()
+ # - Gandiva has a compile-time (header-only) dependency on Boost, not runtime.
+ # - Tests need Boost at runtime.
+ # - S3FS and Flight benchmarks need Boost at runtime.
++# - arrow_testing uses boost::filesystem. So arrow_testing requires
++# Boost library. (boost::filesystem isn't header-only.) But if we
++# use arrow_testing as a static library without
++# using arrow::util::Process, we don't need boost::filesystem.
+ if(ARROW_BUILD_INTEGRATION
+ OR ARROW_BUILD_TESTS
+ OR (ARROW_FLIGHT AND (ARROW_TESTING OR ARROW_BUILD_BENCHMARKS))
+- OR (ARROW_S3 AND ARROW_BUILD_BENCHMARKS))
++ OR (ARROW_S3 AND ARROW_BUILD_BENCHMARKS)
++ OR (ARROW_TESTING AND ARROW_BUILD_SHARED))
+ set(ARROW_USE_BOOST TRUE)
+ set(ARROW_BOOST_REQUIRE_LIBRARY TRUE)
+ elseif(ARROW_GANDIVA
++ OR ARROW_TESTING
+ OR ARROW_WITH_THRIFT
+ OR (NOT ARROW_USE_NATIVE_INT128))
+ set(ARROW_USE_BOOST TRUE)
+diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
+index c911f0f..84673d4 100644
+--- a/cpp/src/arrow/CMakeLists.txt
++++ b/cpp/src/arrow/CMakeLists.txt
+@@ -645,11 +645,11 @@ else()
+ endif()
+
+ set(ARROW_TESTING_SHARED_LINK_LIBS arrow_shared ${ARROW_GTEST_GTEST})
+-set(ARROW_TESTING_SHARED_PRIVATE_LINK_LIBS arrow::flatbuffers RapidJSON
Boost::process)
++set(ARROW_TESTING_SHARED_PRIVATE_LINK_LIBS arrow::flatbuffers RapidJSON
Boost::filesystem Boost::system)
+ set(ARROW_TESTING_STATIC_LINK_LIBS
+ arrow::flatbuffers
+ RapidJSON
+- Boost::process
++ Boost::filesystem Boost::system
+ arrow_static
+ ${ARROW_GTEST_GTEST})
+ set(ARROW_TESTING_SHARED_INSTALL_INTERFACE_LIBS Arrow::arrow_shared)
diff --git a/dev/vcpkg/ports/arrow/arrow-testing-static-boost.patch
b/dev/vcpkg/ports/arrow/arrow-testing-static-boost.patch
new file mode 100644
index 0000000000..c87382699c
--- /dev/null
+++ b/dev/vcpkg/ports/arrow/arrow-testing-static-boost.patch
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake
b/cpp/cmake_modules/ThirdpartyToolchain.cmake
+--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
++++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
+@@ -1267,15 +1267,12 @@ endif()
+ # - Gandiva has a compile-time (header-only) dependency on Boost, not runtime.
+ # - Tests need Boost at runtime.
+ # - S3FS and Flight benchmarks need Boost at runtime.
+-# - arrow_testing uses boost::filesystem. So arrow_testing requires
+-# Boost library. (boost::filesystem isn't header-only.) But if we
+-# use arrow_testing as a static library without
+-# using arrow::util::Process, we don't need boost::filesystem.
++# - arrow_testing directly links Boost::filesystem and Boost::system.
+ if(ARROW_BUILD_INTEGRATION
+ OR ARROW_BUILD_TESTS
+ OR (ARROW_FLIGHT AND (ARROW_TESTING OR ARROW_BUILD_BENCHMARKS))
+ OR (ARROW_S3 AND ARROW_BUILD_BENCHMARKS)
+- OR (ARROW_TESTING AND ARROW_BUILD_SHARED))
++ OR ARROW_TESTING)
+ set(ARROW_USE_BOOST TRUE)
+ set(ARROW_BOOST_REQUIRE_LIBRARY TRUE)
+ elseif(ARROW_GANDIVA
diff --git a/dev/vcpkg/ports/arrow/msvc-static-name.patch
b/dev/vcpkg/ports/arrow/msvc-static-name.patch
new file mode 100644
index 0000000000..61d99ebac5
--- /dev/null
+++ b/dev/vcpkg/ports/arrow/msvc-static-name.patch
@@ -0,0 +1,13 @@
+diff --git a/cpp/cmake_modules/BuildUtils.cmake
b/cpp/cmake_modules/BuildUtils.cmake
+index 391c43e0a..50f6d3d3c 100644
+--- a/cpp/cmake_modules/BuildUtils.cmake
++++ b/cpp/cmake_modules/BuildUtils.cmake
+@@ -427,7 +427,7 @@ function(ADD_ARROW_LIB LIB_NAME)
+ target_include_directories(${LIB_NAME}_static PRIVATE
${ARG_PRIVATE_INCLUDES})
+ endif()
+
+- if(MSVC_TOOLCHAIN)
++ if(MSVC_TOOLCHAIN AND 0)
+ set(LIB_NAME_STATIC ${LIB_NAME}_static)
+ else()
+ set(LIB_NAME_STATIC ${LIB_NAME})
diff --git a/dev/vcpkg/ports/arrow/portfile.cmake
b/dev/vcpkg/ports/arrow/portfile.cmake
new file mode 100644
index 0000000000..993c27dbed
--- /dev/null
+++ b/dev/vcpkg/ports/arrow/portfile.cmake
@@ -0,0 +1,154 @@
+vcpkg_download_distfile(
+ ARCHIVE_PATH
+ URLS
"https://archive.apache.org/dist/arrow/arrow-${VERSION}/apache-arrow-${VERSION}.tar.gz";
+ FILENAME apache-arrow-${VERSION}.tar.gz
+ SHA512
4df30ab5561da695eaa864422626b9898555d86ca56835c3b8a8ca93a1dbaf081582bb36e2440d1daf7e1dd48c76941f1152a4f25ce0dbcc1c2abe244a00c05e
+)
+vcpkg_extract_source_archive(
+ SOURCE_PATH
+ ARCHIVE ${ARCHIVE_PATH}
+ PATCHES
+ # Patches inherited from the vcpkg Arrow 18.1.0 port.
+ android.patch
+ msvc-static-name.patch
+ utf8proc.patch
+ thrift.patch
+ # Patches required by Velox's static arrow_testing dependency.
+ arrow-testing-boost.patch
+ arrow-testing-static-boost.patch
+)
+
+vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS
+ FEATURES
+ acero ARROW_ACERO
+ compute ARROW_COMPUTE
+ csv ARROW_CSV
+ cuda ARROW_CUDA
+ dataset ARROW_DATASET
+ filesystem ARROW_FILESYSTEM
+ flight ARROW_FLIGHT
+ flightsql ARROW_FLIGHT_SQL
+ gcs ARROW_GCS
+ jemalloc ARROW_JEMALLOC
+ json ARROW_JSON
+ mimalloc ARROW_MIMALLOC
+ orc ARROW_ORC
+ parquet ARROW_PARQUET
+ parquet PARQUET_REQUIRE_ENCRYPTION
+ s3 ARROW_S3
+ testing ARROW_TESTING
+)
+
+if(VCPKG_TARGET_IS_WINDOWS AND NOT VCPKG_TARGET_IS_MINGW)
+ list(APPEND FEATURE_OPTIONS "-DARROW_USE_NATIVE_INT128=OFF")
+endif()
+
+string(COMPARE EQUAL ${VCPKG_LIBRARY_LINKAGE} "dynamic" ARROW_BUILD_SHARED)
+string(COMPARE EQUAL ${VCPKG_LIBRARY_LINKAGE} "static" ARROW_BUILD_STATIC)
+string(COMPARE EQUAL ${VCPKG_LIBRARY_LINKAGE} "dynamic"
ARROW_DEPENDENCY_USE_SHARED)
+
+vcpkg_cmake_configure(
+ SOURCE_PATH "${SOURCE_PATH}/cpp"
+ OPTIONS
+ ${FEATURE_OPTIONS}
+ -DARROW_BUILD_SHARED=${ARROW_BUILD_SHARED}
+ -DARROW_BUILD_STATIC=${ARROW_BUILD_STATIC}
+ -DARROW_BUILD_TESTS=OFF
+ -DARROW_DEPENDENCY_SOURCE=SYSTEM
+ -DARROW_DEPENDENCY_USE_SHARED=${ARROW_DEPENDENCY_USE_SHARED}
+ -DARROW_PACKAGE_KIND=vcpkg
+ -DARROW_WITH_BROTLI=OFF
+ -DARROW_WITH_BZ2=OFF
+ -DARROW_WITH_LZ4=ON
+ -DARROW_WITH_SNAPPY=ON
+ -DARROW_WITH_ZLIB=ON
+ -DARROW_WITH_ZSTD=ON
+ -DBUILD_WARNING_LEVEL=PRODUCTION
+ -DZSTD_MSVC_LIB_PREFIX=
+ MAYBE_UNUSED_VARIABLES
+ ZSTD_MSVC_LIB_PREFIX
+)
+
+vcpkg_cmake_install()
+vcpkg_copy_pdbs()
+
+vcpkg_fixup_pkgconfig()
+
+if(EXISTS "${CURRENT_PACKAGES_DIR}/lib/arrow_static.lib")
+ message(FATAL_ERROR "Installed lib file should be named 'arrow.lib' via
patching the upstream build.")
+endif()
+
+if("dataset" IN_LIST FEATURES)
+ vcpkg_cmake_config_fixup(
+ PACKAGE_NAME arrowdataset
+ CONFIG_PATH lib/cmake/ArrowDataset
+ DO_NOT_DELETE_PARENT_CONFIG_PATH
+ )
+endif()
+
+if("acero" IN_LIST FEATURES)
+ vcpkg_cmake_config_fixup(
+ PACKAGE_NAME arrowacero
+ CONFIG_PATH lib/cmake/ArrowAcero
+ DO_NOT_DELETE_PARENT_CONFIG_PATH
+ )
+endif()
+
+if("flight" IN_LIST FEATURES)
+ vcpkg_cmake_config_fixup(
+ PACKAGE_NAME ArrowFlight
+ CONFIG_PATH lib/cmake/ArrowFlight
+ DO_NOT_DELETE_PARENT_CONFIG_PATH
+ )
+endif()
+
+if("flightsql" IN_LIST FEATURES)
+ vcpkg_cmake_config_fixup(
+ PACKAGE_NAME ArrowFlightSql
+ CONFIG_PATH lib/cmake/ArrowFlightSql
+ DO_NOT_DELETE_PARENT_CONFIG_PATH
+ )
+endif()
+
+if("parquet" IN_LIST FEATURES)
+ vcpkg_cmake_config_fixup(
+ PACKAGE_NAME parquet
+ CONFIG_PATH lib/cmake/Parquet
+ DO_NOT_DELETE_PARENT_CONFIG_PATH
+ )
+endif()
+vcpkg_cmake_config_fixup(CONFIG_PATH lib/cmake/Arrow)
+
+file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/usage" DESTINATION
"${CURRENT_PACKAGES_DIR}/share/${PORT}")
+if("parquet" IN_LIST FEATURES)
+ file(READ "${CMAKE_CURRENT_LIST_DIR}/usage-parquet" usage-parquet)
+ file(APPEND "${CURRENT_PACKAGES_DIR}/share/${PORT}/usage"
"${usage-parquet}")
+endif()
+if("dataset" IN_LIST FEATURES)
+ file(READ "${CMAKE_CURRENT_LIST_DIR}/usage-dataset" usage-dataset)
+ file(APPEND "${CURRENT_PACKAGES_DIR}/share/${PORT}/usage"
"${usage-dataset}")
+endif()
+if("acero" IN_LIST FEATURES)
+ file(READ "${CMAKE_CURRENT_LIST_DIR}/usage-acero" usage-acero)
+ file(APPEND "${CURRENT_PACKAGES_DIR}/share/${PORT}/usage" "${usage-acero}")
+endif()
+
+if("flight" IN_LIST FEATURES)
+ file(READ "${CMAKE_CURRENT_LIST_DIR}/usage-flight" usage-flight)
+ file(APPEND "${CURRENT_PACKAGES_DIR}/share/${PORT}/usage"
"${usage-flight}")
+endif()
+
+if("flightsql" IN_LIST FEATURES)
+ file(READ "${CMAKE_CURRENT_LIST_DIR}/usage-flightsql" usage-flightsql)
+ file(APPEND "${CURRENT_PACKAGES_DIR}/share/${PORT}/usage"
"${usage-flightsql}")
+endif()
+
+if("example" IN_LIST FEATURES)
+ file(INSTALL "${SOURCE_PATH}/cpp/examples/minimal_build/" DESTINATION
"${CURRENT_PACKAGES_DIR}/share/${PORT}/example")
+endif()
+
+file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include")
+file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share")
+file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/share/doc")
+
+vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE.txt")
diff --git a/dev/vcpkg/ports/arrow/thrift.patch
b/dev/vcpkg/ports/arrow/thrift.patch
new file mode 100644
index 0000000000..ae284d65b8
--- /dev/null
+++ b/dev/vcpkg/ports/arrow/thrift.patch
@@ -0,0 +1,22 @@
+diff --git a/cpp/cmake_modules/FindThriftAlt.cmake
b/cpp/cmake_modules/FindThriftAlt.cmake
+index f3e4902..65ceac8 100644
+--- a/cpp/cmake_modules/FindThriftAlt.cmake
++++ b/cpp/cmake_modules/FindThriftAlt.cmake
+@@ -45,7 +45,7 @@ endif()
+ # * https://github.com/apache/thrift/pull/2725
+ # * https://github.com/apache/thrift/pull/2726
+ # * https://github.com/conda-forge/thrift-cpp-feedstock/issues/68
+-if(NOT WIN32)
++
+ set(find_package_args "")
+ if(ThriftAlt_FIND_VERSION)
+ list(APPEND find_package_args ${ThriftAlt_FIND_VERSION})
+@@ -61,7 +61,7 @@ if(NOT WIN32)
+ "${THRIFT_COMPILER}")
+ return()
+ endif()
+-endif()
++
+
+ function(extract_thrift_version)
+ if(ThriftAlt_INCLUDE_DIR)
diff --git a/dev/vcpkg/ports/arrow/usage b/dev/vcpkg/ports/arrow/usage
new file mode 100644
index 0000000000..b07f1c9c32
--- /dev/null
+++ b/dev/vcpkg/ports/arrow/usage
@@ -0,0 +1,4 @@
+The package arrow provides CMake targets:
+
+ find_package(Arrow CONFIG REQUIRED)
+ target_link_libraries(main PRIVATE
"$<IF:$<BOOL:${ARROW_BUILD_STATIC}>,Arrow::arrow_static,Arrow::arrow_shared>")
diff --git a/dev/vcpkg/ports/arrow/usage-acero
b/dev/vcpkg/ports/arrow/usage-acero
new file mode 100644
index 0000000000..05c4776b79
--- /dev/null
+++ b/dev/vcpkg/ports/arrow/usage-acero
@@ -0,0 +1,3 @@
+
+ find_package(ArrowAcero CONFIG REQUIRED)
+ target_link_libraries(main PRIVATE
"$<IF:$<BOOL:${ARROW_BUILD_STATIC}>,ArrowAcero::arrow_acero_static,ArrowAcero::arrow_acero_shared>")
diff --git a/dev/vcpkg/ports/arrow/usage-dataset
b/dev/vcpkg/ports/arrow/usage-dataset
new file mode 100644
index 0000000000..8bedfb8324
--- /dev/null
+++ b/dev/vcpkg/ports/arrow/usage-dataset
@@ -0,0 +1,3 @@
+
+ find_package(ArrowDataset CONFIG REQUIRED)
+ target_link_libraries(main PRIVATE
"$<IF:$<BOOL:${ARROW_BUILD_STATIC}>,ArrowDataset::arrow_dataset_static,ArrowDataset::arrow_dataset_shared>")
diff --git a/dev/vcpkg/ports/arrow/usage-flight
b/dev/vcpkg/ports/arrow/usage-flight
new file mode 100644
index 0000000000..c9656253f9
--- /dev/null
+++ b/dev/vcpkg/ports/arrow/usage-flight
@@ -0,0 +1,3 @@
+
+ find_package(ArrowFlight CONFIG REQUIRED)
+ target_link_libraries(main PRIVATE
"$<IF:$<BOOL:${ARROW_BUILD_STATIC}>,ArrowFlight::arrow_flight_static,ArrowFlight::arrow_flight_shared>")
diff --git a/dev/vcpkg/ports/arrow/usage-flightsql
b/dev/vcpkg/ports/arrow/usage-flightsql
new file mode 100644
index 0000000000..3159c05903
--- /dev/null
+++ b/dev/vcpkg/ports/arrow/usage-flightsql
@@ -0,0 +1,3 @@
+
+ find_package(ArrowFlightSql CONFIG REQUIRED)
+ target_link_libraries(main PRIVATE
"$<IF:$<BOOL:${ARROW_BUILD_STATIC}>,ArrowFlightSql::arrow_flight_sql_static,ArrowFlightSql::arrow_flight_sql_shared>")
diff --git a/dev/vcpkg/ports/arrow/usage-parquet
b/dev/vcpkg/ports/arrow/usage-parquet
new file mode 100644
index 0000000000..ce04039a64
--- /dev/null
+++ b/dev/vcpkg/ports/arrow/usage-parquet
@@ -0,0 +1,3 @@
+
+ find_package(Parquet CONFIG REQUIRED)
+ target_link_libraries(main PRIVATE
"$<IF:$<BOOL:${ARROW_BUILD_STATIC}>,Parquet::parquet_static,Parquet::parquet_shared>")
diff --git a/dev/vcpkg/ports/arrow/utf8proc.patch
b/dev/vcpkg/ports/arrow/utf8proc.patch
new file mode 100644
index 0000000000..eea9ca59e2
--- /dev/null
+++ b/dev/vcpkg/ports/arrow/utf8proc.patch
@@ -0,0 +1,13 @@
+diff --git a/cpp/cmake_modules/Findutf8proc.cmake
b/cpp/cmake_modules/Findutf8proc.cmake
+index e347414..83f2aa1 100644
+--- a/cpp/cmake_modules/Findutf8proc.cmake
++++ b/cpp/cmake_modules/Findutf8proc.cmake
+@@ -30,7 +30,7 @@ if(ARROW_PACKAGE_KIND STREQUAL "vcpkg")
+ if(utf8proc_FIND_REQUIRED)
+ list(APPEND find_package_args REQUIRED)
+ endif()
+- find_package(utf8proc NAMES unofficial-utf8proc ${find_package_args})
++ find_package(utf8proc NAMES unofficial-utf8proc)
+ if(utf8proc_FOUND)
+ add_library(utf8proc::utf8proc ALIAS utf8proc)
+ return()
diff --git a/dev/vcpkg/ports/arrow/vcpkg.json b/dev/vcpkg/ports/arrow/vcpkg.json
new file mode 100644
index 0000000000..059c33fe7d
--- /dev/null
+++ b/dev/vcpkg/ports/arrow/vcpkg.json
@@ -0,0 +1,149 @@
+{
+ "name": "arrow",
+ "version": "18.0.0",
+ "description": "Cross-language development platform for in-memory analytics",
+ "homepage": "https://arrow.apache.org";,
+ "license": "Apache-2.0",
+ "supports": "x64 | (arm64 & !windows)",
+ "dependencies": [
+ "boost-filesystem",
+ "boost-multiprecision",
+ "boost-system",
+ "gflags",
+ "lz4",
+ "openssl",
+ "re2",
+ "snappy",
+ {
+ "name": "vcpkg-cmake",
+ "host": true
+ },
+ {
+ "name": "vcpkg-cmake-config",
+ "host": true
+ },
+ "xsimd",
+ "zlib",
+ "zstd"
+ ],
+ "default-features": [
+ "csv",
+ "filesystem",
+ "json",
+ "parquet"
+ ],
+ "features": {
+ "acero": {
+ "description": "Acero support"
+ },
+ "compute": {
+ "description": "Build all computational kernel functions",
+ "dependencies": [
+ "utf8proc"
+ ]
+ },
+ "csv": {
+ "description": "CSV support"
+ },
+ "cuda": {
+ "description": "cuda support",
+ "dependencies": [
+ "cuda"
+ ]
+ },
+ "dataset": {
+ "description": "Dataset support"
+ },
+ "example": {
+ "description": "Install the minimal example (source code)"
+ },
+ "filesystem": {
+ "description": "Filesystem support"
+ },
+ "flight": {
+ "description": "Arrow Flight RPC support",
+ "dependencies": [
+ "abseil",
+ "c-ares",
+ "grpc",
+ "protobuf"
+ ]
+ },
+ "flightsql": {
+ "description": "FlightSQL support",
+ "dependencies": [
+ {
+ "name": "arrow",
+ "default-features": false,
+ "features": [
+ "flight"
+ ]
+ }
+ ]
+ },
+ "gcs": {
+ "description": "GCS support",
+ "dependencies": [
+ {
+ "name": "google-cloud-cpp",
+ "default-features": false,
+ "features": [
+ "storage"
+ ]
+ }
+ ]
+ },
+ "jemalloc": {
+ "description": "jemalloc allocator",
+ "supports": "!windows"
+ },
+ "json": {
+ "description": "JSON support",
+ "dependencies": [
+ "rapidjson"
+ ]
+ },
+ "mimalloc": {
+ "description": "mimalloc allocator",
+ "supports": "windows"
+ },
+ "orc": {
+ "description": "ORC support",
+ "dependencies": [
+ "orc"
+ ]
+ },
+ "parquet": {
+ "description": "Parquet support",
+ "dependencies": [
+ "rapidjson",
+ "thrift"
+ ]
+ },
+ "s3": {
+ "description": "S3 support",
+ "dependencies": [
+ {
+ "name": "aws-sdk-cpp",
+ "default-features": false,
+ "features": [
+ "cognito-identity",
+ "config",
+ "identity-management",
+ "s3",
+ "sts",
+ "transfer"
+ ]
+ }
+ ]
+ },
+ "testing": {
+ "description": "Build the Arrow testing support library",
+ "dependencies": [
+ "boost-process",
+ "gtest",
+ "rapidjson"
+ ]
+ }
+ }
+}
diff --git a/dev/vcpkg/toolchain.cmake b/dev/vcpkg/toolchain.cmake
index f4db8e7af5..52aebad399 100644
--- a/dev/vcpkg/toolchain.cmake
+++ b/dev/vcpkg/toolchain.cmake
@@ -33,11 +33,30 @@ set(VCPKG_HOST_TRIPLET $ENV{VCPKG_TRIPLET})
set(VCPKG_INSTALLED_DIR $ENV{VCPKG_MANIFEST_DIR}/vcpkg_installed)
set(VCPKG_INSTALL_OPTIONS --no-print-usage)
-# Force read CMAKE_PREFIX_PATH from env
-set(CMAKE_PREFIX_PATH $ENV{CMAKE_PREFIX_PATH})
-
include($ENV{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake)
+if(NOT DEFINED CMAKE_FIND_PACKAGE_PREFER_CONFIG)
+ set(CMAKE_FIND_PACKAGE_PREFER_CONFIG ON)
+endif()
+set(CMAKE_FIND_PACKAGE_PREFER_CONFIG "${CMAKE_FIND_PACKAGE_PREFER_CONFIG}"
CACHE BOOL
+ "Prefer package configuration files." FORCE)
+set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY CACHE STRING "Search for packages
only in root paths." FORCE)
+set(CMAKE_FIND_USE_PACKAGE_REGISTRY OFF CACHE BOOL "Disable the user package
registry." FORCE)
+set(CMAKE_FIND_USE_SYSTEM_PACKAGE_REGISTRY OFF CACHE BOOL "Disable the system
package registry." FORCE)
+
+set(_GLUTEN_VCPKG_IGNORED_PREFIXES /usr/local)
+if(APPLE)
+ list(APPEND _GLUTEN_VCPKG_IGNORED_PREFIXES /opt/homebrew /opt/local)
+endif()
+if(NOT "$ENV{CONDA_PREFIX}" STREQUAL "")
+ list(APPEND _GLUTEN_VCPKG_IGNORED_PREFIXES "$ENV{CONDA_PREFIX}")
+endif()
+list(APPEND CMAKE_IGNORE_PREFIX_PATH ${_GLUTEN_VCPKG_IGNORED_PREFIXES})
+list(REMOVE_DUPLICATES CMAKE_IGNORE_PREFIX_PATH)
+set(CMAKE_IGNORE_PREFIX_PATH "${CMAKE_IGNORE_PREFIX_PATH}" CACHE STRING
+ "Prefixes ignored by Gluten's vcpkg toolchain." FORCE)
+unset(_GLUTEN_VCPKG_IGNORED_PREFIXES)
+
set(CMAKE_EXE_LINKER_FLAGS "-static-libstdc++ -static-libgcc")
set(CMAKE_SHARED_LINKER_FLAGS "-static-libstdc++ -static-libgcc")
diff --git a/dev/vcpkg/vcpkg-configuration.json
b/dev/vcpkg/vcpkg-configuration.json
index ba7c7f58b3..a7aa80d84e 100644
--- a/dev/vcpkg/vcpkg-configuration.json
+++ b/dev/vcpkg/vcpkg-configuration.json
@@ -3,6 +3,12 @@
"overlay-ports": [ "./ports" ],
"overlay-triplets": [ "./triplets" ],
"registries": [
+ {
+ "kind": "git",
+ "repository": "https://github.com/Microsoft/vcpkg";,
+ "baseline": "bcf3d00d2116056fda0ce47615f6074ffecb7524",
+ "packages": [ "boost", "boost-*"]
+ },
{
"kind": "git",
"repository": "https://github.com/Microsoft/vcpkg";,
diff --git a/dev/vcpkg/vcpkg.json b/dev/vcpkg/vcpkg.json
index 2f62d1bf26..a7ad399b50 100644
--- a/dev/vcpkg/vcpkg.json
+++ b/dev/vcpkg/vcpkg.json
@@ -12,6 +12,11 @@
"lz4",
"double-conversion",
"bzip2",
+ {
+ "name": "arrow",
+ "default-features": false,
+ "features": ["testing"]
+ },
"curl",
"lzo",
"re2",
diff --git a/ep/build-velox/src/build-velox.sh
b/ep/build-velox/src/build-velox.sh
index 8b8fd32ca6..95d6c959f1 100755
--- a/ep/build-velox/src/build-velox.sh
+++ b/ep/build-velox/src/build-velox.sh
@@ -120,7 +120,7 @@ function compile {
-Wno-error=uninitialized -Wno-unknown-warning-option
-Wno-deprecated-declarations'
if [[ "$(uname)" == "Darwin" ]]; then
CXX_FLAGS="$CXX_FLAGS -Wno-inconsistent-missing-override
-Wno-macro-redefined"
- if [[ -n "${INSTALL_PREFIX:-}" && "${INSTALL_PREFIX:-}" != "/usr/local" &&
"${INSTALL_PREFIX:-}" != /usr/local/* ]]; then
+ if [[ -z "${GLUTEN_VCPKG_ENABLED:-}" && -n "${INSTALL_PREFIX:-}" &&
"${INSTALL_PREFIX:-}" != "/usr/local" && "${INSTALL_PREFIX:-}" != /usr/local/*
]]; then
# Add the dependency prefix as a system include: this finds deps that
only
# publish loose headers (e.g. xsimd) and demotes warnings in vendored
# dependency headers (abseil's __is_trivially_relocatable, arrow's
vendored
@@ -134,7 +134,10 @@ function compile {
-DVELOX_MONO_LIBRARY=ON -DVELOX_BUILD_RUNNER=OFF
-DVELOX_SIMDJSON_SKIPUTF8VALIDATION=ON \
-DVELOX_ENABLE_GEO=OFF"
if [ -n "${INSTALL_PREFIX:-}" ]; then
- COMPILE_OPTION="$COMPILE_OPTION -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX}
-DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}"
+ COMPILE_OPTION="$COMPILE_OPTION -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}"
+ if [ -z "${GLUTEN_VCPKG_ENABLED:-}" ]; then
+ COMPILE_OPTION="$COMPILE_OPTION -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX}"
+ fi
fi
if [[ "$(uname)" == "Darwin" && -n "${INSTALL_PREFIX:-}" &&
"${INSTALL_PREFIX:-}" != "/usr/local" && "${INSTALL_PREFIX:-}" != /usr/local/*
]]; then
COMPILE_OPTION="$COMPILE_OPTION -DCMAKE_IGNORE_PREFIX_PATH=/usr/local"
@@ -186,6 +189,7 @@ function compile {
fi
if [ -n "${GLUTEN_VCPKG_ENABLED:-}" ]; then
COMPILE_OPTION="$COMPILE_OPTION -DVELOX_GFLAGS_TYPE=static"
+ COMPILE_OPTION="$COMPILE_OPTION -DCMAKE_FIND_PACKAGE_PREFER_CONFIG=OFF"
fi
COMPILE_OPTION="$COMPILE_OPTION -DCMAKE_BUILD_TYPE=${BUILD_TYPE}"
@@ -198,8 +202,14 @@ function compile {
fi
echo "NUM_THREADS_OPTS: $NUM_THREADS_OPTS"
- export simdjson_SOURCE=AUTO
- export Arrow_SOURCE=AUTO
+ if [ -n "${GLUTEN_VCPKG_ENABLED:-}" ]; then
+ export VELOX_DEPENDENCY_SOURCE=SYSTEM
+ export simdjson_SOURCE=SYSTEM
+ export Arrow_SOURCE=SYSTEM
+ else
+ export simdjson_SOURCE=AUTO
+ export Arrow_SOURCE=AUTO
+ fi
if [ $ARCH == 'x86_64' ]; then
make $COMPILE_TYPE $NUM_THREADS_OPTS EXTRA_CMAKE_FLAGS="${COMPILE_OPTION}"
elif [[ "$ARCH" == 'arm64' || "$ARCH" == 'aarch64' || "$ARCH" == "ppc64le"
]]; then
diff --git a/ep/build-velox/src/get-velox.sh b/ep/build-velox/src/get-velox.sh
index 4ed92d89b1..dbe9e602ce 100755
--- a/ep/build-velox/src/get-velox.sh
+++ b/ep/build-velox/src/get-velox.sh
@@ -152,16 +152,6 @@ function apply_provided_velox_patch {
fi
}
-function apply_compilation_fixes {
- local SUDO_CMD=""
- if [ "$OS" == "Linux" ] && [ "${EUID:-$(id -u)}" -ne 0 ]; then
- SUDO_CMD="sudo"
- fi
- $SUDO_CMD cp ${CURRENT_DIR}/modify_arrow.patch
${VELOX_HOME}/CMake/resolve_dependency_modules/arrow/
-
- git add
${VELOX_HOME}/CMake/resolve_dependency_modules/arrow/modify_arrow.patch # to
avoid the file from being deleted by git clean -dffx :/
-}
-
function setup_linux {
local LINUX_DISTRIBUTION=$(. /etc/os-release && echo ${ID})
local LINUX_VERSION_ID=$(. /etc/os-release && echo ${VERSION_ID})
@@ -243,6 +233,5 @@ fi
apply_provided_velox_patch
-apply_compilation_fixes
echo "Finished getting Velox code"
diff --git a/ep/build-velox/src/setup-openeuler24.sh
b/ep/build-velox/src/setup-openeuler24.sh
index 0c2a425f38..65ac853286 100755
--- a/ep/build-velox/src/setup-openeuler24.sh
+++ b/ep/build-velox/src/setup-openeuler24.sh
@@ -36,13 +36,13 @@ NPROC=$(getconf _NPROCESSORS_ONLN)
export CFLAGS=$(get_cxx_flags $CPU_TARGET) # Used by LZO.
export CXXFLAGS=$CFLAGS # Used by boost.
export CPPFLAGS=$CFLAGS # Used by LZO.
+export INSTALL_PREFIX=${INSTALL_PREFIX:-"/usr/local"}
EXTRA_PKG_CXXFLAGS=" -isystem ${INSTALL_PREFIX}/include"
CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}"
VELOX_BUILD_SHARED=${VELOX_BUILD_SHARED:-"OFF"} #Build folly and gflags shared
for use in libvelox.so.
BUILD_DUCKDB="${BUILD_DUCKDB:-true}"
BUILD_GEOS="${BUILD_GEOS:-true}"
VERSION=$(cat /etc/os-release | grep VERSION_ID)
-export INSTALL_PREFIX=${INSTALL_PREFIX:-"/usr/local"}
DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)/deps-download}
FB_OS_VERSION="v2026.09.07.00"
@@ -252,8 +252,8 @@ function install_velox_deps {
function install_s3 {
install_aws_deps
- local MINIO_OS="linux"
- install_minio ${MINIO_OS}
+ local SILO_OS="linux"
+ install_silo ${SILO_OS}
}
function install_gcs {
diff --git a/tools/workload/benchmark_velox/initialize.ipynb
b/tools/workload/benchmark_velox/initialize.ipynb
index 3c504137e4..58f55a4101 100644
--- a/tools/workload/benchmark_velox/initialize.ipynb
+++ b/tools/workload/benchmark_velox/initialize.ipynb
@@ -2232,18 +2232,6 @@
" !git clone https://github.com/apache/gluten.git gluten"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "hidden": true
- },
- "outputs": [],
- "source": [
- "# Build Arrow for the first time build.\n",
- "!sed -i 's/--build_arrow=OFF/--build_arrow=ON/'
~/gluten/dev/package-vcpkg.sh"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
