This is an automated email from the ASF dual-hosted git repository.
baibaichen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new bf964154c0 [GLUTEN-13167][VL] Fix macOS builds and decouple native
tests from JNI libraries (#13168)
bf964154c0 is described below
commit bf964154c0854afa6496069e0524df5bd24ccf40
Author: Chang Chen <[email protected]>
AuthorDate: Sat Oct 17 21:04:02 2026 +0800
[GLUTEN-13167][VL] Fix macOS builds and decouple native tests from JNI
libraries (#13168)
* [VL] Fix macOS OpenSSL 3 setup and GoogleTest installation
* build(vcpkg): own Roaring and resolve packages without shell setup
* fix(native): link tests to implementation objects instead of JNI DSOs
* [MINOR] Format native CMake targets
* [GLUTEN-13167][VL] Expose imported Arrow targets across native directories
* [GLUTEN-13167][VL] Preserve shared-library linkage for UDF integration
tests
* [MINOR][CI] Trigger weekly native platform validation
---
.github/workflows/velox_weekly.yml | 1 +
cpp/CMake/ConfigArrow.cmake | 2 +-
cpp/CMake/FindRoaring.cmake | 9 ++++
cpp/CMakeLists.txt | 2 +-
cpp/core/CMakeLists.txt | 34 +++++++-------
cpp/velox/CMakeLists.txt | 67 +++++++++++++++-------------
cpp/velox/benchmarks/CMakeLists.txt | 8 ++--
cpp/velox/compute/delta/tests/CMakeLists.txt | 8 ++--
cpp/velox/tests/CMakeLists.txt | 27 +++++++++--
cpp/velox/tests/GlutenS3FileSystemTest.cc | 7 +--
dev/builddeps-veloxbe.sh | 1 +
dev/vcpkg/vcpkg.json | 2 +
ep/build-velox/src/build-velox.sh | 1 +
ep/build-velox/src/get-velox.sh | 4 ++
14 files changed, 112 insertions(+), 61 deletions(-)
diff --git a/.github/workflows/velox_weekly.yml
b/.github/workflows/velox_weekly.yml
index c4968a8c3c..b5b4104d41 100644
--- a/.github/workflows/velox_weekly.yml
+++ b/.github/workflows/velox_weekly.yml
@@ -14,6 +14,7 @@
# limitations under the License.
name: Velox Backend Weekly Job
+# Validate native build changes across the weekly platform matrix.
on:
pull_request:
diff --git a/cpp/CMake/ConfigArrow.cmake b/cpp/CMake/ConfigArrow.cmake
index 4cf061a769..f82314ef63 100644
--- a/cpp/CMake/ConfigArrow.cmake
+++ b/cpp/CMake/ConfigArrow.cmake
@@ -36,7 +36,7 @@ function(FIND_ARROW_LIB LIB_NAME)
if(NOT TARGET Arrow::${LIB_NAME})
set(ARROW_LIB_FULL_NAME
${CMAKE_SHARED_LIBRARY_PREFIX}${LIB_NAME}${ARROW_STATIC_LIBRARY_SUFFIX})
- add_library(Arrow::${LIB_NAME} STATIC IMPORTED)
+ add_library(Arrow::${LIB_NAME} STATIC IMPORTED GLOBAL)
# Firstly find the lib from bundled path in Velox. If not found, try to
find
# it from system.
find_library(
diff --git a/cpp/CMake/FindRoaring.cmake b/cpp/CMake/FindRoaring.cmake
index bfb7e9fb22..0ac1a7d9a9 100644
--- a/cpp/CMake/FindRoaring.cmake
+++ b/cpp/CMake/FindRoaring.cmake
@@ -13,6 +13,15 @@
# License for the specific language governing permissions and limitations under
# the License.
+if(ENABLE_GLUTEN_VCPKG)
+ find_package(roaring CONFIG REQUIRED)
+ if(NOT TARGET roaring)
+ add_library(roaring ALIAS roaring::roaring)
+ endif()
+ set(Roaring_FOUND TRUE)
+ return()
+endif()
+
set(_roaring_pkgconfig_hints "")
set(_roaring_include_hints "")
set(_roaring_library_hints "")
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index f9918de593..9eb98e7893 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -249,7 +249,7 @@ function(ADD_TEST_CASE TEST_NAME)
endif()
add_executable(${TEST_NAME} ${SOURCES})
- target_link_libraries(${TEST_NAME} gluten glog::glog GTest::gtest
+ target_link_libraries(${TEST_NAME} gluten_core_impl glog::glog GTest::gtest
GTest::gtest_main Threads::Threads)
target_include_directories(${TEST_NAME} PRIVATE ${CMAKE_SOURCE_DIR}/core)
diff --git a/cpp/core/CMakeLists.txt b/cpp/core/CMakeLists.txt
index 66c800595c..b16ce48c96 100644
--- a/cpp/core/CMakeLists.txt
+++ b/cpp/core/CMakeLists.txt
@@ -120,7 +120,6 @@ set(SPARK_COLUMNAR_PLUGIN_SRCS
compute/ProtobufUtils.cc
compute/ResultIterator.cc
config/GlutenConfig.cc
- jni/JniWrapper.cc
memory/AllocationListener.cc
memory/MemoryAllocator.cc
memory/MemoryManager.cc
@@ -152,8 +151,11 @@ set(SPARK_COLUMNAR_PLUGIN_SRCS
jni/TaskContextJniWrapper.cc)
file(MAKE_DIRECTORY ${root_directory}/releases)
-add_library(gluten SHARED ${SPARK_COLUMNAR_PLUGIN_SRCS})
-add_dependencies(gluten jni_proto)
+add_library(gluten_core_impl OBJECT ${SPARK_COLUMNAR_PLUGIN_SRCS})
+set_target_properties(gluten_core_impl PROPERTIES POSITION_INDEPENDENT_CODE ON)
+add_dependencies(gluten_core_impl jni_proto)
+add_library(gluten SHARED jni/JniWrapper.cc)
+target_link_libraries(gluten PRIVATE gluten_core_impl)
# Hide symbols of some static dependencies. Otherwise, if such dependencies are
# already statically linked to libvelox.so, a runtime error will be reported:
@@ -192,10 +194,10 @@ if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0)
OUTPUT_STRIP_TRAILING_WHITESPACE)
if(LIBSTDCXXFS_STATIC_RESULT EQUAL 0 AND EXISTS "${LIBSTDCXXFS_STATIC_PATH}")
message(STATUS "libstdc++fs.a found at: ${LIBSTDCXXFS_STATIC_PATH}")
- target_link_libraries(gluten PRIVATE ${LIBSTDCXXFS_STATIC_PATH})
+ target_link_libraries(gluten_core_impl PRIVATE ${LIBSTDCXXFS_STATIC_PATH})
else()
find_library(LIBSTDCXXFS stdc++fs REQUIRED)
- target_link_libraries(gluten PUBLIC ${LIBSTDCXXFS})
+ target_link_libraries(gluten_core_impl PUBLIC ${LIBSTDCXXFS})
endif()
endif()
@@ -207,15 +209,15 @@ endif()
if(ENABLE_QAT)
include(BuildQATzip)
include(BuildQATZstd)
- target_sources(gluten PRIVATE utils/qat/QatCodec.cc)
- target_include_directories(gluten PUBLIC ${QATZIP_INCLUDE_DIR}
- ${QATZSTD_INCLUDE_DIR})
- target_link_libraries(gluten PUBLIC qatzip::qatzip qatzstd::qatzstd)
+ target_sources(gluten_core_impl PRIVATE utils/qat/QatCodec.cc)
+ target_include_directories(gluten_core_impl PUBLIC ${QATZIP_INCLUDE_DIR}
+ ${QATZSTD_INCLUDE_DIR})
+ target_link_libraries(gluten_core_impl PUBLIC qatzip::qatzip
qatzstd::qatzstd)
endif()
find_protobuf()
message(STATUS "Found Protobuf: ${PROTOBUF_LIBRARY}")
-target_link_libraries(gluten LINK_PUBLIC ${PROTOBUF_LIBRARY})
+target_link_libraries(gluten_core_impl LINK_PUBLIC ${PROTOBUF_LIBRARY})
add_custom_command(
OUTPUT ${SUBSTRAIT_PROTO_OUTPUT_FILES}
@@ -238,11 +240,12 @@ add_custom_target(jni_proto ALL DEPENDS
${SUBSTRAIT_PROTO_OUTPUT_FILES}
add_dependencies(jni_proto protobuf::libprotobuf)
target_include_directories(
- gluten
+ gluten_core_impl
PUBLIC ${CMAKE_SYSTEM_INCLUDE_PATH} ${JNI_INCLUDE_DIRS}
${CMAKE_CURRENT_SOURCE_DIR} ${PROTO_OUTPUT_DIR} ${PROTOBUF_INCLUDE})
if(GLUTEN_PREFIX_INCLUDE_DIRS)
- target_include_directories(gluten BEFORE PUBLIC
${GLUTEN_PREFIX_INCLUDE_DIRS})
+ target_include_directories(gluten_core_impl BEFORE
+ PUBLIC ${GLUTEN_PREFIX_INCLUDE_DIRS})
endif()
set_target_properties(gluten PROPERTIES LIBRARY_OUTPUT_DIRECTORY
${root_directory}/releases)
@@ -251,11 +254,12 @@ if(BUILD_TESTS)
add_subdirectory(tests)
endif()
-target_link_libraries(gluten PUBLIC Arrow::arrow)
+target_link_libraries(gluten_core_impl PUBLIC Arrow::arrow)
if(NOT ENABLE_GLUTEN_VCPKG)
- target_link_libraries(gluten PUBLIC Arrow::arrow_bundled_dependencies)
+ target_link_libraries(gluten_core_impl
+ PUBLIC Arrow::arrow_bundled_dependencies)
endif()
-target_link_libraries(gluten PRIVATE glog::glog)
+target_link_libraries(gluten_core_impl PRIVATE glog::glog)
install(TARGETS gluten DESTINATION ${CMAKE_INSTALL_LIBDIR})
diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt
index a15f201528..86f23171f5 100644
--- a/cpp/velox/CMakeLists.txt
+++ b/cpp/velox/CMakeLists.txt
@@ -75,7 +75,7 @@ if(NOT DEFINED VELOX_BUILD_PATH)
endif()
endif()
-find_package(Roaring REQUIRED)
+find_package(Roaring MODULE REQUIRED)
set(GLUTEN_ROARING_LINK_LIBRARY roaring)
set(VELOX_PROTO_SRC_DIR
@@ -169,7 +169,6 @@ set(VELOX_SRCS
compute/iceberg/IcebergPlanConverter.cc
jni/JniFileSystem.cc
jni/JniUdf.cc
- jni/VeloxJniWrapper.cc
jni/JniHashTable.cc
memory/BufferOutputStream.cc
memory/VeloxColumnarBatch.cc
@@ -246,11 +245,14 @@ if(BUILD_TEST_UTILS)
tests/utils/TestAllocationListener.cc)
endif()
-add_library(velox SHARED ${VELOX_SRCS})
+add_library(gluten_velox_impl OBJECT ${VELOX_SRCS})
+set_target_properties(gluten_velox_impl PROPERTIES POSITION_INDEPENDENT_CODE
ON)
+add_library(velox SHARED jni/VeloxJniWrapper.cc)
+target_link_libraries(velox PUBLIC gluten_velox_impl gluten)
if(ENABLE_GLUTEN_VCPKG)
find_package(xxHash CONFIG REQUIRED)
- target_link_libraries(velox PUBLIC xxHash::xxhash)
+ target_link_libraries(gluten_velox_impl PUBLIC xxHash::xxhash)
endif()
# Keep ABI sensitive Velox feature macros in sync
@@ -261,7 +263,7 @@ if(NOT GLUTEN_VELOX_ENABLE_PARQUET AND EXISTS
"${GLUTEN_VELOX_CMAKE_CACHE}")
REGEX "^VELOX_ENABLE_PARQUET:BOOL=(ON|TRUE|1)$")
endif()
if(GLUTEN_VELOX_ENABLE_PARQUET)
- target_compile_definitions(velox PRIVATE VELOX_ENABLE_PARQUET)
+ target_compile_definitions(gluten_velox_impl PRIVATE VELOX_ENABLE_PARQUET)
message(STATUS "Propagating VELOX_ENABLE_PARQUET to Gluten Velox backend")
endif()
@@ -292,10 +294,10 @@ add_custom_target(velox_jni_proto ALL DEPENDS
${SUBSTRAIT_PROTO_OUTPUT_FILES}
${VELOX_PROTO_OUTPUT_FILES})
add_dependencies(velox_jni_proto protobuf::libprotobuf)
-add_dependencies(velox velox_jni_proto)
+add_dependencies(gluten_velox_impl velox_jni_proto)
target_include_directories(
- velox
+ gluten_velox_impl
PUBLIC ${CMAKE_SYSTEM_INCLUDE_PATH}
${JNI_INCLUDE_DIRS}
${VELOX_BUILD_PATH}
@@ -304,7 +306,8 @@ target_include_directories(
${PROTO_OUTPUT_DIR}
${PROTOBUF_INCLUDE})
if(GLUTEN_PREFIX_INCLUDE_DIRS)
- target_include_directories(velox BEFORE PUBLIC ${GLUTEN_PREFIX_INCLUDE_DIRS})
+ target_include_directories(gluten_velox_impl BEFORE
+ PUBLIC ${GLUTEN_PREFIX_INCLUDE_DIRS})
endif()
set_target_properties(velox PROPERTIES LIBRARY_OUTPUT_DIRECTORY
@@ -318,13 +321,14 @@ if(ENABLE_JEMALLOC_STATS)
build_jemalloc()
endif()
add_definitions(-DENABLE_JEMALLOC_STATS)
- target_link_libraries(velox PUBLIC jemalloc::jemalloc)
+ target_link_libraries(gluten_velox_impl PUBLIC jemalloc::jemalloc)
endif()
-target_link_libraries(velox PUBLIC gluten)
+target_link_libraries(gluten_velox_impl PUBLIC gluten_core_impl)
# Requires VELOX_MONO_LIBRARY=ON when building Velox.
import_library(facebook::velox ${VELOX_BUILD_PATH}/lib/libvelox.a)
+target_link_libraries(facebook::velox INTERFACE Arrow::arrow)
set(GLUTEN_VELOX_LINK_SCOPE PUBLIC)
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
set(GLUTEN_VELOX_LINK_SCOPE PRIVATE)
@@ -374,15 +378,16 @@ if(BUILD_TESTS)
INTERFACE facebook::velox::vector_test_lib facebook::velox::vector_fuzzer
facebook::velox::dwio_common_test
facebook::velox::file_test_utils)
- target_link_libraries(velox PUBLIC facebook::velox::exec_test_lib)
+ set(GLUTEN_VELOX_TEST_LIBRARIES facebook::velox::exec_test_lib)
endif()
-target_link_libraries(velox ${GLUTEN_VELOX_LINK_SCOPE} facebook::velox)
+target_link_libraries(gluten_velox_impl ${GLUTEN_VELOX_LINK_SCOPE}
+ facebook::velox)
-target_link_libraries(velox PUBLIC ${GLUTEN_ROARING_LINK_LIBRARY})
+target_link_libraries(gluten_velox_impl PUBLIC ${GLUTEN_ROARING_LINK_LIBRARY})
find_re2()
-target_link_libraries(velox PUBLIC ${RE2_LIBRARY})
+target_link_libraries(gluten_velox_impl PUBLIC ${RE2_LIBRARY})
if(ENABLE_GLUTEN_VCPKG)
# FBThrift -> mvfst -> fizz -> OpenSSL; with VCPKG_DYNAMIC_OPENSSL=ON vcpkg
@@ -404,7 +409,7 @@ else()
external::stemmer
${VELOX_BUILD_PATH}/_deps/libstemmer/src/libstemmer/libstemmer.a)
endif()
-target_link_libraries(velox PUBLIC external::stemmer)
+target_link_libraries(gluten_velox_impl PUBLIC external::stemmer)
find_package(geos)
if(geos_FOUND AND TARGET GEOS::geos)
@@ -413,7 +418,7 @@ else()
message(STATUS "import Velox bundled geos")
import_library(external::geos ${VELOX_BUILD_PATH}/lib/libgeos.a)
endif()
-target_link_libraries(velox PUBLIC external::geos)
+target_link_libraries(gluten_velox_impl PUBLIC external::geos)
find_package(veloxthrift CONFIG)
if(veloxthrift_FOUND AND TARGET veloxthrift::veloxthrift)
@@ -425,7 +430,7 @@ else()
${VELOX_BUILD_PATH}/velox/dwio/parquet/thrift/libvelox_dwio_parquet_thrift_raw.a
)
endif()
-target_link_libraries(velox PUBLIC external::veloxthrift)
+target_link_libraries(gluten_velox_impl PUBLIC external::veloxthrift)
# If folly is not installed in system lib paths, please add
# `-DCMAKE_PREFIX_PATH="${folly lib path}" to cmake arguments. It is also
@@ -449,12 +454,12 @@ if(CMAKE_SYSTEM_NAME MATCHES "Darwin" AND TARGET
gflags_shared)
endif()
endforeach()
endif()
-target_link_libraries(velox PUBLIC Folly::folly)
+target_link_libraries(gluten_velox_impl PUBLIC Folly::folly)
# Link thrift libraries - check vcpkg first, then fall back to system libraries
if(ENABLE_GLUTEN_VCPKG)
find_package(FBThrift CONFIG REQUIRED)
- target_link_libraries(velox PUBLIC FBThrift::thriftcpp2)
+ target_link_libraries(gluten_velox_impl PUBLIC FBThrift::thriftcpp2)
else()
set(THRIFT_LIBRARY_DIRS)
if(CMAKE_INSTALL_PREFIX)
@@ -479,7 +484,8 @@ else()
NO_DEFAULT_PATH)
if(THRIFTCPP2_LIB AND THRIFTPROTOCOL_LIB)
- target_link_libraries(velox PRIVATE ${THRIFTCPP2_LIB}
${THRIFTPROTOCOL_LIB})
+ target_link_libraries(gluten_velox_impl PRIVATE ${THRIFTCPP2_LIB}
+ ${THRIFTPROTOCOL_LIB})
else()
message(
FATAL_ERROR
@@ -489,11 +495,11 @@ endif()
find_package(simdjson CONFIG)
if(simdjson_FOUND AND TARGET simdjson::simdjson)
- target_link_libraries(velox PUBLIC simdjson::simdjson)
+ target_link_libraries(gluten_velox_impl PUBLIC simdjson::simdjson)
else()
import_library(external::simdjson
${VELOX_BUILD_PATH}/_deps/simdjson-build/libsimdjson.a)
- target_link_libraries(velox PUBLIC external::simdjson)
+ target_link_libraries(gluten_velox_impl PUBLIC external::simdjson)
endif()
# Restore the platform's default suffixes before looking for optional dynamic
@@ -556,7 +562,7 @@ find_package(
ICU
COMPONENTS i18n uc data
REQUIRED)
-target_link_libraries(velox PUBLIC ICU::i18n ICU::uc ICU::data)
+target_link_libraries(gluten_velox_impl PUBLIC ICU::i18n ICU::uc ICU::data)
if(BUILD_TESTS)
add_subdirectory(compute/delta/tests)
@@ -574,21 +580,22 @@ endif()
if(ENABLE_S3)
add_definitions(-DENABLE_S3)
find_awssdk()
- target_link_libraries(velox PUBLIC ${AWSSDK_LIBRARIES})
+ target_link_libraries(gluten_velox_impl PUBLIC ${AWSSDK_LIBRARIES})
endif()
if(ENABLE_GCS)
add_definitions(-DENABLE_GCS)
find_gcssdk()
- target_link_libraries(velox PUBLIC google-cloud-cpp::storage)
+ target_link_libraries(gluten_velox_impl PUBLIC google-cloud-cpp::storage)
endif()
if(ENABLE_ABFS)
add_definitions(-DENABLE_ABFS)
find_azure()
- target_link_libraries(velox PUBLIC Azure::azure-storage-blobs)
- target_link_libraries(velox PUBLIC Azure::azure-storage-files-datalake)
- target_link_libraries(velox PUBLIC Azure::azure-identity)
+ target_link_libraries(gluten_velox_impl PUBLIC Azure::azure-storage-blobs)
+ target_link_libraries(gluten_velox_impl
+ PUBLIC Azure::azure-storage-files-datalake)
+ target_link_libraries(gluten_velox_impl PUBLIC Azure::azure-identity)
endif()
if(BUILD_EXAMPLES)
@@ -629,7 +636,7 @@ if(ENABLE_GPU)
)
target_compile_definitions(
- velox PRIVATE LIBCUDACXX_ENABLE_EXPERIMENTAL_MEMORY_RESOURCE)
+ gluten_velox_impl PRIVATE LIBCUDACXX_ENABLE_EXPERIMENTAL_MEMORY_RESOURCE)
add_library(velox_curl SHARED IMPORTED GLOBAL)
set_target_properties(
@@ -637,7 +644,7 @@ if(ENABLE_GPU)
${VELOX_BUILD_PATH}/_deps/curl-build/lib/libcurl.so)
target_link_libraries(
- velox
+ gluten_velox_impl
PUBLIC facebook::velox::velox_cudf_exec
facebook::velox::velox_cudf_vector
facebook::velox::velox_cudf_iceberg_connector
diff --git a/cpp/velox/benchmarks/CMakeLists.txt
b/cpp/velox/benchmarks/CMakeLists.txt
index bb1c9d2d90..453533b4c6 100644
--- a/cpp/velox/benchmarks/CMakeLists.txt
+++ b/cpp/velox/benchmarks/CMakeLists.txt
@@ -18,11 +18,13 @@ add_library(velox_benchmark_common STATIC
${VELOX_BENCHMARK_COMMON_SRCS})
target_include_directories(
velox_benchmark_common PUBLIC ${CMAKE_SOURCE_DIR}/velox
${CMAKE_SOURCE_DIR}/core)
-target_link_libraries(velox_benchmark_common PUBLIC velox benchmark::benchmark
- glog::glog)
+target_link_libraries(velox_benchmark_common
+ PUBLIC gluten_velox_impl benchmark::benchmark glog::glog)
function(add_velox_benchmark BM_EXEC BM_FILE)
add_executable(${BM_EXEC} ${BM_FILE})
- target_link_libraries(${BM_EXEC} PRIVATE velox_benchmark_common)
+ target_link_libraries(
+ ${BM_EXEC} PRIVATE velox_benchmark_common gluten_velox_impl
+ gluten_core_impl ${GLUTEN_VELOX_TEST_LIBRARIES})
if(ENABLE_GPU)
target_link_libraries(${BM_EXEC} PRIVATE velox_curl)
endif()
diff --git a/cpp/velox/compute/delta/tests/CMakeLists.txt
b/cpp/velox/compute/delta/tests/CMakeLists.txt
index 3c65d8443f..c37bd67470 100644
--- a/cpp/velox/compute/delta/tests/CMakeLists.txt
+++ b/cpp/velox/compute/delta/tests/CMakeLists.txt
@@ -16,8 +16,8 @@
add_executable(velox_roaring_bitmap_array_test RoaringBitmapArrayTest.cpp)
target_link_libraries(
- velox_roaring_bitmap_array_test velox ${GLUTEN_ROARING_LINK_LIBRARY}
- GTest::gtest GTest::gtest_main)
+ velox_roaring_bitmap_array_test gluten_velox_impl gluten_core_impl
+ ${GLUTEN_ROARING_LINK_LIBRARY} GTest::gtest GTest::gtest_main)
add_test(
NAME velox_roaring_bitmap_array_test
@@ -28,8 +28,8 @@ add_executable(
velox_delta_read_test DeltaConnectorTest.cpp DeltaBitmapAggregatorTest.cpp
DeltaDeletionVectorReaderTest.cpp DeltaSplitTest.cpp)
-target_link_libraries(velox_delta_read_test velox roaring
- facebook::velox::exec_test_lib GTest::gtest)
+target_link_libraries(velox_delta_read_test gluten_velox_impl gluten_core_impl
+ roaring facebook::velox::exec_test_lib GTest::gtest)
add_test(
NAME velox_delta_read_test
diff --git a/cpp/velox/tests/CMakeLists.txt b/cpp/velox/tests/CMakeLists.txt
index 369c92f5d4..f8afb308cd 100644
--- a/cpp/velox/tests/CMakeLists.txt
+++ b/cpp/velox/tests/CMakeLists.txt
@@ -31,8 +31,14 @@ function(add_velox_test TEST_EXEC)
target_compile_definitions(
${TEST_EXEC}
PRIVATE GLUTEN_VELOX_TEST_DATA_DIR="${CMAKE_CURRENT_SOURCE_DIR}/data")
- target_link_libraries(${TEST_EXEC} velox ${GLUTEN_ROARING_LINK_LIBRARY}
- GTest::gtest GTest::gtest_main)
+ target_link_libraries(
+ ${TEST_EXEC}
+ gluten_velox_impl
+ gluten_core_impl
+ ${GLUTEN_VELOX_TEST_LIBRARIES}
+ ${GLUTEN_ROARING_LINK_LIBRARY}
+ GTest::gtest
+ GTest::gtest_main)
if(ENABLE_GPU)
target_link_libraries(${TEST_EXEC} velox_curl)
endif()
@@ -82,8 +88,15 @@ if(ENABLE_GPU)
facebook::velox::velox_cudf_vector
facebook::velox::velox_cudf_hive_connector)
target_link_libraries(
- ${TEST_EXEC} PRIVATE velox ${GLUTEN_ROARING_LINK_LIBRARY} cudf::cudf
- velox_curl GTest::gtest GTest::gtest_main)
+ ${TEST_EXEC}
+ PRIVATE gluten_velox_impl
+ gluten_core_impl
+ ${GLUTEN_VELOX_TEST_LIBRARIES}
+ ${GLUTEN_ROARING_LINK_LIBRARY}
+ cudf::cudf
+ velox_curl
+ GTest::gtest
+ GTest::gtest_main)
gtest_discover_tests(
${TEST_EXEC} DISCOVERY_MODE PRE_TEST
PROPERTIES LABELS "cuda_driver"
@@ -152,6 +165,12 @@ add_velox_test(scoped_timer_test SOURCES
ScopedTimerTest.cc)
add_velox_test(row_based_checksum_test SOURCES RowBasedChecksumTest.cc)
if(BUILD_EXAMPLES)
add_velox_test(my_udf_test SOURCES MyUdfTest.cc)
+ # UDF loading exercises the production shared library, so use the same
+ # runtime.
+ set_property(
+ TARGET my_udf_test
+ PROPERTY LINK_LIBRARIES velox ${GLUTEN_VELOX_TEST_LIBRARIES} GTest::gtest
+ GTest::gtest_main)
endif()
if(ENABLE_ENHANCED_FEATURES)
diff --git a/cpp/velox/tests/GlutenS3FileSystemTest.cc
b/cpp/velox/tests/GlutenS3FileSystemTest.cc
index d99fb37c64..772843b50f 100644
--- a/cpp/velox/tests/GlutenS3FileSystemTest.cc
+++ b/cpp/velox/tests/GlutenS3FileSystemTest.cc
@@ -62,9 +62,10 @@ TEST(GlutenS3FileSystemTest, extractsAsyncUploadConfigs) {
const auto hiveConfig = createHiveConnectorConfig(config,
FileSystemType::kS3);
- EXPECT_EQ(hiveConfig->get<std::string>("hive.s3.part-upload-async"), "true");
- EXPECT_EQ(hiveConfig->get<std::string>("hive.s3.max-concurrent-upload-num"),
"8");
- EXPECT_EQ(hiveConfig->get<std::string>("hive.s3.upload-threads"), "32");
+ const std::string prefix = filesystems::S3Config::kS3Prefix;
+ EXPECT_EQ(hiveConfig->get<std::string>(prefix + "part-upload-async"),
"true");
+ EXPECT_EQ(hiveConfig->get<std::string>(prefix +
"max-concurrent-upload-num"), "8");
+ EXPECT_EQ(hiveConfig->get<std::string>(prefix + "upload-threads"), "32");
}
} // namespace
diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh
index cbc516fe97..b85db703ab 100755
--- a/dev/builddeps-veloxbe.sh
+++ b/dev/builddeps-veloxbe.sh
@@ -350,6 +350,7 @@ function setup_dependencies {
setup_linux
elif [ $OS == 'Darwin' ]; then
setup_macos
+ export OPENSSL_ROOT_DIR="${OPENSSL_ROOT_DIR:-$(brew --prefix openssl@3)}"
else
echo "Unsupported kernel: $OS"
exit 1
diff --git a/dev/vcpkg/vcpkg.json b/dev/vcpkg/vcpkg.json
index 403f6299f6..b102af6146 100644
--- a/dev/vcpkg/vcpkg.json
+++ b/dev/vcpkg/vcpkg.json
@@ -20,6 +20,7 @@
"curl",
"lzo",
"re2",
+ "roaring",
"gflags",
"glog",
"s2geometry",
@@ -131,6 +132,7 @@
}
},
"overrides": [
+ { "name": "roaring", "version": "4.3.11" },
{ "name": "fmt", "version": "11.2.0" },
{ "name": "liblzma", "version": "5.6.2" },
{ "name": "xsimd", "version": "10.0.0" },
diff --git a/ep/build-velox/src/build-velox.sh
b/ep/build-velox/src/build-velox.sh
index 95d6c959f1..79aa1ca488 100755
--- a/ep/build-velox/src/build-velox.sh
+++ b/ep/build-velox/src/build-velox.sh
@@ -234,6 +234,7 @@ function compile {
cd googletest-src; cmake . ; sudo make install -j
#sudo cmake --install googletest-build/
elif [ $OS == 'Darwin' ]; then
+ cmake --build .. --target gtest gtest_main gmock gmock_main --parallel
"${NUM_THREADS:-1}"
install_cmake_dependency googletest-build/
fi
fi
diff --git a/ep/build-velox/src/get-velox.sh b/ep/build-velox/src/get-velox.sh
index 860739dca2..5cdd749958 100755
--- a/ep/build-velox/src/get-velox.sh
+++ b/ep/build-velox/src/get-velox.sh
@@ -96,6 +96,10 @@ function process_setup_tencentos32 {
# independent of this. Header-search isolation from /usr/local is handled by
# SDKROOT exported in builddeps-veloxbe.sh / build-velox.sh.
function process_setup_macos {
+ sed -i '' 's/ lz4 openssl simdjson/ lz4 openssl@3 simdjson/'
scripts/setup-macos.sh
+ sed -i '' '/run_and_time install_velox_deps_from_brew/a\
+ export OPENSSL_ROOT_DIR="$(brew --prefix openssl@3)"
+' scripts/setup-macos.sh
if ! grep -Fq 'FOLLY_USE_JEMALLOC=OFF' scripts/setup-common.sh; then
sed -i '' 's/local FOLLY_FLAGS=(/local
FOLLY_FLAGS=(-DFOLLY_USE_JEMALLOC=OFF /' scripts/setup-common.sh
fi
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
