This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new a6d834783 [VL] Use Velox's monolithic build (#6731)
a6d834783 is described below
commit a6d8347839f2f7bfd6cd3e69fa857fd7a9375bbe
Author: PHILO-HE <[email protected]>
AuthorDate: Thu Aug 8 16:27:05 2024 +0800
[VL] Use Velox's monolithic build (#6731)
---
cpp/velox/CMakeLists.txt | 393 +++++---------------------------------
ep/build-velox/src/build_velox.sh | 2 +-
2 files changed, 54 insertions(+), 341 deletions(-)
diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt
index f1aa96277..195265133 100644
--- a/cpp/velox/CMakeLists.txt
+++ b/cpp/velox/CMakeLists.txt
@@ -91,343 +91,24 @@ endif()
set(VELOX_COMPONENTS_PATH "${VELOX_BUILD_PATH}/velox")
-function(ADD_VELOX_DEPENDENCY VELOX_DEP_LIB_NAME VELOX_DEP_LIB_PATH)
- if(NOT EXISTS ${VELOX_DEP_LIB_PATH})
- message(FATAL_ERROR "Velox library not exists: ${VELOX_DEP_LIB_PATH}")
+function(import_library TARGET_NAME LIB_PATH)
+ if(NOT EXISTS ${LIB_PATH})
+ message(FATAL_ERROR "Library does not exist: ${LIB_PATH}")
endif()
- set(VELOX_DEP_LIB facebook::velox::${VELOX_DEP_LIB_NAME})
- add_library(${VELOX_DEP_LIB} STATIC IMPORTED)
- set_target_properties(${VELOX_DEP_LIB} PROPERTIES IMPORTED_LOCATION
- ${VELOX_DEP_LIB_PATH})
- target_link_libraries(velox PUBLIC ${VELOX_DEP_LIB})
+ add_library(${TARGET_NAME} STATIC IMPORTED)
+ set_target_properties(${TARGET_NAME} PROPERTIES IMPORTED_LOCATION
${LIB_PATH})
endfunction()
-macro(ADD_VELOX_OBJECTS)
- add_library(velox_objects OBJECT IMPORTED GLOBAL)
- set_property(
- TARGET velox_objects
- PROPERTY
- IMPORTED_OBJECTS
-
"${VELOX_COMPONENTS_PATH}/connectors/hive/CMakeFiles/velox_hive_connector.dir/FileHandle.cpp.o"
-
"${VELOX_COMPONENTS_PATH}/connectors/hive/CMakeFiles/velox_hive_connector.dir/HiveConfig.cpp.o"
-
"${VELOX_COMPONENTS_PATH}/connectors/hive/CMakeFiles/velox_hive_connector.dir/HiveConnector.cpp.o"
-
"${VELOX_COMPONENTS_PATH}/connectors/hive/CMakeFiles/velox_hive_connector.dir/HiveDataSink.cpp.o"
-
"${VELOX_COMPONENTS_PATH}/connectors/hive/CMakeFiles/velox_hive_connector.dir/HiveDataSource.cpp.o"
-
"${VELOX_COMPONENTS_PATH}/connectors/hive/CMakeFiles/velox_hive_connector.dir/HivePartitionUtil.cpp.o"
-
"${VELOX_COMPONENTS_PATH}/connectors/hive/CMakeFiles/velox_hive_connector.dir/PartitionIdGenerator.cpp.o"
-
"${VELOX_COMPONENTS_PATH}/connectors/hive/CMakeFiles/velox_hive_connector.dir/SplitReader.cpp.o"
-
"${VELOX_COMPONENTS_PATH}/connectors/hive/CMakeFiles/velox_hive_connector.dir/TableHandle.cpp.o"
-
"${VELOX_COMPONENTS_PATH}/connectors/hive/CMakeFiles/velox_hive_connector.dir/HiveConnectorUtil.cpp.o"
- )
- target_link_libraries(velox PUBLIC velox_objects)
-endmacro()
-
macro(add_duckdb)
find_package(DuckDB)
if(NOT DuckDB_FOUND)
message(FATAL_ERROR "Cannot find DuckDB.")
else()
message(STATUS "Found DuckDB library from ${DuckDB_DIR}")
- target_link_libraries(velox PUBLIC duckdb_static)
+ target_link_libraries(facebook::velox INTERFACE duckdb_static)
endif()
endmacro()
-macro(ADD_VELOX_DEPENDENCIES)
- add_velox_objects()
- add_velox_dependency(
- expression::sigparser
-
"${VELOX_COMPONENTS_PATH}/expression/signature_parser/libvelox_signature_parser.a"
- )
- add_velox_dependency(
- functions::sparksql::lib
- "${VELOX_COMPONENTS_PATH}/functions/sparksql/libvelox_functions_spark.a")
- add_velox_dependency(
- functions::sparksql::agg
-
"${VELOX_COMPONENTS_PATH}/functions/sparksql/aggregates/libvelox_functions_spark_aggregates.a"
- )
- add_velox_dependency(
- functions::window::sparksql
-
"${VELOX_COMPONENTS_PATH}/functions/sparksql/window/libvelox_functions_spark_window.a"
- )
- add_velox_dependency(
- functions::prestosql::agg
-
"${VELOX_COMPONENTS_PATH}/functions/prestosql/aggregates/libvelox_aggregates.a"
- )
- add_velox_dependency(
- functions::lib::agg
-
"${VELOX_COMPONENTS_PATH}/functions/lib/aggregates/libvelox_functions_aggregates.a"
- )
- add_velox_dependency(
- functions::prestosql::window
- "${VELOX_COMPONENTS_PATH}/functions/prestosql/window/libvelox_window.a")
- add_velox_dependency(
- functions::lib::window
-
"${VELOX_COMPONENTS_PATH}/functions/lib/window/libvelox_functions_window.a")
- add_velox_dependency(velox::buffer
- "${VELOX_COMPONENTS_PATH}/buffer/libvelox_buffer.a")
-
- add_velox_dependency(
- functions::isnull
- "${VELOX_COMPONENTS_PATH}/functions/lib/libvelox_is_null_functions.a")
- add_velox_dependency(
- functions::prestosql
-
"${VELOX_COMPONENTS_PATH}/functions/prestosql/registration/libvelox_functions_prestosql.a"
- )
- add_velox_dependency(
- functions::prestosql::impl
-
"${VELOX_COMPONENTS_PATH}/functions/prestosql/libvelox_functions_prestosql_impl.a"
- )
- add_velox_dependency(
- functions::json
-
"${VELOX_COMPONENTS_PATH}/functions/prestosql/json/libvelox_functions_json.a"
- )
- add_velox_dependency(
- functions::hyperloglog
-
"${VELOX_COMPONENTS_PATH}/common/hyperloglog/libvelox_common_hyperloglog.a")
- add_velox_dependency(
- functions::lib
- "${VELOX_COMPONENTS_PATH}/functions/lib/libvelox_functions_lib.a")
- add_velox_dependency(
- functions::lib::date_time_formatter
-
"${VELOX_COMPONENTS_PATH}/functions/lib/libvelox_functions_lib_date_time_formatter.a"
- )
- if(BUILD_TESTS)
- add_velox_dependency(
- exec::test
- "${VELOX_COMPONENTS_PATH}/exec/tests/utils/libvelox_exec_test_lib.a")
- add_velox_dependency(
- temp::path
- "${VELOX_COMPONENTS_PATH}/exec/tests/utils/libvelox_temp_path.a")
- add_velox_dependency(
- dwio::common::test::utils
-
"${VELOX_COMPONENTS_PATH}/dwio/common/tests/utils/libvelox_dwio_common_test_utils.a"
- )
- endif()
- add_velox_dependency(exec "${VELOX_COMPONENTS_PATH}/exec/libvelox_exec.a")
-
- if(BUILD_TESTS)
- add_velox_dependency(
- parse::parser "${VELOX_COMPONENTS_PATH}/parse/libvelox_parse_parser.a")
- add_velox_dependency(
- duckdb::parser
- "${VELOX_COMPONENTS_PATH}/duckdb/conversion/libvelox_duckdb_parser.a")
- add_velox_dependency(
- parse::expression
- "${VELOX_COMPONENTS_PATH}/parse/libvelox_parse_expression.a")
- add_velox_dependency(
- parse::utils "${VELOX_COMPONENTS_PATH}/parse/libvelox_parse_utils.a")
- add_velox_dependency(
- function::registry
- "${VELOX_COMPONENTS_PATH}/functions/libvelox_function_registry.a")
- endif()
- add_velox_dependency(
- vector::arrow::bridge
- "${VELOX_COMPONENTS_PATH}/vector/arrow/libvelox_arrow_bridge.a")
- add_velox_dependency(row "${VELOX_COMPONENTS_PATH}/row/libvelox_row_fast.a")
- add_velox_dependency(
- connector "${VELOX_COMPONENTS_PATH}/connectors/libvelox_connector.a")
- add_velox_dependency(
- connector::hive_parition
-
"${VELOX_COMPONENTS_PATH}/connectors/hive/libvelox_hive_partition_function.a"
- )
- add_velox_dependency(
- connector::hive::iceberg::IcebergSplitReader
-
"${VELOX_COMPONENTS_PATH}/connectors/hive/iceberg/libvelox_hive_iceberg_splitreader.a"
- )
- add_velox_dependency(
- connector::hive::hdfs
-
"${VELOX_COMPONENTS_PATH}/connectors/hive/storage_adapters/hdfs/libvelox_hdfs.a"
- )
- add_velox_dependency(
- connector::hive::gcs
-
"${VELOX_COMPONENTS_PATH}/connectors/hive/storage_adapters/gcs/libvelox_gcs.a"
- )
- add_velox_dependency(
- connector::hive::s3fs
-
"${VELOX_COMPONENTS_PATH}/connectors/hive/storage_adapters/s3fs/libvelox_s3fs.a"
- )
- add_velox_dependency(
- connector::hive::abfs
-
"${VELOX_COMPONENTS_PATH}/connectors/hive/storage_adapters/abfs/libvelox_abfs.a"
- )
- add_velox_dependency(
- dwio::dwrf::writer
- "${VELOX_COMPONENTS_PATH}/dwio/dwrf/writer/libvelox_dwio_dwrf_writer.a")
- add_velox_dependency(
- dwio::dwrf::reader
- "${VELOX_COMPONENTS_PATH}/dwio/dwrf/reader/libvelox_dwio_dwrf_reader.a")
- add_velox_dependency(
- dwio::orc::reader
- "${VELOX_COMPONENTS_PATH}/dwio/orc/reader/libvelox_dwio_orc_reader.a")
- add_velox_dependency(
- dwio::dwrf::utils
- "${VELOX_COMPONENTS_PATH}/dwio/dwrf/utils/libvelox_dwio_dwrf_utils.a")
- add_velox_dependency(
- dwio::dwrf::common
- "${VELOX_COMPONENTS_PATH}/dwio/dwrf/common/libvelox_dwio_dwrf_common.a")
- add_velox_dependency(
- parquet
- "${VELOX_COMPONENTS_PATH}/dwio/parquet/libvelox_dwio_parquet_reader.a")
- add_velox_dependency(
- parquet::reader::native
-
"${VELOX_COMPONENTS_PATH}/dwio/parquet/reader/libvelox_dwio_native_parquet_reader.a"
- )
- if(BUILD_TESTS)
- add_velox_dependency(
- dwio::common::utils
-
"${VELOX_COMPONENTS_PATH}/dwio/common/tests/utils/libvelox_dwio_common_test_utils.a"
- )
- add_velox_dependency(
- dwio::dwrf::test_utils
-
"${VELOX_COMPONENTS_PATH}/dwio/dwrf/test/utils/libvelox_dwrf_test_utils.a"
- )
- add_velox_dependency(
- parquet::reader::duckdb_conversion
-
"${VELOX_COMPONENTS_PATH}/duckdb/conversion/libvelox_duckdb_conversion.a")
-
- add_duckdb()
-
- add_velox_dependency(
- tpch::gen "${VELOX_COMPONENTS_PATH}/tpch/gen/libvelox_tpch_gen.a")
- add_velox_dependency(dbgen
- "${VELOX_COMPONENTS_PATH}/tpch/gen/dbgen/libdbgen.a")
- endif()
-
- add_velox_dependency(
- parquet::reader::thrift
-
"${VELOX_COMPONENTS_PATH}/dwio/parquet/thrift/libvelox_dwio_parquet_thrift.a"
- )
-
- add_velox_dependency(
- velox::arrow::parquet::writer
-
"${VELOX_COMPONENTS_PATH}/dwio/parquet/writer/libvelox_dwio_arrow_parquet_writer.a"
- )
- add_velox_dependency(
- dwio::arrow::parquet::writer
-
"${VELOX_COMPONENTS_PATH}/dwio/parquet/writer/arrow/libvelox_dwio_arrow_parquet_writer_lib.a"
- )
- add_velox_dependency(
- dwio::arrow::parquet::writer::util
-
"${VELOX_COMPONENTS_PATH}/dwio/parquet/writer/arrow/util/libvelox_dwio_arrow_parquet_writer_util_lib.a"
- )
- add_velox_dependency(
- dwio::arrow::parquet::writer::thrift::lib
-
"${VELOX_COMPONENTS_PATH}/dwio/parquet/writer/arrow/generated/libvelox_dwio_arrow_parquet_writer_thrift_lib.a"
- )
- add_velox_dependency(
- dwio::common::compression
-
"${VELOX_COMPONENTS_PATH}/dwio/common/compression/libvelox_dwio_common_compression.a"
- )
- add_velox_dependency(
- dwio::common "${VELOX_COMPONENTS_PATH}/dwio/common/libvelox_dwio_common.a")
- add_velox_dependency(
- functions::prestosql::types
-
"${VELOX_COMPONENTS_PATH}/functions/prestosql/types/libvelox_presto_types.a"
- )
- add_velox_dependency(
- functions::spark::specialforms
-
"${VELOX_COMPONENTS_PATH}/functions/sparksql/specialforms/libvelox_functions_spark_specialforms.a"
- )
- add_velox_dependency(
- expression "${VELOX_COMPONENTS_PATH}/expression/libvelox_expression.a")
- add_velox_dependency(core "${VELOX_COMPONENTS_PATH}/core/libvelox_core.a")
-
- add_velox_dependency(
- type::fbhive "${VELOX_COMPONENTS_PATH}/type/fbhive/libvelox_type_fbhive.a")
- add_velox_dependency(type "${VELOX_COMPONENTS_PATH}/type/libvelox_type.a")
- add_velox_dependency(
- vector::serializes
- "${VELOX_COMPONENTS_PATH}/serializers/libvelox_presto_serializer.a")
- add_velox_dependency(
- functions::lib::util
- "${VELOX_COMPONENTS_PATH}/functions/lib/libvelox_functions_util.a")
- add_velox_dependency(vector
- "${VELOX_COMPONENTS_PATH}/vector/libvelox_vector.a")
- add_velox_dependency(
- expression::function
- "${VELOX_COMPONENTS_PATH}/expression/libvelox_expression_functions.a")
- add_velox_dependency(
- expression::type_calculation
-
"${VELOX_COMPONENTS_PATH}/expression/type_calculation/libvelox_type_calculation.a"
- )
-
- add_velox_dependency(
- common::caching
- "${VELOX_COMPONENTS_PATH}/common/caching/libvelox_caching.a")
- add_velox_dependency(
- common::base "${VELOX_COMPONENTS_PATH}/common/base/libvelox_common_base.a")
- add_velox_dependency(
- common::memory "${VELOX_COMPONENTS_PATH}/common/memory/libvelox_memory.a")
- add_velox_dependency(
- common::serialization
- "${VELOX_COMPONENTS_PATH}/common/serialization/libvelox_serialization.a")
- add_velox_dependency(
- common::base::exception
- "${VELOX_COMPONENTS_PATH}/common/base/libvelox_exception.a")
-
- add_velox_dependency(type::tz
- "${VELOX_COMPONENTS_PATH}/type/tz/libvelox_type_tz.a")
- add_velox_dependency(
- dwio::dwrf::proto
- "${VELOX_COMPONENTS_PATH}/dwio/dwrf/proto/libvelox_dwio_dwrf_proto.a")
- add_velox_dependency(
- dwio::catalog::fbhive
-
"${VELOX_COMPONENTS_PATH}/dwio/catalog/fbhive/libvelox_dwio_catalog_fbhive.a"
- )
- add_velox_dependency(
- dwio::common::exception
-
"${VELOX_COMPONENTS_PATH}/dwio/common/exception/libvelox_dwio_common_exception.a"
- )
- add_velox_dependency(
- dwio::common::encryption
-
"${VELOX_COMPONENTS_PATH}/dwio/common/encryption/libvelox_dwio_common_encryption.a"
- )
-
- add_velox_dependency(core::config
- "${VELOX_COMPONENTS_PATH}/core/libvelox_config.a")
- add_velox_dependency(
- common::encode "${VELOX_COMPONENTS_PATH}/common/encode/libvelox_encode.a")
- add_velox_dependency(common::time
- "${VELOX_COMPONENTS_PATH}/common/time/libvelox_time.a")
- if(BUILD_TESTS)
- add_velox_dependency(
- common::file::test
- "${VELOX_COMPONENTS_PATH}/common/file/tests/libvelox_file_test_utils.a")
- endif()
- add_velox_dependency(common::file
- "${VELOX_COMPONENTS_PATH}/common/file/libvelox_file.a")
- add_velox_dependency(
- common::process
- "${VELOX_COMPONENTS_PATH}/common/process/libvelox_process.a")
-
- add_velox_dependency(
- common::test_util
- "${VELOX_COMPONENTS_PATH}/common/testutil/libvelox_test_util.a")
-
- add_velox_dependency(
- external::md5
- "${VELOX_COMPONENTS_PATH}/external/md5/libvelox_external_md5.a")
- add_velox_dependency(
- external::date
- "${VELOX_COMPONENTS_PATH}/external/date/libvelox_external_date.a")
- add_velox_dependency(
- velox::parquet::writer
- "${VELOX_COMPONENTS_PATH}/dwio/parquet/libvelox_dwio_parquet_writer.a")
-
- if(BUILD_TESTS)
- add_velox_dependency(
- vector::test::util
- "${VELOX_COMPONENTS_PATH}/vector/tests/utils/libvelox_vector_test_lib.a")
- endif()
- add_velox_dependency(
- common::compression
-
"${VELOX_COMPONENTS_PATH}/common/compression/libvelox_common_compression.a")
- add_velox_dependency(
- common::io "${VELOX_COMPONENTS_PATH}/common/io/libvelox_common_io.a")
- add_velox_dependency(velox::status
-
"${VELOX_COMPONENTS_PATH}/common/base/libvelox_status.a")
-endmacro()
-
macro(find_libhdfs3)
find_package(libhdfs3 CONFIG)
if(libhdfs3_FOUND AND TARGET HDFS::hdfs3)
@@ -582,20 +263,51 @@ if(BUILD_JEMALLOC)
endif()
target_link_libraries(velox PUBLIC gluten)
-add_velox_dependencies()
+
+# Requires VELOX_MONO_LIBRARY=ON when building Velox.
+import_library(facebook::velox ${VELOX_BUILD_PATH}/lib/libvelox.a)
+
+if(BUILD_TESTS)
+ add_duckdb()
+
+ import_library(facebook::velox::dbgen
+ ${VELOX_BUILD_PATH}/velox/tpch/gen/dbgen/libdbgen.a)
+ target_link_libraries(facebook::velox INTERFACE facebook::velox::dbgen)
+
+ import_library(
+ facebook::velox::vector_test_lib
+ ${VELOX_BUILD_PATH}/velox/vector/tests/utils/libvelox_vector_test_lib.a)
+ import_library(
+ facebook::velox::dwio_common_test
+
${VELOX_BUILD_PATH}/velox/dwio/common/tests/utils/libvelox_dwio_common_test_utils.a
+ )
+ import_library(
+ facebook::velox::file_test_utils
+ ${VELOX_BUILD_PATH}/velox/common/file/tests/libvelox_file_test_utils.a)
+ import_library(
+ facebook::velox::temp_path
+ ${VELOX_BUILD_PATH}/velox/exec/tests/utils/libvelox_temp_path.a)
+ import_library(
+ facebook::velox::exec_test_lib
+ ${VELOX_COMPONENTS_PATH}/exec/tests/utils/libvelox_exec_test_lib.a)
+ target_link_libraries(
+ facebook::velox::exec_test_lib
+ INTERFACE facebook::velox::vector_test_lib
+ facebook::velox::dwio_common_test
+ facebook::velox::file_test_utils facebook::velox::temp_path)
+ target_link_libraries(velox PUBLIC facebook::velox::exec_test_lib)
+endif()
+
+target_link_libraries(velox PUBLIC facebook::velox)
target_link_libraries(velox PUBLIC Folly::folly)
+
find_re2()
target_link_libraries(velox PUBLIC ${RE2_LIBRARY})
-# since
-#
https://github.com/facebookincubator/velox/commit/47970417ac92135e862c0fde350d4d60fa2f1423
-if(Stemmer_FOUND)
- target_link_libraries(velox PUBLIC stemmer::stemmer)
-else()
- add_velox_dependency(
- velox "${VELOX_BUILD_PATH}/_deps/libstemmer/src/libstemmer/libstemmer.a")
-endif()
+import_library(external::stemmer
+
${VELOX_BUILD_PATH}/_deps/libstemmer/src/libstemmer/libstemmer.a)
+target_link_libraries(velox PUBLIC external::stemmer)
set(CMAKE_FIND_LIBRARY_SUFFIXES_BCK ${CMAKE_FIND_LIBRARY_SUFFIXES})
set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
@@ -603,8 +315,9 @@ find_package(simdjson CONFIG)
if(simdjson_FOUND AND TARGET simdjson::simdjson)
target_link_libraries(velox PUBLIC simdjson::simdjson)
else()
- add_velox_dependency(external::simdjson
-
"${VELOX_BUILD_PATH}/_deps/simdjson-build/libsimdjson.a")
+ import_library(external::simdjson
+ ${VELOX_BUILD_PATH}/_deps/simdjson-build/libsimdjson.a)
+ target_link_libraries(velox PUBLIC external::simdjson)
endif()
set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_BCK})
@@ -634,10 +347,6 @@ if(ENABLE_GCS)
target_link_libraries(velox PUBLIC google-cloud-cpp::storage)
endif()
-if(BUILD_EXAMPLES)
- add_subdirectory(udf/examples)
-endif()
-
if(ENABLE_ABFS)
add_definitions(-DENABLE_ABFS)
find_azure()
@@ -645,6 +354,10 @@ if(ENABLE_ABFS)
target_link_libraries(velox PUBLIC Azure::azure-storage-files-datalake)
endif()
+if(BUILD_EXAMPLES)
+ add_subdirectory(udf/examples)
+endif()
+
add_custom_command(
TARGET velox
POST_BUILD
diff --git a/ep/build-velox/src/build_velox.sh
b/ep/build-velox/src/build_velox.sh
index 873d3638c..b48f28a37 100755
--- a/ep/build-velox/src/build_velox.sh
+++ b/ep/build-velox/src/build_velox.sh
@@ -97,7 +97,7 @@ function compile {
set -exu
CXX_FLAGS='-Wno-missing-field-initializers'
- COMPILE_OPTION="-DCMAKE_CXX_FLAGS=\"$CXX_FLAGS\" -DVELOX_ENABLE_PARQUET=ON
-DVELOX_BUILD_TESTING=OFF"
+ COMPILE_OPTION="-DCMAKE_CXX_FLAGS=\"$CXX_FLAGS\" -DVELOX_ENABLE_PARQUET=ON
-DVELOX_BUILD_TESTING=OFF -DVELOX_MONO_LIBRARY=ON"
if [ $BUILD_TEST_UTILS == "ON" ]; then
COMPILE_OPTION="$COMPILE_OPTION -DVELOX_BUILD_TEST_UTILS=ON"
fi
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
