This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 60b8aadf4 [VL] Daily Update Velox Version (2024_06_05) (#5998)
60b8aadf4 is described below
commit 60b8aadf41b668847e750c23c93ab263a4530503
Author: Gluten Performance Bot
<[email protected]>
AuthorDate: Thu Jun 6 19:51:16 2024 +0800
[VL] Daily Update Velox Version (2024_06_05) (#5998)
Co-authored-by: PHILO-HE <[email protected]>
---
cpp/velox/CMakeLists.txt | 12 +++++++++++-
cpp/velox/memory/VeloxMemoryManager.cc | 10 +++++-----
cpp/velox/operators/writer/VeloxParquetDatasource.cc | 2 +-
cpp/velox/tests/BufferOutputStreamTest.cc | 6 +++---
ep/build-velox/src/get_velox.sh | 2 +-
5 files changed, 21 insertions(+), 11 deletions(-)
diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt
index 9bedfe45b..05ecf9635 100644
--- a/cpp/velox/CMakeLists.txt
+++ b/cpp/velox/CMakeLists.txt
@@ -128,6 +128,7 @@ macro(ADD_VELOX_DEPENDENCIES)
add_velox_dependency(functions::json
"${VELOX_COMPONENTS_PATH}/functions/prestosql/json/libvelox_functions_json.a")
add_velox_dependency(functions::hyperloglog
"${VELOX_COMPONENTS_PATH}/common/hyperloglog/libvelox_common_hyperloglog.a")
add_velox_dependency(functions::lib
"${VELOX_COMPONENTS_PATH}/functions/lib/libvelox_functions_lib.a")
+ add_velox_dependency(functions::lib::date_time_formatter
"${VELOX_COMPONENTS_PATH}/functions/lib/libvelox_functions_lib_date_time_formatter.a")
if(BUILD_TESTS)
add_velox_dependency(exec::test
"${VELOX_COMPONENTS_PATH}/exec/tests/utils/libvelox_exec_test_lib.a")
add_velox_dependency(temp::path
"${VELOX_COMPONENTS_PATH}/exec/tests/utils/libvelox_temp_path.a")
@@ -230,7 +231,6 @@ macro(ADD_VELOX_DEPENDENCIES)
add_velox_dependency(common::compression
"${VELOX_COMPONENTS_PATH}/common/compression/libvelox_common_compression.a")
add_velox_dependency(common::io
"${VELOX_COMPONENTS_PATH}/common/io/libvelox_common_io.a")
add_velox_dependency(velox::status
"${VELOX_COMPONENTS_PATH}/common/base/libvelox_status.a")
- add_velox_dependency(external::simdjson
"${VELOX_BUILD_PATH}/_deps/simdjson-build/libsimdjson.a")
endmacro()
macro(find_libhdfs3)
@@ -389,6 +389,16 @@ else()
add_velox_dependency(velox
"${VELOX_BUILD_PATH}/_deps/libstemmer/src/libstemmer/libstemmer.a")
endif()
+set(CMAKE_FIND_LIBRARY_SUFFIXES_BCK ${CMAKE_FIND_LIBRARY_SUFFIXES})
+set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
+find_package(simdjson CONFIG)
+if(simdjson_FOUND AND TARGET simdjson::simdjson)
+ target_link_libraries(velox PUBLIC simdjson::simdjson)
+else()
+ add_velox_dependency(external::simdjson
"${VELOX_BUILD_PATH}/_deps/simdjson-build/libsimdjson.a")
+endif()
+set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_BCK})
+
if(ENABLE_GLUTEN_VCPKG)
find_package(Thrift CONFIG)
else()
diff --git a/cpp/velox/memory/VeloxMemoryManager.cc
b/cpp/velox/memory/VeloxMemoryManager.cc
index 496ebf452..60c79ffe8 100644
--- a/cpp/velox/memory/VeloxMemoryManager.cc
+++ b/cpp/velox/memory/VeloxMemoryManager.cc
@@ -195,7 +195,7 @@ VeloxMemoryManager::VeloxMemoryManager(
namespace {
MemoryUsageStats collectVeloxMemoryUsageStats(const velox::memory::MemoryPool*
pool) {
MemoryUsageStats stats;
- stats.set_current(pool->currentBytes());
+ stats.set_current(pool->usedBytes());
stats.set_peak(pool->peakBytes());
// walk down root and all children
pool->visitChildren([&](velox::memory::MemoryPool* pool) -> bool {
@@ -216,7 +216,7 @@ int64_t shrinkVeloxMemoryPool(velox::memory::MemoryManager*
mm, velox::memory::M
std::string poolName{pool->root()->name() + "/" + pool->name()};
std::string logPrefix{"Shrink[" + poolName + "]: "};
VLOG(2) << logPrefix << "Trying to shrink " << size << " bytes of data...";
- VLOG(2) << logPrefix << "Pool has reserved " << pool->currentBytes() << "/"
<< pool->root()->reservedBytes() << "/"
+ VLOG(2) << logPrefix << "Pool has reserved " << pool->usedBytes() << "/" <<
pool->root()->reservedBytes() << "/"
<< pool->root()->capacity() << "/" << pool->root()->maxCapacity() <<
" bytes.";
VLOG(2) << logPrefix << "Shrinking...";
const uint64_t oldCapacity = pool->capacity();
@@ -263,14 +263,14 @@ void VeloxMemoryManager::hold() {
bool VeloxMemoryManager::tryDestructSafe() {
// Velox memory pools considered safe to destruct when no alive allocations.
for (const auto& pool : heldVeloxPools_) {
- if (pool && pool->currentBytes() != 0) {
+ if (pool && pool->usedBytes() != 0) {
return false;
}
}
- if (veloxLeafPool_ && veloxLeafPool_->currentBytes() != 0) {
+ if (veloxLeafPool_ && veloxLeafPool_->usedBytes() != 0) {
return false;
}
- if (veloxAggregatePool_ && veloxAggregatePool_->currentBytes() != 0) {
+ if (veloxAggregatePool_ && veloxAggregatePool_->usedBytes() != 0) {
return false;
}
heldVeloxPools_.clear();
diff --git a/cpp/velox/operators/writer/VeloxParquetDatasource.cc
b/cpp/velox/operators/writer/VeloxParquetDatasource.cc
index 16558229e..58aa9f33a 100644
--- a/cpp/velox/operators/writer/VeloxParquetDatasource.cc
+++ b/cpp/velox/operators/writer/VeloxParquetDatasource.cc
@@ -120,7 +120,7 @@ void VeloxParquetDatasource::inspectSchema(struct
ArrowSchema* out) {
std::shared_ptr<velox::ReadFile> readFile{fs->openFileForRead(filePath_)};
std::unique_ptr<velox::dwio::common::Reader> reader =
- velox::dwio::common::getReaderFactory(readerOptions.getFileFormat())
+ velox::dwio::common::getReaderFactory(readerOptions.fileFormat())
->createReader(
std::make_unique<velox::dwio::common::BufferedInput>(
std::make_shared<velox::dwio::common::ReadFileInputStream>(readFile),
*pool_.get()),
diff --git a/cpp/velox/tests/BufferOutputStreamTest.cc
b/cpp/velox/tests/BufferOutputStreamTest.cc
index 3b3f78cea..324d8c5e6 100644
--- a/cpp/velox/tests/BufferOutputStreamTest.cc
+++ b/cpp/velox/tests/BufferOutputStreamTest.cc
@@ -55,16 +55,16 @@ TEST_F(BufferOutputStreamTest, outputStream) {
reference->write(data.data(), data.size());
}
auto str = referenceSStream.str();
- auto numBytes = veloxPool_->currentBytes();
+ auto numBytes = veloxPool_->usedBytes();
EXPECT_LT(0, numBytes);
{
auto buffer = out->getBuffer();
- EXPECT_EQ(numBytes, veloxPool_->currentBytes());
+ EXPECT_EQ(numBytes, veloxPool_->usedBytes());
EXPECT_EQ(str, std::string(buffer->as<char>(), buffer->size()));
}
out.reset();
// We expect dropping the stream frees the backing memory.
- EXPECT_EQ(0, veloxPool_->currentBytes());
+ EXPECT_EQ(0, veloxPool_->usedBytes());
}
} // namespace gluten
diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh
index 5aa3f2b37..584c9d40d 100755
--- a/ep/build-velox/src/get_velox.sh
+++ b/ep/build-velox/src/get_velox.sh
@@ -17,7 +17,7 @@
set -exu
VELOX_REPO=https://github.com/oap-project/velox.git
-VELOX_BRANCH=2024_06_04
+VELOX_BRANCH=2024_06_05
VELOX_HOME=""
#Set on run gluten on HDFS
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
