This is an automated email from the ASF dual-hosted git repository.
marong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 7e5172ab21 [GLUTEN-6887][VL] Daily Update Velox Version (2024_11_10)
(#7881)
7e5172ab21 is described below
commit 7e5172ab2131faa22a2f933d8020d03ac1936b36
Author: Gluten Performance Bot
<[email protected]>
AuthorDate: Mon Nov 11 18:44:18 2024 +0800
[GLUTEN-6887][VL] Daily Update Velox Version (2024_11_10) (#7881)
62e589ecc by Xiaoxuan Meng, Support different vector serialization format
for streaming shuffle (11445)
ec825034e by Ke, Add faulty injection in writer fuzzer (11375)
5e0f52c52 by Richard Barnes, Remove unused-variable in
velox/common/memory/tests/MockSharedArbitratorTest.cpp (11485)
9a1718142 by Richard Barnes, Remove unused-variable in
velox/experimental/wave/exec/Wave.cpp (11484)
4a79bc55f by Bikramjeet Vig, Add support for testing peeling in expression
fuzzer (11379)
a33e8d733 by Jimmy Lu, Fix integer overflow while skipping on a stream
(11477)
5544d0a83 by Bikramjeet Vig, Add option to disable flatenning optimization
in PrestoSerializer (11465)
b146deea0 by Jialiang Tan, Fix
MockSharedArbitrationTest.localArbitrationRunInParallelWithGlobalArbitration
(11476)
---
cpp/velox/compute/VeloxBackend.cc | 4 ++++
cpp/velox/shuffle/VeloxShuffleReader.cc | 5 +++--
cpp/velox/shuffle/VeloxShuffleReader.h | 1 +
cpp/velox/utils/tests/VeloxShuffleWriterTestBase.h | 4 ++++
ep/build-velox/src/get_velox.sh | 2 +-
5 files changed, 13 insertions(+), 3 deletions(-)
diff --git a/cpp/velox/compute/VeloxBackend.cc
b/cpp/velox/compute/VeloxBackend.cc
index d39b0902c2..c1e907bea1 100644
--- a/cpp/velox/compute/VeloxBackend.cc
+++ b/cpp/velox/compute/VeloxBackend.cc
@@ -162,6 +162,10 @@ void VeloxBackend::init(const
std::unordered_map<std::string, std::string>& conf
// serde, for spill
facebook::velox::serializer::presto::PrestoVectorSerde::registerVectorSerde();
}
+ if
(!isRegisteredNamedVectorSerde(facebook::velox::VectorSerde::Kind::kPresto)) {
+ // RSS shuffle serde.
+
facebook::velox::serializer::presto::PrestoVectorSerde::registerNamedVectorSerde();
+ }
velox::exec::Operator::registerOperator(std::make_unique<RowVectorStreamOperatorTranslator>());
initUdf();
diff --git a/cpp/velox/shuffle/VeloxShuffleReader.cc
b/cpp/velox/shuffle/VeloxShuffleReader.cc
index 32e346d737..92751d454a 100644
--- a/cpp/velox/shuffle/VeloxShuffleReader.cc
+++ b/cpp/velox/shuffle/VeloxShuffleReader.cc
@@ -538,6 +538,7 @@
VeloxRssSortShuffleReaderDeserializer::VeloxRssSortShuffleReaderDeserializer(
rowType_(rowType),
batchSize_(batchSize),
veloxCompressionType_(veloxCompressionType),
+ serde_(getNamedVectorSerde(facebook::velox::VectorSerde::Kind::kPresto)),
deserializeTime_(deserializeTime) {
constexpr uint64_t kMaxReadBufferSize = (1 << 20) -
AlignedBuffer::kPaddedSize;
auto buffer = AlignedBuffer::allocate<char>(kMaxReadBufferSize,
veloxPool_.get());
@@ -553,7 +554,7 @@ std::shared_ptr<ColumnarBatch>
VeloxRssSortShuffleReaderDeserializer::next() {
ScopedTimer timer(&deserializeTime_);
RowVectorPtr rowVector;
- VectorStreamGroup::read(in_.get(), veloxPool_.get(), rowType_, &rowVector,
&serdeOptions_);
+ VectorStreamGroup::read(in_.get(), veloxPool_.get(), rowType_, serde_,
&rowVector, &serdeOptions_);
if (rowVector->size() >= batchSize_) {
return std::make_shared<VeloxColumnarBatch>(std::move(rowVector));
@@ -561,7 +562,7 @@ std::shared_ptr<ColumnarBatch>
VeloxRssSortShuffleReaderDeserializer::next() {
while (rowVector->size() < batchSize_ && in_->hasNext()) {
RowVectorPtr rowVectorTemp;
- VectorStreamGroup::read(in_.get(), veloxPool_.get(), rowType_,
&rowVectorTemp, &serdeOptions_);
+ VectorStreamGroup::read(in_.get(), veloxPool_.get(), rowType_, serde_,
&rowVectorTemp, &serdeOptions_);
rowVector->append(rowVectorTemp.get());
}
diff --git a/cpp/velox/shuffle/VeloxShuffleReader.h
b/cpp/velox/shuffle/VeloxShuffleReader.h
index f7ff05c5d1..d39e389363 100644
--- a/cpp/velox/shuffle/VeloxShuffleReader.h
+++ b/cpp/velox/shuffle/VeloxShuffleReader.h
@@ -125,6 +125,7 @@ class VeloxRssSortShuffleReaderDeserializer : public
ColumnarBatchIterator {
int32_t rowCount_;
int32_t batchSize_;
facebook::velox::common::CompressionKind veloxCompressionType_;
+ facebook::velox::VectorSerde* const serde_;
facebook::velox::serializer::presto::PrestoVectorSerde::PrestoOptions
serdeOptions_;
int64_t& deserializeTime_;
std::shared_ptr<VeloxInputStream> in_;
diff --git a/cpp/velox/utils/tests/VeloxShuffleWriterTestBase.h
b/cpp/velox/utils/tests/VeloxShuffleWriterTestBase.h
index f5dd6f4f38..4b4abfb525 100644
--- a/cpp/velox/utils/tests/VeloxShuffleWriterTestBase.h
+++ b/cpp/velox/utils/tests/VeloxShuffleWriterTestBase.h
@@ -90,6 +90,10 @@ class VeloxShuffleWriterTestBase : public
facebook::velox::test::VectorTestBase
protected:
void setUp() {
+ if
(!isRegisteredNamedVectorSerde(facebook::velox::VectorSerde::Kind::kPresto)) {
+ // RSS shuffle serde.
+
facebook::velox::serializer::presto::PrestoVectorSerde::registerNamedVectorSerde();
+ }
// Set up test data.
children1_ = {
makeNullableFlatVector<int8_t>({1, 2, 3, std::nullopt, 4,
std::nullopt, 5, 6, std::nullopt, 7}),
diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh
index 1d27180699..759400de87 100755
--- a/ep/build-velox/src/get_velox.sh
+++ b/ep/build-velox/src/get_velox.sh
@@ -17,7 +17,7 @@
set -exu
VELOX_REPO=https://github.com/oap-project/velox.git
-VELOX_BRANCH=2024_11_09
+VELOX_BRANCH=2024_11_10
VELOX_HOME=""
OS=`uname -s`
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
