This is an automated email from the ASF dual-hosted git repository.
marong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 40d3a64f9d [GLUTEN-11259][VL][CORE] Reverse memory for some vector
(#11260)
40d3a64f9d is described below
commit 40d3a64f9d2d0487657ca65a3638ed2da9a9e1fd
Author: Jiaan Geng <[email protected]>
AuthorDate: Tue Dec 9 11:03:08 2025 +0800
[GLUTEN-11259][VL][CORE] Reverse memory for some vector (#11260)
---
cpp/core/jni/JniWrapper.cc | 2 ++
cpp/core/shuffle/Utils.cc | 1 +
cpp/velox/benchmarks/GenericBenchmark.cc | 2 ++
cpp/velox/compute/iceberg/IcebergWriter.cc | 1 +
cpp/velox/jni/JniFileSystem.cc | 1 +
cpp/velox/jni/VeloxJniWrapper.cc | 1 +
cpp/velox/shuffle/VeloxHashShuffleWriter.cc | 3 +++
cpp/velox/shuffle/VeloxRssSortShuffleWriter.cc | 1 +
cpp/velox/shuffle/VeloxSortShuffleWriter.cc | 1 +
cpp/velox/substrait/SubstraitToVeloxPlan.cc | 1 +
cpp/velox/tests/MemoryManagerTest.cc | 3 +++
cpp/velox/tests/VeloxBatchResizerTest.cc | 1 +
cpp/velox/tests/VeloxSubstraitRoundTripTest.cc | 1 +
13 files changed, 19 insertions(+)
diff --git a/cpp/core/jni/JniWrapper.cc b/cpp/core/jni/JniWrapper.cc
index bfab41e365..adada15f91 100644
--- a/cpp/core/jni/JniWrapper.cc
+++ b/cpp/core/jni/JniWrapper.cc
@@ -490,6 +490,7 @@
Java_org_apache_gluten_vectorized_PlanEvaluatorJniWrapper_nativeCreateKernelWith
// Handle the Java iters
jsize itersLen = env->GetArrayLength(iterArr);
std::vector<std::shared_ptr<ResultIterator>> inputIters;
+ inputIters.reserve(itersLen);
for (int idx = 0; idx < itersLen; idx++) {
jobject iter = env->GetObjectArrayElement(iterArr, idx);
auto arrayIter = std::make_unique<JniColumnarBatchIterator>(env, iter,
ctx, idx);
@@ -835,6 +836,7 @@ JNIEXPORT jlong JNICALL
Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWra
auto safeArray = getIntArrayElementsSafe(env, jcolumnIndices);
int size = env->GetArrayLength(jcolumnIndices);
std::vector<int32_t> columnIndices;
+ columnIndices.reserve(size);
for (int32_t i = 0; i < size; i++) {
columnIndices.push_back(safeArray.elems()[i]);
}
diff --git a/cpp/core/shuffle/Utils.cc b/cpp/core/shuffle/Utils.cc
index 51667274ed..d2c9c6fe6d 100644
--- a/cpp/core/shuffle/Utils.cc
+++ b/cpp/core/shuffle/Utils.cc
@@ -190,6 +190,7 @@ arrow::Result<std::string>
gluten::createTempShuffleFile(const std::string& dir)
arrow::Result<std::vector<std::shared_ptr<arrow::DataType>>>
gluten::toShuffleTypeId(
const std::vector<std::shared_ptr<arrow::Field>>& fields) {
std::vector<std::shared_ptr<arrow::DataType>> shuffleTypeId;
+ shuffleTypeId.reserve(fields.size());
for (auto field : fields) {
switch (field->type()->id()) {
case arrow::BooleanType::type_id:
diff --git a/cpp/velox/benchmarks/GenericBenchmark.cc
b/cpp/velox/benchmarks/GenericBenchmark.cc
index 811ca82697..2121254137 100644
--- a/cpp/velox/benchmarks/GenericBenchmark.cc
+++ b/cpp/velox/benchmarks/GenericBenchmark.cc
@@ -402,6 +402,7 @@ auto BM_Generic = [](::benchmark::State& state,
auto plan = getPlanFromFile("Plan", planFile);
std::vector<std::string> splits{};
+ splits.reserve(splitFiles.size());
for (const auto& splitFile : splitFiles) {
splits.push_back(getPlanFromFile("ReadRel.LocalFiles", splitFile));
}
@@ -428,6 +429,7 @@ auto BM_Generic = [](::benchmark::State& state,
ScopedTimer timer(&elapsedTime);
for (auto _ : state) {
std::vector<std::shared_ptr<gluten::ResultIterator>> inputIters;
+ inputIters.reserve(dataFiles.size());
std::vector<FileReaderIterator*> inputItersRaw;
if (!dataFiles.empty()) {
for (const auto& input : dataFiles) {
diff --git a/cpp/velox/compute/iceberg/IcebergWriter.cc
b/cpp/velox/compute/iceberg/IcebergWriter.cc
index 0e671f1773..2e9aa86605 100644
--- a/cpp/velox/compute/iceberg/IcebergWriter.cc
+++ b/cpp/velox/compute/iceberg/IcebergWriter.cc
@@ -54,6 +54,7 @@ std::shared_ptr<IcebergInsertTableHandle>
createIcebergInsertTableHandle(
std::vector<std::shared_ptr<const iceberg::IcebergColumnHandle>>
columnHandles;
std::vector<std::string> columnNames = outputRowType->names();
+ columnHandles.reserve(columnNames.size());
std::vector<TypePtr> columnTypes = outputRowType->children();
std::vector<std::string> partitionColumns;
partitionColumns.reserve(spec->fields.size());
diff --git a/cpp/velox/jni/JniFileSystem.cc b/cpp/velox/jni/JniFileSystem.cc
index 024d745b2b..a8b7bf3fdb 100644
--- a/cpp/velox/jni/JniFileSystem.cc
+++ b/cpp/velox/jni/JniFileSystem.cc
@@ -347,6 +347,7 @@ class JniFileSystem : public
facebook::velox::filesystems::FileSystem {
static_cast<jobjectArray>(env->CallObjectMethod(obj_,
jniFileSystemList, createJString(env, path)));
checkException(env);
jsize length = env->GetArrayLength(jarray);
+ out.reserve(length);
for (jsize i = 0; i < length; ++i) {
jstring element =
static_cast<jstring>(env->GetObjectArrayElement(jarray, i));
std::string cElement = jStringToCString(env, element);
diff --git a/cpp/velox/jni/VeloxJniWrapper.cc b/cpp/velox/jni/VeloxJniWrapper.cc
index f44fed0888..a0889cf5a9 100644
--- a/cpp/velox/jni/VeloxJniWrapper.cc
+++ b/cpp/velox/jni/VeloxJniWrapper.cc
@@ -262,6 +262,7 @@ JNIEXPORT jlong JNICALL
Java_org_apache_gluten_columnarbatch_VeloxColumnarBatchJ
auto safeArray = getLongArrayElementsSafe(env, batchHandles);
std::vector<std::shared_ptr<ColumnarBatch>> batches;
+ batches.reserve(handleCount);
for (int i = 0; i < handleCount; ++i) {
int64_t handle = safeArray.elems()[i];
auto batch = ObjectStore::retrieve<ColumnarBatch>(handle);
diff --git a/cpp/velox/shuffle/VeloxHashShuffleWriter.cc
b/cpp/velox/shuffle/VeloxHashShuffleWriter.cc
index e83ce8a566..6a1298d743 100644
--- a/cpp/velox/shuffle/VeloxHashShuffleWriter.cc
+++ b/cpp/velox/shuffle/VeloxHashShuffleWriter.cc
@@ -277,6 +277,7 @@ arrow::Status
VeloxHashShuffleWriter::write(std::shared_ptr<ColumnarBatch> cb, i
}
END_TIMING();
std::vector<int32_t> range;
+ range.reserve(numColumns);
for (int32_t i = 1; i < numColumns; i++) {
range.push_back(i);
}
@@ -377,6 +378,7 @@ arrow::Status
VeloxHashShuffleWriter::buildPartition2Row(uint32_t rowNum) {
// calc valid partition list
partitionUsed_.clear();
+ partitionUsed_.reserve(numPartitions_);
for (auto pid = 0; pid != numPartitions_; ++pid) {
if (partition2RowCount_[pid] > 0) {
partitionUsed_.push_back(pid);
@@ -737,6 +739,7 @@ arrow::Status
VeloxHashShuffleWriter::splitComplexType(const facebook::velox::Ro
arrow::Status VeloxHashShuffleWriter::initColumnTypes(const
facebook::velox::RowVector& rv) {
schema_ = toArrowSchema(rv.type(), veloxPool_.get());
+ veloxColumnTypes_.reserve(rv.childrenSize());
for (size_t i = 0; i < rv.childrenSize(); ++i) {
veloxColumnTypes_.push_back(rv.childAt(i)->type());
}
diff --git a/cpp/velox/shuffle/VeloxRssSortShuffleWriter.cc
b/cpp/velox/shuffle/VeloxRssSortShuffleWriter.cc
index c93045c1b5..54e2539720 100644
--- a/cpp/velox/shuffle/VeloxRssSortShuffleWriter.cc
+++ b/cpp/velox/shuffle/VeloxRssSortShuffleWriter.cc
@@ -84,6 +84,7 @@ arrow::Status
VeloxRssSortShuffleWriter::write(std::shared_ptr<ColumnarBatch> cb
RETURN_NOT_OK(partitioner_->compute(pidArr, pidBatch->numRows(),
batches_.size(), rowVectorIndexMap_));
END_TIMING();
std::vector<int32_t> range;
+ range.reserve(numColumns);
for (int32_t i = 1; i < numColumns; i++) {
range.push_back(i);
}
diff --git a/cpp/velox/shuffle/VeloxSortShuffleWriter.cc
b/cpp/velox/shuffle/VeloxSortShuffleWriter.cc
index eb5bb89caf..7a4066153b 100644
--- a/cpp/velox/shuffle/VeloxSortShuffleWriter.cc
+++ b/cpp/velox/shuffle/VeloxSortShuffleWriter.cc
@@ -138,6 +138,7 @@ arrow::Result<facebook::velox::RowVectorPtr>
VeloxSortShuffleWriter::getPeeledRo
RETURN_NOT_OK(partitioner_->compute(pidArr, pidBatch->numRows(),
row2Partition_));
std::vector<int32_t> range;
+ range.reserve(numColumns);
for (int32_t i = 1; i < numColumns; i++) {
range.push_back(i);
}
diff --git a/cpp/velox/substrait/SubstraitToVeloxPlan.cc
b/cpp/velox/substrait/SubstraitToVeloxPlan.cc
index eb10a67e85..e93ea75ba8 100644
--- a/cpp/velox/substrait/SubstraitToVeloxPlan.cc
+++ b/cpp/velox/substrait/SubstraitToVeloxPlan.cc
@@ -674,6 +674,7 @@ std::shared_ptr<CudfHiveInsertTableHandle>
makeCudfHiveInsertTableHandle(
const std::unordered_map<std::string, std::string>& serdeParameters,
const std::shared_ptr<dwio::common::WriterOptions>& writerOptions) {
std::vector<std::shared_ptr<const CudfHiveColumnHandle>> columnHandles;
+ columnHandles.reserve(tableColumnNames.size());
for (int i = 0; i < tableColumnNames.size(); ++i) {
columnHandles.push_back(std::make_shared<CudfHiveColumnHandle>(
diff --git a/cpp/velox/tests/MemoryManagerTest.cc
b/cpp/velox/tests/MemoryManagerTest.cc
index 8dd41aec7d..1367e2e048 100644
--- a/cpp/velox/tests/MemoryManagerTest.cc
+++ b/cpp/velox/tests/MemoryManagerTest.cc
@@ -78,6 +78,7 @@ TEST_F(MemoryManagerTest, memoryPoolWithBlockReseravtion) {
std::vector<Allocation> allocations;
std::vector<uint64_t> sizes{
kMemoryReservationBlockSizeDefault - 1 * kMB,
kMemoryReservationBlockSizeDefault - 2 * kMB};
+ allocations.reserve(sizes.size());
for (const auto& size : sizes) {
auto buf = pool->allocate(size);
allocations.push_back({buf, size, pool.get()});
@@ -100,6 +101,7 @@ TEST_F(MemoryManagerTest,
memoryAllocatorWithBlockReservation) {
std::vector<Allocation> allocations;
std::vector<uint64_t> sizes{
kMemoryReservationBlockSizeDefault - 1 * kMB,
kMemoryReservationBlockSizeDefault - 2 * kMB};
+ allocations.reserve(sizes.size());
for (auto i = 0; i < sizes.size(); i++) {
auto size = sizes[i];
auto currentBytes = allocator_->getBytes();
@@ -353,6 +355,7 @@ TEST_F(MultiMemoryManagerTest, spill) {
std::vector<std::unique_ptr<VeloxMemoryManager>> vmms{};
std::vector<std::thread> threads{};
std::vector<std::vector<void*>> buffs{};
+ buffs.reserve(numThreads);
for (size_t i = 0; i < numThreads; ++i) {
buffs.push_back({});
vmms.emplace_back(nullptr);
diff --git a/cpp/velox/tests/VeloxBatchResizerTest.cc
b/cpp/velox/tests/VeloxBatchResizerTest.cc
index f1157991e5..f5333f8b62 100644
--- a/cpp/velox/tests/VeloxBatchResizerTest.cc
+++ b/cpp/velox/tests/VeloxBatchResizerTest.cc
@@ -55,6 +55,7 @@ class VeloxBatchResizerTest : public ::testing::Test, public
test::VectorTestBas
void checkResize(int32_t min, int32_t max, int64_t preferredBatchBytes,
std::vector<int32_t> inSizes, std::vector<int32_t> outSizes) {
auto inBatches = std::vector<std::shared_ptr<ColumnarBatch>>();
+ inBatches.reserve(inSizes.size());
for (const auto& size : inSizes) {
inBatches.push_back(std::make_shared<VeloxColumnarBatch>(newVector(size)));
}
diff --git a/cpp/velox/tests/VeloxSubstraitRoundTripTest.cc
b/cpp/velox/tests/VeloxSubstraitRoundTripTest.cc
index cec61afd07..7b8e8336ea 100644
--- a/cpp/velox/tests/VeloxSubstraitRoundTripTest.cc
+++ b/cpp/velox/tests/VeloxSubstraitRoundTripTest.cc
@@ -47,6 +47,7 @@ class VeloxSubstraitRoundTripTest : public OperatorTestBase {
/// @param batchSize The batch Size of the data.
std::vector<RowVectorPtr> makeVectors(int64_t size, int64_t childSize,
int64_t batchSize) {
std::vector<RowVectorPtr> vectors;
+ vectors.reserve(size);
std::mt19937 gen(std::mt19937::default_seed);
for (int i = 0; i < size; i++) {
std::vector<VectorPtr> children;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
