This is an automated email from the ASF dual-hosted git repository.
kejia pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new d992106131 [GLUTEN-9737][VL] Pass ArrowMemoryPool into velox parquet
writer (#10121)
d992106131 is described below
commit d992106131307bfdd73768d36e61e40861b9d1ee
Author: JiaKe <[email protected]>
AuthorDate: Wed Aug 13 15:05:30 2025 +0800
[GLUTEN-9737][VL] Pass ArrowMemoryPool into velox parquet writer (#10121)
---
cpp/velox/utils/VeloxWriterUtils.cc | 3 +++
1 file changed, 3 insertions(+)
diff --git a/cpp/velox/utils/VeloxWriterUtils.cc
b/cpp/velox/utils/VeloxWriterUtils.cc
index f63e9d0d82..9a9922423b 100644
--- a/cpp/velox/utils/VeloxWriterUtils.cc
+++ b/cpp/velox/utils/VeloxWriterUtils.cc
@@ -23,6 +23,7 @@
#include "utils/ConfigExtractor.h"
#include "utils/Exception.h"
+#include "memory/VeloxMemoryManager.h"
#include "velox/common/compression/Compression.h"
#include "velox/type/Type.h"
@@ -89,6 +90,8 @@ std::unique_ptr<WriterOptions> makeParquetWriteOption(const
std::unordered_map<s
return std::make_unique<LambdaFlushPolicy>(maxRowGroupRows,
maxRowGroupBytes, [&]() { return false; });
};
writeOption->parquetWriteTimestampTimeZone = getConfigValue(sparkConfs,
kSessionTimezone, std::nullopt);
+ writeOption->arrowMemoryPool =
+
getDefaultMemoryManager()->getOrCreateArrowMemoryPool("VeloxParquetWrite.ArrowMemoryPool");
if (auto it = sparkConfs.find(kParquetDataPageSize); it != sparkConfs.end())
{
auto dataPageSize = std::stoi(it->second);
writeOption->dataPageSize = dataPageSize;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
