This is an automated email from the ASF dual-hosted git repository.
pitrou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 896e91308d GH-51264: [C++][Parquet] Make MemoryPool settable on
ReaderProperties (#51269)
896e91308d is described below
commit 896e91308d1d25b906421ad5775e0604fea12b76
Author: Diveyam Mishra <[email protected]>
AuthorDate: Mon Sep 14 18:38:09 2026 +0530
GH-51264: [C++][Parquet] Make MemoryPool settable on ReaderProperties
(#51269)
**Rationale for this change**
In arrow::dataset::MakeReaderProperties, all parquet::ReaderProperties
options had to be copied field-by-field from
parquet_scan_options->reader_properties because ReaderProperties did not
support mutating or setting MemoryPool* after construction.
**What changes are included in this PR?**
Added set_memory_pool to parquet::ReaderProperties in
cpp/src/parquet/properties.h.
Added unit test TestReaderProperties.SetMemoryPool in
cpp/src/parquet/properties_test.cc.
Are these changes tested?
Yes
Are there any user-facing changes?
No
* GitHub Issue: #51264
Authored-by: Diveyam Mishra <[email protected]>
Signed-off-by: Antoine Pitrou <[email protected]>
---
cpp/src/arrow/dataset/file_parquet.cc | 31 +++++--------------------------
cpp/src/parquet/properties.h | 2 ++
cpp/src/parquet/properties_test.cc | 17 +++++++++++++++++
python/pyarrow/includes/libparquet.pxd | 3 +++
4 files changed, 27 insertions(+), 26 deletions(-)
diff --git a/cpp/src/arrow/dataset/file_parquet.cc
b/cpp/src/arrow/dataset/file_parquet.cc
index a1fcfe7090..8aa96926e4 100644
--- a/cpp/src/arrow/dataset/file_parquet.cc
+++ b/cpp/src/arrow/dataset/file_parquet.cc
@@ -68,28 +68,19 @@ parquet::ReaderProperties MakeReaderProperties(
const ParquetFileFormat& format, ParquetFragmentScanOptions*
parquet_scan_options,
const std::string& path = "", std::shared_ptr<fs::FileSystem> filesystem =
nullptr,
MemoryPool* pool = default_memory_pool()) {
- // FIXME (GH-51264): Can't mutate pool after ReaderProperties construction.
- parquet::ReaderProperties properties(pool);
- if (parquet_scan_options->reader_properties->is_buffered_stream_enabled()) {
- properties.enable_buffered_stream();
- } else {
- properties.disable_buffered_stream();
- }
-
properties.set_buffer_size(parquet_scan_options->reader_properties->buffer_size());
- properties.set_footer_read_size(
- parquet_scan_options->reader_properties->footer_read_size());
-
- auto file_decryption_prop =
- parquet_scan_options->reader_properties->file_decryption_properties();
+ parquet::ReaderProperties properties =
*parquet_scan_options->reader_properties;
+ properties.set_memory_pool(pool);
+ properties.disable_read_dense_for_nullable();
#ifdef PARQUET_REQUIRE_ENCRYPTION
auto parquet_decrypt_config =
parquet_scan_options->parquet_decryption_config;
if (parquet_decrypt_config != nullptr) {
- file_decryption_prop =
+ auto file_decryption_prop =
parquet_decrypt_config->crypto_factory->GetFileDecryptionProperties(
*parquet_decrypt_config->kms_connection_config,
*parquet_decrypt_config->decryption_config, path, filesystem);
+ properties.file_decryption_properties(file_decryption_prop);
}
#else
if (parquet_scan_options->parquet_decryption_config != nullptr) {
@@ -97,18 +88,6 @@ parquet::ReaderProperties MakeReaderProperties(
}
#endif
- properties.file_decryption_properties(file_decryption_prop);
-
- properties.set_thrift_string_size_limit(
- parquet_scan_options->reader_properties->thrift_string_size_limit());
- properties.set_thrift_container_size_limit(
- parquet_scan_options->reader_properties->thrift_container_size_limit());
- properties.set_schema_depth_limit(
- parquet_scan_options->reader_properties->schema_depth_limit());
-
- properties.set_page_checksum_verification(
- parquet_scan_options->reader_properties->page_checksum_verification());
-
return properties;
}
diff --git a/cpp/src/parquet/properties.h b/cpp/src/parquet/properties.h
index f135faa018..f8ad605dec 100644
--- a/cpp/src/parquet/properties.h
+++ b/cpp/src/parquet/properties.h
@@ -81,6 +81,8 @@ class PARQUET_EXPORT ReaderProperties {
: pool_(pool) {}
MemoryPool* memory_pool() const { return pool_; }
+ /// Set the memory pool.
+ void set_memory_pool(MemoryPool* pool) { pool_ = pool; }
std::shared_ptr<ArrowInputStream> GetStream(std::shared_ptr<ArrowInputFile>
source,
int64_t start, int64_t
num_bytes);
diff --git a/cpp/src/parquet/properties_test.cc
b/cpp/src/parquet/properties_test.cc
index 324ea2026a..2f22844f8b 100644
--- a/cpp/src/parquet/properties_test.cc
+++ b/cpp/src/parquet/properties_test.cc
@@ -24,6 +24,7 @@
#include "arrow/buffer.h"
#include "arrow/io/memory.h"
+#include "arrow/memory_pool.h"
#include "parquet/file_reader.h"
#include "parquet/properties.h"
@@ -44,6 +45,22 @@ TEST(TestReaderProperties, Basics) {
ASSERT_FALSE(props.page_checksum_verification());
}
+TEST(TestReaderProperties, SetMemoryPool) {
+ ReaderProperties props;
+ ASSERT_EQ(props.memory_pool(), ::arrow::default_memory_pool());
+
+ ::arrow::ProxyMemoryPool custom_pool(::arrow::default_memory_pool());
+ props.set_memory_pool(&custom_pool);
+ ASSERT_EQ(props.memory_pool(), &custom_pool);
+
+ ReaderProperties copied_props = props;
+ ASSERT_EQ(copied_props.memory_pool(), &custom_pool);
+
+ copied_props.set_memory_pool(::arrow::default_memory_pool());
+ ASSERT_EQ(copied_props.memory_pool(), ::arrow::default_memory_pool());
+ ASSERT_EQ(props.memory_pool(), &custom_pool);
+}
+
TEST(TestWriterProperties, Basics) {
std::shared_ptr<WriterProperties> props =
WriterProperties::Builder().build();
diff --git a/python/pyarrow/includes/libparquet.pxd
b/python/pyarrow/includes/libparquet.pxd
index 915b1d6dd3..3a5f91716c 100644
--- a/python/pyarrow/includes/libparquet.pxd
+++ b/python/pyarrow/includes/libparquet.pxd
@@ -418,6 +418,9 @@ cdef extern from "parquet/api/reader.h" namespace "parquet"
nogil:
uint32_t* metadata_len)
cdef cppclass CReaderProperties" parquet::ReaderProperties":
+ CMemoryPool* memory_pool() const
+ void set_memory_pool(CMemoryPool* pool)
+
c_bool is_buffered_stream_enabled() const
void enable_buffered_stream()
void disable_buffered_stream()
