This is an automated email from the ASF dual-hosted git repository.
lxy-9602 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new 99f8bf03 test(parquet): allocate raw Arrow reader buffers from the
default pool (#386)
99f8bf03 is described below
commit 99f8bf03c03eae9fc072538458489d9b4213ce2b
Author: Nicholas Jiang <[email protected]>
AuthorDate: Thu Sep 24 09:10:48 2026 +0800
test(parquet): allocate raw Arrow reader buffers from the default pool
(#386)
---
.../parquet/parquet_file_batch_reader_test.cpp | 14 ++++---
.../format/parquet/parquet_format_writer_test.cpp | 44 +++++++++++++---------
src/paimon/format/parquet/variant_parquet_test.cpp | 25 ++++++------
3 files changed, 48 insertions(+), 35 deletions(-)
diff --git a/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp
b/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp
index f25e0527..10253fb5 100644
--- a/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp
+++ b/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp
@@ -2056,11 +2056,12 @@ TEST_F(ParquetFileBatchReaderTest,
TestDictionaryPassthroughSkipsFallbackToPlain
// its presence cannot be the signal; the data pages are what say the
column went plain. Both
// are asserted so the test fails loudly if the fixture stops producing a
mixed chunk rather
// than quietly passing for the wrong reason.
- auto metadata_file = arrow::io::ReadableFile::Open(file_path, pool_.get());
+ auto metadata_file = arrow::io::ReadableFile::Open(file_path,
arrow::default_memory_pool());
ASSERT_TRUE(metadata_file.ok());
std::unique_ptr<::parquet::arrow::FileReader> metadata_reader;
- ASSERT_TRUE(
- ::parquet::arrow::OpenFile(metadata_file.ValueOrDie(), pool_.get(),
&metadata_reader).ok());
+ ASSERT_TRUE(::parquet::arrow::OpenFile(metadata_file.ValueOrDie(),
arrow::default_memory_pool(),
+ &metadata_reader)
+ .ok());
std::unique_ptr<::parquet::ColumnChunkMetaData> column_chunk =
metadata_reader->parquet_reader()->metadata()->RowGroup(0)->ColumnChunk(0);
ASSERT_TRUE(column_chunk->has_dictionary_page());
@@ -2151,11 +2152,12 @@ TEST_F(ParquetFileBatchReaderTest,
TestDictionaryPassthroughRequiresEveryRowGrou
// Pin the fixture: without this the read assertion below would also pass
on a file whose
// first row group was never dictionary-encoded in the first place.
- auto metadata_file = arrow::io::ReadableFile::Open(file_path, pool_.get());
+ auto metadata_file = arrow::io::ReadableFile::Open(file_path,
arrow::default_memory_pool());
ASSERT_TRUE(metadata_file.ok());
std::unique_ptr<::parquet::arrow::FileReader> metadata_reader;
- ASSERT_TRUE(
- ::parquet::arrow::OpenFile(metadata_file.ValueOrDie(), pool_.get(),
&metadata_reader).ok());
+ ASSERT_TRUE(::parquet::arrow::OpenFile(metadata_file.ValueOrDie(),
arrow::default_memory_pool(),
+ &metadata_reader)
+ .ok());
std::shared_ptr<::parquet::FileMetaData> metadata =
metadata_reader->parquet_reader()->metadata();
ASSERT_EQ(2, metadata->num_row_groups());
diff --git a/src/paimon/format/parquet/parquet_format_writer_test.cpp
b/src/paimon/format/parquet/parquet_format_writer_test.cpp
index 9ba27646..2915f9a0 100644
--- a/src/paimon/format/parquet/parquet_format_writer_test.cpp
+++ b/src/paimon/format/parquet/parquet_format_writer_test.cpp
@@ -142,10 +142,13 @@ class ParquetFormatWriterTest : public ::testing::Test {
void CheckResult(const std::string& file_path, int32_t row_count,
int32_t row_group_count) const {
- auto file = arrow::io::ReadableFile::Open(file_path,
arrow_pool_.get());
- ASSERT_TRUE(file.ok());
+ // Use the process-wide pool: pre-buffered reads may release their
buffers on Arrow IO
+ // threads after this fixture's pool is destroyed.
+ auto file = arrow::io::ReadableFile::Open(file_path,
arrow::default_memory_pool());
+ ASSERT_TRUE(file.ok()) << file.status().ToString();
std::unique_ptr<::parquet::arrow::FileReader> reader;
- auto status = ::parquet::arrow::OpenFile(file.ValueOrDie(),
arrow_pool_.get(), &reader);
+ auto status =
+ ::parquet::arrow::OpenFile(file.ValueOrDie(),
arrow::default_memory_pool(), &reader);
ASSERT_TRUE(status.ok()) << status.ToString();
const ::parquet::FileMetaData* metadata =
reader->parquet_reader()->metadata().get();
const ::parquet::SchemaDescriptor* schema = metadata->schema();
@@ -266,10 +269,11 @@ class ParquetFormatWriterTest : public ::testing::Test {
void CheckEncodedResult(const std::string& file_path, int32_t row_count,
bool null_in_dictionary) const {
- auto file = arrow::io::ReadableFile::Open(file_path,
arrow_pool_.get());
- ASSERT_TRUE(file.ok());
+ auto file = arrow::io::ReadableFile::Open(file_path,
arrow::default_memory_pool());
+ ASSERT_TRUE(file.ok()) << file.status().ToString();
std::unique_ptr<::parquet::arrow::FileReader> reader;
- auto status = ::parquet::arrow::OpenFile(file.ValueOrDie(),
arrow_pool_.get(), &reader);
+ auto status =
+ ::parquet::arrow::OpenFile(file.ValueOrDie(),
arrow::default_memory_pool(), &reader);
ASSERT_TRUE(status.ok()) << status.ToString();
const ::parquet::FileMetaData* metadata =
reader->parquet_reader()->metadata().get();
ASSERT_EQ(metadata->num_rows(), row_count);
@@ -636,10 +640,11 @@ TEST_F(ParquetFormatWriterTest,
TestWriteDictionaryChangingAcrossBatches) {
ASSERT_OK(out->Flush());
ASSERT_OK(out->Close());
- auto file = arrow::io::ReadableFile::Open(file_path, arrow_pool_.get());
- ASSERT_TRUE(file.ok());
+ auto file = arrow::io::ReadableFile::Open(file_path,
arrow::default_memory_pool());
+ ASSERT_TRUE(file.ok()) << file.status().ToString();
std::unique_ptr<::parquet::arrow::FileReader> reader;
- auto status = ::parquet::arrow::OpenFile(file.ValueOrDie(),
arrow_pool_.get(), &reader);
+ auto status =
+ ::parquet::arrow::OpenFile(file.ValueOrDie(),
arrow::default_memory_pool(), &reader);
ASSERT_TRUE(status.ok()) << status.ToString();
const ::parquet::FileMetaData* metadata =
reader->parquet_reader()->metadata().get();
ASSERT_EQ(3 * kBatchRows, metadata->num_rows());
@@ -708,10 +713,11 @@ TEST_F(ParquetFormatWriterTest,
TestWriteDictionaryWithNullRows) {
ASSERT_OK(out->Flush());
ASSERT_OK(out->Close());
- auto file = arrow::io::ReadableFile::Open(file_path, arrow_pool_.get());
- ASSERT_TRUE(file.ok());
+ auto file = arrow::io::ReadableFile::Open(file_path,
arrow::default_memory_pool());
+ ASSERT_TRUE(file.ok()) << file.status().ToString();
std::unique_ptr<::parquet::arrow::FileReader> reader;
- auto status = ::parquet::arrow::OpenFile(file.ValueOrDie(),
arrow_pool_.get(), &reader);
+ auto status =
+ ::parquet::arrow::OpenFile(file.ValueOrDie(),
arrow::default_memory_pool(), &reader);
ASSERT_TRUE(status.ok()) << status.ToString();
const ::parquet::FileMetaData* metadata =
reader->parquet_reader()->metadata().get();
ASSERT_EQ(6, metadata->num_rows());
@@ -766,10 +772,11 @@ TEST_F(ParquetFormatWriterTest,
TestWriteDictionaryWithDuplicateValues) {
ASSERT_OK(out->Flush());
ASSERT_OK(out->Close());
- auto file = arrow::io::ReadableFile::Open(file_path, arrow_pool_.get());
- ASSERT_TRUE(file.ok());
+ auto file = arrow::io::ReadableFile::Open(file_path,
arrow::default_memory_pool());
+ ASSERT_TRUE(file.ok()) << file.status().ToString();
std::unique_ptr<::parquet::arrow::FileReader> reader;
- auto status = ::parquet::arrow::OpenFile(file.ValueOrDie(),
arrow_pool_.get(), &reader);
+ auto status =
+ ::parquet::arrow::OpenFile(file.ValueOrDie(),
arrow::default_memory_pool(), &reader);
ASSERT_TRUE(status.ok()) << status.ToString();
const ::parquet::FileMetaData* metadata =
reader->parquet_reader()->metadata().get();
ASSERT_EQ(4, metadata->num_rows());
@@ -889,10 +896,11 @@ TEST_F(ParquetFormatWriterTest,
TestWriteDictionaryOfBinaryColumn) {
ASSERT_OK(out->Flush());
ASSERT_OK(out->Close());
- auto file = arrow::io::ReadableFile::Open(file_path, arrow_pool_.get());
- ASSERT_TRUE(file.ok());
+ auto file = arrow::io::ReadableFile::Open(file_path,
arrow::default_memory_pool());
+ ASSERT_TRUE(file.ok()) << file.status().ToString();
std::unique_ptr<::parquet::arrow::FileReader> reader;
- auto status = ::parquet::arrow::OpenFile(file.ValueOrDie(),
arrow_pool_.get(), &reader);
+ auto status =
+ ::parquet::arrow::OpenFile(file.ValueOrDie(),
arrow::default_memory_pool(), &reader);
ASSERT_TRUE(status.ok()) << status.ToString();
const ::parquet::FileMetaData* metadata =
reader->parquet_reader()->metadata().get();
ASSERT_EQ(4, metadata->num_rows());
diff --git a/src/paimon/format/parquet/variant_parquet_test.cpp
b/src/paimon/format/parquet/variant_parquet_test.cpp
index 94c501f5..c3bccce2 100644
--- a/src/paimon/format/parquet/variant_parquet_test.cpp
+++ b/src/paimon/format/parquet/variant_parquet_test.cpp
@@ -545,10 +545,11 @@ TEST_F(VariantParquetTest, PhysicalLayoutMatchesJava) {
// reader is required because these parquet-level properties (repetition,
physical types,
// field ids, the absence of a logical-type annotation) are not visible in
the Arrow schema
// surfaced by the paimon reader.
- auto file = arrow::io::ReadableFile::Open(file_path_, arrow_pool_.get());
- ASSERT_TRUE(file.ok());
+ auto file = arrow::io::ReadableFile::Open(file_path_,
arrow::default_memory_pool());
+ ASSERT_TRUE(file.ok()) << file.status().ToString();
std::unique_ptr<::parquet::arrow::FileReader> reader;
- auto status = ::parquet::arrow::OpenFile(file.ValueOrDie(),
arrow_pool_.get(), &reader);
+ auto status =
+ ::parquet::arrow::OpenFile(file.ValueOrDie(),
arrow::default_memory_pool(), &reader);
ASSERT_TRUE(status.ok()) << status.ToString();
const ::parquet::SchemaDescriptor* schema =
reader->parquet_reader()->metadata()->schema();
ASSERT_EQ(schema->num_columns(), 3);
@@ -591,12 +592,14 @@ TEST_F(VariantParquetTest, WriteAndReadRoundTrip) {
{
// Sanity-check the raw file through the plain parquet-arrow reader:
the struct child
- // arrays must align with the logical rows.
- auto file = arrow::io::ReadableFile::Open(file_path_,
arrow_pool_.get());
- ASSERT_TRUE(file.ok());
+ // arrays must align with the logical rows. Use the process-wide pool:
pre-buffered reads
+ // may release their buffers on Arrow IO threads after this fixture's
pool is destroyed.
+ auto file = arrow::io::ReadableFile::Open(file_path_,
arrow::default_memory_pool());
+ ASSERT_TRUE(file.ok()) << file.status().ToString();
std::unique_ptr<::parquet::arrow::FileReader> raw_reader;
ASSERT_TRUE(
- ::parquet::arrow::OpenFile(file.ValueOrDie(), arrow_pool_.get(),
&raw_reader).ok());
+ ::parquet::arrow::OpenFile(file.ValueOrDie(),
arrow::default_memory_pool(), &raw_reader)
+ .ok());
std::shared_ptr<arrow::Table> table;
ASSERT_TRUE(raw_reader->ReadTable(&table).ok());
auto raw_variant =
checked_pointer_cast<arrow::StructArray>(table->column(1)->chunk(0));
@@ -712,11 +715,11 @@ TEST_F(VariantParquetTest, ShreddedWriteAndReadRoundTrip)
{
}
{
- auto file = arrow::io::ReadableFile::Open(file_path_,
arrow_pool_.get());
- ASSERT_TRUE(file.ok());
+ auto file = arrow::io::ReadableFile::Open(file_path_,
arrow::default_memory_pool());
+ ASSERT_TRUE(file.ok()) << file.status().ToString();
std::unique_ptr<::parquet::arrow::FileReader> reader;
- auto status =
- ::parquet::arrow::OpenFile(file.ValueOrDie(),
arrow_pool_.get(), &reader);
+ auto status = ::parquet::arrow::OpenFile(file.ValueOrDie(),
+
arrow::default_memory_pool(), &reader);
ASSERT_TRUE(status.ok()) << status.ToString();
const auto* root =
reader->parquet_reader()->metadata()->schema()->group_node();
ASSERT_EQ(root->field_count(), 2);
