SteNicholas commented on code in PR #563:
URL: https://github.com/apache/paimon-cpp/pull/563#discussion_r4599775034
##########
src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp:
##########
@@ -2637,63 +2628,332 @@ static std::shared_ptr<arrow::StructArray>
MakeMisalignedNestedDataWithNulls(int
.ValueOrDie();
}
-/// Test: nested columns containing NULLs, with MISALIGNED leaf pages, split
across
-/// MULTIPLE row groups. This combines the three stress dimensions: null-driven
-/// definition levels, byte-based misaligned pages, and row-group boundaries
that the
-/// selection bitmap crosses. Every returned row is deep-compared (nulls
included)
-/// against the original data identified by its non-null key.
-/// Bitmap: [0,15), [77, 87) (to avoid bitmap hole filling)
-/// Expected: 25 rows -> keys 0..14 and 77..86
TEST_F(PageFilteredRowGroupReaderTest,
NestedColumnsWithNullsMisalignedPagesMultiRowGroup) {
- std::string file_name = dir_->Str() +
"/nested_nulls_misaligned_multi_rg.parquet";
- constexpr int32_t kNumRows = 100;
- auto data = MakeMisalignedNestedDataWithNulls(kNumRows);
-
- // write_batch_size=1 + byte-based data page size -> misaligned leaf pages.
- // max_row_group_length=40 -> 3 row groups (40, 40, 20).
- WriteTestFile(file_name, data, /*write_batch_size=*/1,
/*max_row_group_length=*/40,
- /*enable_dictionary=*/false, /*data_page_size=*/40);
-
- auto read_schema =
- arrow::schema({arrow::field("key", arrow::int64()),
- arrow::field("s", arrow::struct_({arrow::field("x",
arrow::int32()),
- arrow::field("y",
arrow::int64())})),
- arrow::field("tags", arrow::list(arrow::field("item",
arrow::int32()))),
- arrow::field("props", arrow::map(arrow::utf8(),
arrow::int32()))});
-
- // bitmap: [0,15), [77, 87) -> spans all three row groups.
- RoaringBitmap32 bitmap;
- bitmap.AddRange(0, 15);
- bitmap.AddRange(77, 87);
+ constexpr int32_t kRows = 100;
+ constexpr int32_t kBatchSize = 7;
+ auto data = MakeMisalignedNestedDataWithNulls(kRows);
+ const std::vector<RowRanges> selections = {
+ RowRanges({RowRanges::Range(0, 14), RowRanges::Range(77, 86)}),
+ RowRanges({RowRanges::Range(2, 15), RowRanges::Range(43, 68),
RowRanges::Range(83, 96)}),
+ RowRanges({RowRanges::Range(2, 15), RowRanges::Range(40, 79),
RowRanges::Range(83, 96)})};
+ for (bool enable_page_index : {true, false}) {
+ SCOPED_TRACE(enable_page_index);
+ const std::string file_name =
+ dir_->Str() +
+ (enable_page_index ? "/nested_indexed.parquet" :
"/nested_no_index.parquet");
+ WriteTestFile(file_name, data, 1, 40, false, 40, enable_page_index);
+ for (size_t selection_index = 0; selection_index < selections.size();
++selection_index) {
+ SCOPED_TRACE(selection_index);
+ RoaringBitmap32 selection;
+ for (const auto& range : selections[selection_index].GetRanges()) {
+ selection.AddRange(range.from, range.to + 1);
+ }
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> input,
fs_->Open(file_name));
+ ASSERT_OK_AND_ASSIGN(int64_t length, input->Length());
+ auto stream = std::make_shared<ArrowInputStreamAdapter>(input,
length, arrow_pool_);
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<ParquetFileBatchReader> reader,
+ ParquetFileBatchReader::Create(
+ std::move(stream),
{{PARQUET_READ_ROW_RANGES_COALESCE_HOLE_SIZE_LIMIT, "0"}},
+ kBatchSize, nullptr, nullptr, arrow_pool_, std::nullopt));
+ ArrowSchema schema;
+
ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(data->type()->fields()),
&schema).ok());
+ ASSERT_OK(reader->SetReadSchema(&schema, nullptr, selection));
+ std::vector<uint32_t> actual_rows;
+ arrow::ArrayVector retained_batches;
+ while (true) {
+ ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch,
reader->NextBatch());
+ if (BatchReader::IsEofBatch(batch)) {
+ break;
+ }
+
ASSERT_OK(paimon::test::ReadResultCollector::CheckBatchOffset(batch));
+ auto result = arrow::ImportArray(batch.first.get(),
batch.second.get());
+ ASSERT_TRUE(result.ok()) << result.status().ToString();
+ std::shared_ptr<arrow::Array> array =
std::move(result).ValueOrDie();
+ ASSERT_TRUE(array->ValidateFull().ok());
+ ASSERT_GT(array->length(), 0);
+ ASSERT_LE(array->length(), kBatchSize);
+ for (int64_t row = 0; row < array->length(); ++row) {
+ ASSERT_OK_AND_ASSIGN(uint64_t file_row,
reader->GetPreviousBatchFileRowId(row));
+ ASSERT_TRUE(selection.Contains(file_row));
+ actual_rows.push_back(static_cast<uint32_t>(file_row));
+ }
+ retained_batches.push_back(std::move(array));
+ }
+ std::vector<uint32_t> expected_rows;
+ for (auto it = selection.Begin(); it != selection.End(); ++it) {
+ expected_rows.push_back(*it);
+ }
+ ASSERT_EQ(expected_rows, actual_rows);
+ ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch eof,
reader->NextBatch());
+ ASSERT_TRUE(BatchReader::IsEofBatch(eof));
+ reader->Close();
+ reader.reset();
+ size_t selected_row = 0;
+ for (const auto& array : retained_batches) {
+ ASSERT_TRUE(array->ValidateFull().ok());
+ for (int64_t row = 0; row < array->length(); ++row) {
+ const uint32_t file_row = expected_rows[selected_row++];
+ ASSERT_TRUE(data->Slice(file_row,
1)->Equals(array->Slice(row, 1)))
+ << "row content mismatch at file row " << file_row;
+ }
+ }
+ }
+ }
+}
- std::shared_ptr<arrow::ChunkedArray> result;
- ReadWithPredicateAndBitmapImpl(file_name, read_schema, nullptr, bitmap,
&result);
- ASSERT_TRUE(result);
+TEST_F(PageFilteredRowGroupReaderTest,
StreamingPreservesDictionaryValuesAcrossSelections) {
+ constexpr int32_t kRows = 60;
+ constexpr int64_t kBatchSize = 7;
+ arrow::StringBuilder strings;
+ for (int32_t row = 0; row < kRows; ++row) {
+ if (row % 5 == 0) {
+ ASSERT_TRUE(strings.AppendNull().ok());
+ } else {
+ ASSERT_TRUE(strings.Append(std::to_string(row % 4)).ok());
+ }
+ }
+ std::shared_ptr<arrow::Array> values;
+ ASSERT_TRUE(strings.Finish(&values).ok());
+ auto data = arrow::StructArray::Make(
+ {MakeSequentialIntData(kRows)->field(0), values},
+ {arrow::field("id", arrow::int32()), arrow::field("text",
arrow::utf8())})
+ .ValueOrDie();
+ const std::string file_name = dir_->Str() +
"/streaming_dictionary.parquet";
+ WriteTestFile(file_name, data, 10, kRows, true);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> input,
fs_->Open(file_name));
+ ASSERT_OK_AND_ASSIGN(int64_t length, input->Length());
+ auto stream = std::make_shared<ArrowInputStreamAdapter>(input, length,
arrow_pool_);
+ ::parquet::ArrowReaderProperties properties;
+ properties.set_read_dictionary(1, true);
+ ::parquet::arrow::FileReaderBuilder builder;
+ ASSERT_TRUE(builder.Open(stream).ok());
+ std::unique_ptr<::parquet::arrow::FileReader> file_reader;
+ ASSERT_TRUE(
+
builder.memory_pool(arrow_pool_.get())->properties(properties)->Build(&file_reader).ok());
+ auto indexes =
file_reader->parquet_reader()->GetPageIndexReader()->RowGroup(0);
+ RowRanges ranges({RowRanges::Range(2, 16), RowRanges::Range(32, 46),
RowRanges::Range(52, 58)});
+ TargetRowGroup target(0, true, ranges);
+ const auto page_ranges = PageFilteredRowGroupReader::ComputePageRanges(
+ target, {0, 1}, indexes, file_reader->parquet_reader());
+ for (int64_t batch_size : {-1, 0}) {
+ auto result = PageFilteredRowGroupReader::ReadFilteredRowGroup(
+ target, {0, 1}, arrow::io::CacheOptions::Defaults(), false,
page_ranges, batch_size,
+ indexes, arrow_pool_, file_reader.get());
+ ASSERT_FALSE(result.ok());
+ ASSERT_TRUE(result.status().IsInvalid());
+ }
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<arrow::RecordBatchReader> batches,
+ PageFilteredRowGroupReader::ReadFilteredRowGroup(
+ target, {0, 1},
arrow::io::CacheOptions::Defaults(), false,
+ page_ranges, kBatchSize, indexes, arrow_pool_,
file_reader.get()));
+ ASSERT_EQ(arrow::Type::DICTIONARY,
batches->schema()->field(1)->type()->id());
+ std::vector<std::shared_ptr<arrow::RecordBatch>> retained_batches;
+ std::vector<int32_t> actual_rows;
+ while (true) {
+ std::shared_ptr<arrow::RecordBatch> batch;
+ ASSERT_TRUE(batches->ReadNext(&batch).ok());
+ if (!batch) {
+ break;
+ }
+ ASSERT_TRUE(batch->ValidateFull().ok());
+ ASSERT_GT(batch->num_rows(), 0);
+ ASSERT_LE(batch->num_rows(), kBatchSize);
+ ASSERT_TRUE(batch->schema()->Equals(*batches->schema()));
+ ASSERT_EQ(arrow::Type::DICTIONARY, batch->column(1)->type_id());
+ const auto& ids = checked_cast<const
arrow::Int32Array&>(*batch->column(0));
+ for (int64_t row = 0; row < batch->num_rows(); ++row) {
+ actual_rows.push_back(ids.Value(row));
+ }
+ retained_batches.push_back(std::move(batch));
+ }
+ std::vector<int32_t> expected_rows;
+ for (const auto& range : ranges.GetRanges()) {
+ for (int64_t row = range.from; row <= range.to; ++row) {
+ expected_rows.push_back(static_cast<int32_t>(row));
+ }
+ }
+ ASSERT_EQ(expected_rows, actual_rows);
+ ASSERT_GT(retained_batches.size(), 1);
+ ASSERT_TRUE(batches->Close().ok());
+ batches.reset();
+ file_reader.reset();
+ for (const auto& batch : retained_batches) {
+ const auto& ids = checked_cast<const
arrow::Int32Array&>(*batch->column(0));
+ const auto& encoded = checked_cast<const
arrow::DictionaryArray&>(*batch->column(1));
+ const auto& dictionary = checked_cast<const
arrow::StringArray&>(*encoded.dictionary());
+ for (int64_t row = 0; row < batch->num_rows(); ++row) {
+ const int32_t file_row = ids.Value(row);
+ ASSERT_EQ(file_row % 5 == 0, encoded.IsNull(row));
+ if (!encoded.IsNull(row)) {
+ ASSERT_EQ(std::to_string(file_row % 4),
+ dictionary.GetString(encoded.GetValueIndex(row)));
+ }
+ }
+ }
+}
- std::vector<int64_t> keys;
+TEST_F(PageFilteredRowGroupReaderTest,
StreamingBoundsDecodedMemoryAndPreservesReturnedBatches) {
+ constexpr int32_t kRows = 16384;
+ constexpr int64_t kBatchSize = 64;
+ const std::string value(1024, 'x');
+ arrow::StringBuilder strings;
+ for (int32_t row = 0; row < kRows; ++row) {
+ ASSERT_TRUE(strings.Append(value).ok());
+ }
+ std::shared_ptr<arrow::Array> values;
+ ASSERT_TRUE(strings.Finish(&values).ok());
+ auto data = arrow::StructArray::Make(
+ {MakeSequentialIntData(kRows)->field(0), values},
+ {arrow::field("id", arrow::int32()), arrow::field("text",
arrow::utf8())})
+ .ValueOrDie();
+ const std::string file_name = dir_->Str() + "/streaming_memory.parquet";
+ WriteTestFile(file_name, data, 256, kRows, true);
+
+ const RowRanges ranges(RowRanges::Range(3, kRows - 5));
+ for (int64_t expected_rows : {int64_t{0}, 4 * kBatchSize,
ranges.RowCount()}) {
+ SCOPED_TRACE(expected_rows);
+ auto read_pool = GetArrowPool(pool_);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> input,
fs_->Open(file_name));
+ ASSERT_OK_AND_ASSIGN(int64_t length, input->Length());
+ auto stream = std::make_shared<ArrowInputStreamAdapter>(input, length,
read_pool);
+ std::unique_ptr<::parquet::arrow::FileReader> file_reader;
+ ASSERT_TRUE(::parquet::arrow::OpenFile(stream, read_pool.get(),
&file_reader).ok());
+ auto indexes =
file_reader->parquet_reader()->GetPageIndexReader()->RowGroup(0);
+ TargetRowGroup target(0, true, ranges);
+ const auto page_ranges = PageFilteredRowGroupReader::ComputePageRanges(
+ target, {0, 1}, indexes, file_reader->parquet_reader());
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<arrow::RecordBatchReader> batches,
+ PageFilteredRowGroupReader::ReadFilteredRowGroup(
+ target, {0, 1},
arrow::io::CacheOptions::Defaults(), false,
+ page_ranges, kBatchSize, indexes, read_pool,
file_reader.get()));
+
ASSERT_TRUE(batches->schema()->Equals(*arrow::schema(data->type()->fields())));
- auto concated = arrow::Concatenate(result->chunks()).ValueOrDie();
- auto top = std::dynamic_pointer_cast<arrow::StructArray>(concated);
- ASSERT_TRUE(top);
- auto key_arr = std::dynamic_pointer_cast<arrow::Int64Array>(top->field(0));
- ASSERT_TRUE(key_arr);
- for (int64_t i = 0; i < key_arr->length(); ++i) {
- ASSERT_FALSE(key_arr->IsNull(i)) << "key column must stay non-null";
- int64_t k = key_arr->Value(i);
- // Deep compare including nulls across struct/list/map leaves.
- ASSERT_TRUE(data->Slice(k, 1)->Equals(*top->Slice(i, 1)))
- << "row content mismatch at key " << k;
- keys.push_back(k);
+ std::shared_ptr<arrow::RecordBatch> first;
+ int64_t rows_read = 0;
+ while (rows_read < expected_rows) {
+ std::shared_ptr<arrow::RecordBatch> batch;
+ ASSERT_TRUE(batches->ReadNext(&batch).ok());
+ ASSERT_NE(nullptr, batch);
+ ASSERT_TRUE(batch->ValidateFull().ok());
+ ASSERT_EQ(std::min(kBatchSize, expected_rows - rows_read),
batch->num_rows());
+ const auto& ids = checked_cast<const
arrow::Int32Array&>(*batch->column(0));
+ const auto& text = checked_cast<const
arrow::StringArray&>(*batch->column(1));
+ for (int64_t row = 0; row < batch->num_rows(); ++row) {
+ ASSERT_EQ(3 + rows_read + row, ids.Value(row));
+ ASSERT_EQ(value, text.GetString(row));
+ }
+ if (!first) {
+ first = batch;
+ ASSERT_LT(read_pool->max_memory(), 4 * 1024 * 1024);
+ }
+ rows_read += batch->num_rows();
+ }
+ ASSERT_LT(read_pool->max_memory(), 4 * 1024 * 1024);
+ if (expected_rows == ranges.RowCount()) {
+ std::shared_ptr<arrow::RecordBatch> eof = first;
+ ASSERT_TRUE(batches->ReadNext(&eof).ok());
+ ASSERT_EQ(nullptr, eof);
+ const int64_t bytes_at_eof = read_pool->bytes_allocated();
+ ASSERT_TRUE(batches->ReadNext(&eof).ok());
+ ASSERT_EQ(nullptr, eof);
+ ASSERT_TRUE(batches->Close().ok());
+ ASSERT_EQ(bytes_at_eof, read_pool->bytes_allocated());
+ }
+ ASSERT_TRUE(batches->Close().ok());
+ const int64_t bytes_after_close = read_pool->bytes_allocated();
+ ASSERT_TRUE(batches->Close().ok());
+ std::shared_ptr<arrow::RecordBatch> after_close = first;
+ ASSERT_TRUE(batches->ReadNext(&after_close).ok());
+ ASSERT_EQ(nullptr, after_close);
+ batches.reset();
+ ASSERT_EQ(bytes_after_close, read_pool->bytes_allocated());
+ file_reader.reset();
+ indexes.reset();
+
+ if (first) {
+ const auto& ids = checked_cast<const
arrow::Int32Array&>(*first->column(0));
+ const auto& text = checked_cast<const
arrow::StringArray&>(*first->column(1));
+ for (int64_t row = 0; row < kBatchSize; ++row) {
+ ASSERT_EQ(3 + row, ids.Value(row));
+ ASSERT_EQ(value, text.GetString(row));
+ }
+ }
+ first.reset();
+ ASSERT_EQ(0, read_pool->bytes_allocated());
}
+}
- std::vector<int64_t> expected;
- for (int64_t i = 0; i < 15; ++i) {
- expected.push_back(i);
+TEST_F(PageFilteredRowGroupReaderTest, StreamingDefersLaterPageDecodeErrors) {
+ const std::string file_name = dir_->Str() +
"/streaming_corrupt_tail.parquet";
+ auto data = MakeTwoColumnData(100);
+ WriteTestFile(file_name, data, 10, 100);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> input,
fs_->Open(file_name));
+ ASSERT_OK_AND_ASSIGN(int64_t length, input->Length());
+ auto stream = std::make_shared<ArrowInputStreamAdapter>(input, length,
arrow_pool_);
+ std::unique_ptr<::parquet::arrow::FileReader> original;
+ ASSERT_TRUE(::parquet::arrow::OpenFile(stream, arrow_pool_.get(),
&original).ok());
+ auto index =
original->parquet_reader()->GetPageIndexReader()->RowGroup(0)->GetOffsetIndex(1);
+ ASSERT_NE(nullptr, index);
+ ASSERT_EQ(10, index->page_locations().size());
+ const auto last_page = index->page_locations().back();
+ ASSERT_EQ(90, last_page.first_row_index);
+ ASSERT_GE(last_page.offset, 0);
+ ASSERT_LE(last_page.offset + last_page.compressed_page_size, length);
+ std::string contents(length, '\0');
+ ASSERT_OK_AND_ASSIGN(int64_t read, input->Read(contents.data(), length,
0));
+ ASSERT_EQ(length, read);
+ std::fill(contents.begin() + last_page.offset,
+ contents.begin() + last_page.offset +
last_page.compressed_page_size,
+ static_cast<char>(0xff));
+ auto buffer = std::make_shared<arrow::Buffer>(reinterpret_cast<const
uint8_t*>(contents.data()),
+ contents.size());
+ auto corrupted = std::make_shared<arrow::io::BufferReader>(buffer);
+ std::unique_ptr<::parquet::arrow::FileReader> file_reader;
+ ASSERT_TRUE(::parquet::arrow::OpenFile(corrupted, arrow_pool_.get(),
&file_reader).ok());
+ auto indexes =
file_reader->parquet_reader()->GetPageIndexReader()->RowGroup(0);
+ TargetRowGroup target(0, true, RowRanges::CreateSingle(100));
+ const auto ranges = PageFilteredRowGroupReader::ComputePageRanges(
+ target, {0, 1}, indexes, file_reader->parquet_reader());
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<arrow::RecordBatchReader> batches,
+ PageFilteredRowGroupReader::ReadFilteredRowGroup(
+ target, {0, 1},
arrow::io::CacheOptions::Defaults(), false, ranges, 5,
+ indexes, arrow_pool_, file_reader.get()));
+ int64_t rows = 0;
+ arrow::Status status;
+ std::shared_ptr<arrow::RecordBatch> first;
+ for (int32_t i = 0; i < 20; ++i) {
+ std::shared_ptr<arrow::RecordBatch> batch = first;
+ status = batches->ReadNext(&batch);
+ if (!status.ok()) {
+ ASSERT_EQ(nullptr, batch);
+ break;
+ }
+ ASSERT_NE(nullptr, batch);
+ ASSERT_TRUE(batch->ValidateFull().ok());
+ ASSERT_EQ(5, batch->num_rows());
+ ASSERT_EQ(2, batch->num_columns());
+ for (int32_t column = 0; column < batch->num_columns(); ++column) {
+
ASSERT_TRUE(batch->column(column)->Equals(data->field(column)->Slice(rows, 5)));
+ }
+ if (!first) {
+ first = batch;
+ }
+ rows += batch->num_rows();
}
- for (int64_t i = 77; i < 87; ++i) {
- expected.push_back(i);
+ ASSERT_EQ(90, rows);
+ ASSERT_FALSE(status.ok());
+ std::shared_ptr<arrow::RecordBatch> batch = first;
+ ASSERT_EQ(status, batches->ReadNext(&batch));
+ ASSERT_EQ(nullptr, batch);
+ ASSERT_TRUE(batches->Close().ok());
+ batches.reset();
+ file_reader.reset();
+ ASSERT_NE(nullptr, first);
+ ASSERT_TRUE(first->ValidateFull().ok());
+ for (int32_t column = 0; column < first->num_columns(); ++column) {
+
ASSERT_TRUE(first->column(column)->Equals(data->field(column)->Slice(0, 5)));
}
Review Comment:
Added
[StreamingAlignsMisalignedChunksAcrossDecodingWindows](https://github.com/SteNicholas/paimon-cpp/blob/c34d4891e66c2a4694cefbddadd013233d9bb28d/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp#L997)
in c34d4891.
The test wraps real Parquet column decoders and splits two string columns
into chunks `[2, 2, 2, 1]` and `[3, 3, 1]` within each seven-row decoding
window. It asserts that `TableRecordBatchReader` returns aligned sub-batches
`[2, 1, 1, 2, 1]`, and checks the decoded-window counts after every read to
ensure the next window is decoded only after the current one is drained. This
repeats across two windows, followed by a final single-row window.
It also checks values and nulls across disjoint row selections, normalized
offsets, repeated EOF, and retained-batch validity after reader destruction.
The test-only chunk splitting keeps the fixture small while exercising the
production alignment path.
Validation: the Parquet test target builds, all 53
`PageFilteredRowGroupReaderTest.*` tests pass, and pre-commit checks plus `git
diff --check` pass.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]