lxy-9602 commented on code in PR #563:
URL: https://github.com/apache/paimon-cpp/pull/563#discussion_r4598910145


##########
src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp:
##########
@@ -2637,63 +2628,332 @@ static std::shared_ptr<arrow::StructArray> 
MakeMisalignedNestedDataWithNulls(int
         .ValueOrDie();
 }
 
-/// Test: nested columns containing NULLs, with MISALIGNED leaf pages, split 
across
-/// MULTIPLE row groups. This combines the three stress dimensions: null-driven
-/// definition levels, byte-based misaligned pages, and row-group boundaries 
that the
-/// selection bitmap crosses. Every returned row is deep-compared (nulls 
included)
-/// against the original data identified by its non-null key.
-/// Bitmap: [0,15), [77, 87) (to avoid bitmap hole filling)
-/// Expected: 25 rows -> keys 0..14 and 77..86
 TEST_F(PageFilteredRowGroupReaderTest, 
NestedColumnsWithNullsMisalignedPagesMultiRowGroup) {
-    std::string file_name = dir_->Str() + 
"/nested_nulls_misaligned_multi_rg.parquet";
-    constexpr int32_t kNumRows = 100;
-    auto data = MakeMisalignedNestedDataWithNulls(kNumRows);
-
-    // write_batch_size=1 + byte-based data page size -> misaligned leaf pages.
-    // max_row_group_length=40 -> 3 row groups (40, 40, 20).
-    WriteTestFile(file_name, data, /*write_batch_size=*/1, 
/*max_row_group_length=*/40,
-                  /*enable_dictionary=*/false, /*data_page_size=*/40);
-
-    auto read_schema =
-        arrow::schema({arrow::field("key", arrow::int64()),
-                       arrow::field("s", arrow::struct_({arrow::field("x", 
arrow::int32()),
-                                                         arrow::field("y", 
arrow::int64())})),
-                       arrow::field("tags", arrow::list(arrow::field("item", 
arrow::int32()))),
-                       arrow::field("props", arrow::map(arrow::utf8(), 
arrow::int32()))});
-
-    // bitmap: [0,15), [77, 87) -> spans all three row groups.
-    RoaringBitmap32 bitmap;
-    bitmap.AddRange(0, 15);
-    bitmap.AddRange(77, 87);
+    constexpr int32_t kRows = 100;
+    constexpr int32_t kBatchSize = 7;
+    auto data = MakeMisalignedNestedDataWithNulls(kRows);
+    const std::vector<RowRanges> selections = {
+        RowRanges({RowRanges::Range(0, 14), RowRanges::Range(77, 86)}),
+        RowRanges({RowRanges::Range(2, 15), RowRanges::Range(43, 68), 
RowRanges::Range(83, 96)}),
+        RowRanges({RowRanges::Range(2, 15), RowRanges::Range(40, 79), 
RowRanges::Range(83, 96)})};
+    for (bool enable_page_index : {true, false}) {
+        SCOPED_TRACE(enable_page_index);
+        const std::string file_name =
+            dir_->Str() +
+            (enable_page_index ? "/nested_indexed.parquet" : 
"/nested_no_index.parquet");
+        WriteTestFile(file_name, data, 1, 40, false, 40, enable_page_index);
+        for (size_t selection_index = 0; selection_index < selections.size(); 
++selection_index) {
+            SCOPED_TRACE(selection_index);
+            RoaringBitmap32 selection;
+            for (const auto& range : selections[selection_index].GetRanges()) {
+                selection.AddRange(range.from, range.to + 1);
+            }
+            ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> input, 
fs_->Open(file_name));
+            ASSERT_OK_AND_ASSIGN(int64_t length, input->Length());
+            auto stream = std::make_shared<ArrowInputStreamAdapter>(input, 
length, arrow_pool_);
+            ASSERT_OK_AND_ASSIGN(
+                std::unique_ptr<ParquetFileBatchReader> reader,
+                ParquetFileBatchReader::Create(
+                    std::move(stream), 
{{PARQUET_READ_ROW_RANGES_COALESCE_HOLE_SIZE_LIMIT, "0"}},
+                    kBatchSize, nullptr, nullptr, arrow_pool_, std::nullopt));
+            ArrowSchema schema;
+            
ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(data->type()->fields()), 
&schema).ok());
+            ASSERT_OK(reader->SetReadSchema(&schema, nullptr, selection));
+            std::vector<uint32_t> actual_rows;
+            arrow::ArrayVector retained_batches;
+            while (true) {
+                ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, 
reader->NextBatch());
+                if (BatchReader::IsEofBatch(batch)) {
+                    break;
+                }
+                
ASSERT_OK(paimon::test::ReadResultCollector::CheckBatchOffset(batch));
+                auto result = arrow::ImportArray(batch.first.get(), 
batch.second.get());
+                ASSERT_TRUE(result.ok()) << result.status().ToString();
+                std::shared_ptr<arrow::Array> array = 
std::move(result).ValueOrDie();
+                ASSERT_TRUE(array->ValidateFull().ok());
+                ASSERT_GT(array->length(), 0);
+                ASSERT_LE(array->length(), kBatchSize);
+                for (int64_t row = 0; row < array->length(); ++row) {
+                    ASSERT_OK_AND_ASSIGN(uint64_t file_row, 
reader->GetPreviousBatchFileRowId(row));
+                    ASSERT_TRUE(selection.Contains(file_row));
+                    actual_rows.push_back(static_cast<uint32_t>(file_row));
+                }
+                retained_batches.push_back(std::move(array));
+            }
+            std::vector<uint32_t> expected_rows;
+            for (auto it = selection.Begin(); it != selection.End(); ++it) {
+                expected_rows.push_back(*it);
+            }
+            ASSERT_EQ(expected_rows, actual_rows);
+            ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch eof, 
reader->NextBatch());
+            ASSERT_TRUE(BatchReader::IsEofBatch(eof));
+            reader->Close();
+            reader.reset();
+            size_t selected_row = 0;
+            for (const auto& array : retained_batches) {
+                ASSERT_TRUE(array->ValidateFull().ok());
+                for (int64_t row = 0; row < array->length(); ++row) {
+                    const uint32_t file_row = expected_rows[selected_row++];
+                    ASSERT_TRUE(data->Slice(file_row, 
1)->Equals(array->Slice(row, 1)))
+                        << "row content mismatch at file row " << file_row;
+                }
+            }
+        }
+    }
+}
 
-    std::shared_ptr<arrow::ChunkedArray> result;
-    ReadWithPredicateAndBitmapImpl(file_name, read_schema, nullptr, bitmap, 
&result);
-    ASSERT_TRUE(result);
+TEST_F(PageFilteredRowGroupReaderTest, 
StreamingPreservesDictionaryValuesAcrossSelections) {
+    constexpr int32_t kRows = 60;
+    constexpr int64_t kBatchSize = 7;
+    arrow::StringBuilder strings;
+    for (int32_t row = 0; row < kRows; ++row) {
+        if (row % 5 == 0) {
+            ASSERT_TRUE(strings.AppendNull().ok());
+        } else {
+            ASSERT_TRUE(strings.Append(std::to_string(row % 4)).ok());
+        }
+    }
+    std::shared_ptr<arrow::Array> values;
+    ASSERT_TRUE(strings.Finish(&values).ok());
+    auto data = arrow::StructArray::Make(
+                    {MakeSequentialIntData(kRows)->field(0), values},
+                    {arrow::field("id", arrow::int32()), arrow::field("text", 
arrow::utf8())})
+                    .ValueOrDie();
+    const std::string file_name = dir_->Str() + 
"/streaming_dictionary.parquet";
+    WriteTestFile(file_name, data, 10, kRows, true);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> input, 
fs_->Open(file_name));
+    ASSERT_OK_AND_ASSIGN(int64_t length, input->Length());
+    auto stream = std::make_shared<ArrowInputStreamAdapter>(input, length, 
arrow_pool_);
+    ::parquet::ArrowReaderProperties properties;
+    properties.set_read_dictionary(1, true);
+    ::parquet::arrow::FileReaderBuilder builder;
+    ASSERT_TRUE(builder.Open(stream).ok());
+    std::unique_ptr<::parquet::arrow::FileReader> file_reader;
+    ASSERT_TRUE(
+        
builder.memory_pool(arrow_pool_.get())->properties(properties)->Build(&file_reader).ok());
+    auto indexes = 
file_reader->parquet_reader()->GetPageIndexReader()->RowGroup(0);
+    RowRanges ranges({RowRanges::Range(2, 16), RowRanges::Range(32, 46), 
RowRanges::Range(52, 58)});
+    TargetRowGroup target(0, true, ranges);
+    const auto page_ranges = PageFilteredRowGroupReader::ComputePageRanges(
+        target, {0, 1}, indexes, file_reader->parquet_reader());
+    for (int64_t batch_size : {-1, 0}) {
+        auto result = PageFilteredRowGroupReader::ReadFilteredRowGroup(
+            target, {0, 1}, arrow::io::CacheOptions::Defaults(), false, 
page_ranges, batch_size,
+            indexes, arrow_pool_, file_reader.get());
+        ASSERT_FALSE(result.ok());
+        ASSERT_TRUE(result.status().IsInvalid());
+    }
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<arrow::RecordBatchReader> batches,
+                         PageFilteredRowGroupReader::ReadFilteredRowGroup(
+                             target, {0, 1}, 
arrow::io::CacheOptions::Defaults(), false,
+                             page_ranges, kBatchSize, indexes, arrow_pool_, 
file_reader.get()));
+    ASSERT_EQ(arrow::Type::DICTIONARY, 
batches->schema()->field(1)->type()->id());
+    std::vector<std::shared_ptr<arrow::RecordBatch>> retained_batches;
+    std::vector<int32_t> actual_rows;
+    while (true) {
+        std::shared_ptr<arrow::RecordBatch> batch;
+        ASSERT_TRUE(batches->ReadNext(&batch).ok());
+        if (!batch) {
+            break;
+        }
+        ASSERT_TRUE(batch->ValidateFull().ok());
+        ASSERT_GT(batch->num_rows(), 0);
+        ASSERT_LE(batch->num_rows(), kBatchSize);
+        ASSERT_TRUE(batch->schema()->Equals(*batches->schema()));
+        ASSERT_EQ(arrow::Type::DICTIONARY, batch->column(1)->type_id());
+        const auto& ids = checked_cast<const 
arrow::Int32Array&>(*batch->column(0));
+        for (int64_t row = 0; row < batch->num_rows(); ++row) {
+            actual_rows.push_back(ids.Value(row));
+        }
+        retained_batches.push_back(std::move(batch));
+    }
+    std::vector<int32_t> expected_rows;
+    for (const auto& range : ranges.GetRanges()) {
+        for (int64_t row = range.from; row <= range.to; ++row) {
+            expected_rows.push_back(static_cast<int32_t>(row));
+        }
+    }
+    ASSERT_EQ(expected_rows, actual_rows);
+    ASSERT_GT(retained_batches.size(), 1);
+    ASSERT_TRUE(batches->Close().ok());
+    batches.reset();
+    file_reader.reset();
+    for (const auto& batch : retained_batches) {
+        const auto& ids = checked_cast<const 
arrow::Int32Array&>(*batch->column(0));
+        const auto& encoded = checked_cast<const 
arrow::DictionaryArray&>(*batch->column(1));
+        const auto& dictionary = checked_cast<const 
arrow::StringArray&>(*encoded.dictionary());
+        for (int64_t row = 0; row < batch->num_rows(); ++row) {
+            const int32_t file_row = ids.Value(row);
+            ASSERT_EQ(file_row % 5 == 0, encoded.IsNull(row));
+            if (!encoded.IsNull(row)) {
+                ASSERT_EQ(std::to_string(file_row % 4),
+                          dictionary.GetString(encoded.GetValueIndex(row)));
+            }
+        }
+    }
+}
 
-    std::vector<int64_t> keys;
+TEST_F(PageFilteredRowGroupReaderTest, 
StreamingBoundsDecodedMemoryAndPreservesReturnedBatches) {
+    constexpr int32_t kRows = 16384;
+    constexpr int64_t kBatchSize = 64;
+    const std::string value(1024, 'x');
+    arrow::StringBuilder strings;
+    for (int32_t row = 0; row < kRows; ++row) {
+        ASSERT_TRUE(strings.Append(value).ok());
+    }
+    std::shared_ptr<arrow::Array> values;
+    ASSERT_TRUE(strings.Finish(&values).ok());
+    auto data = arrow::StructArray::Make(
+                    {MakeSequentialIntData(kRows)->field(0), values},
+                    {arrow::field("id", arrow::int32()), arrow::field("text", 
arrow::utf8())})
+                    .ValueOrDie();
+    const std::string file_name = dir_->Str() + "/streaming_memory.parquet";
+    WriteTestFile(file_name, data, 256, kRows, true);
+
+    const RowRanges ranges(RowRanges::Range(3, kRows - 5));
+    for (int64_t expected_rows : {int64_t{0}, 4 * kBatchSize, 
ranges.RowCount()}) {
+        SCOPED_TRACE(expected_rows);
+        auto read_pool = GetArrowPool(pool_);
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> input, 
fs_->Open(file_name));
+        ASSERT_OK_AND_ASSIGN(int64_t length, input->Length());
+        auto stream = std::make_shared<ArrowInputStreamAdapter>(input, length, 
read_pool);
+        std::unique_ptr<::parquet::arrow::FileReader> file_reader;
+        ASSERT_TRUE(::parquet::arrow::OpenFile(stream, read_pool.get(), 
&file_reader).ok());
+        auto indexes = 
file_reader->parquet_reader()->GetPageIndexReader()->RowGroup(0);
+        TargetRowGroup target(0, true, ranges);
+        const auto page_ranges = PageFilteredRowGroupReader::ComputePageRanges(
+            target, {0, 1}, indexes, file_reader->parquet_reader());
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<arrow::RecordBatchReader> batches,
+                             PageFilteredRowGroupReader::ReadFilteredRowGroup(
+                                 target, {0, 1}, 
arrow::io::CacheOptions::Defaults(), false,
+                                 page_ranges, kBatchSize, indexes, read_pool, 
file_reader.get()));
+        
ASSERT_TRUE(batches->schema()->Equals(*arrow::schema(data->type()->fields())));
 
-    auto concated = arrow::Concatenate(result->chunks()).ValueOrDie();
-    auto top = std::dynamic_pointer_cast<arrow::StructArray>(concated);
-    ASSERT_TRUE(top);
-    auto key_arr = std::dynamic_pointer_cast<arrow::Int64Array>(top->field(0));
-    ASSERT_TRUE(key_arr);
-    for (int64_t i = 0; i < key_arr->length(); ++i) {
-        ASSERT_FALSE(key_arr->IsNull(i)) << "key column must stay non-null";
-        int64_t k = key_arr->Value(i);
-        // Deep compare including nulls across struct/list/map leaves.
-        ASSERT_TRUE(data->Slice(k, 1)->Equals(*top->Slice(i, 1)))
-            << "row content mismatch at key " << k;
-        keys.push_back(k);
+        std::shared_ptr<arrow::RecordBatch> first;
+        int64_t rows_read = 0;
+        while (rows_read < expected_rows) {
+            std::shared_ptr<arrow::RecordBatch> batch;
+            ASSERT_TRUE(batches->ReadNext(&batch).ok());
+            ASSERT_NE(nullptr, batch);
+            ASSERT_TRUE(batch->ValidateFull().ok());
+            ASSERT_EQ(std::min(kBatchSize, expected_rows - rows_read), 
batch->num_rows());
+            const auto& ids = checked_cast<const 
arrow::Int32Array&>(*batch->column(0));
+            const auto& text = checked_cast<const 
arrow::StringArray&>(*batch->column(1));
+            for (int64_t row = 0; row < batch->num_rows(); ++row) {
+                ASSERT_EQ(3 + rows_read + row, ids.Value(row));
+                ASSERT_EQ(value, text.GetString(row));
+            }
+            if (!first) {
+                first = batch;
+                ASSERT_LT(read_pool->max_memory(), 4 * 1024 * 1024);
+            }
+            rows_read += batch->num_rows();
+        }
+        ASSERT_LT(read_pool->max_memory(), 4 * 1024 * 1024);
+        if (expected_rows == ranges.RowCount()) {
+            std::shared_ptr<arrow::RecordBatch> eof = first;
+            ASSERT_TRUE(batches->ReadNext(&eof).ok());
+            ASSERT_EQ(nullptr, eof);
+            const int64_t bytes_at_eof = read_pool->bytes_allocated();
+            ASSERT_TRUE(batches->ReadNext(&eof).ok());
+            ASSERT_EQ(nullptr, eof);
+            ASSERT_TRUE(batches->Close().ok());
+            ASSERT_EQ(bytes_at_eof, read_pool->bytes_allocated());
+        }
+        ASSERT_TRUE(batches->Close().ok());
+        const int64_t bytes_after_close = read_pool->bytes_allocated();
+        ASSERT_TRUE(batches->Close().ok());
+        std::shared_ptr<arrow::RecordBatch> after_close = first;
+        ASSERT_TRUE(batches->ReadNext(&after_close).ok());
+        ASSERT_EQ(nullptr, after_close);
+        batches.reset();
+        ASSERT_EQ(bytes_after_close, read_pool->bytes_allocated());
+        file_reader.reset();
+        indexes.reset();
+
+        if (first) {
+            const auto& ids = checked_cast<const 
arrow::Int32Array&>(*first->column(0));
+            const auto& text = checked_cast<const 
arrow::StringArray&>(*first->column(1));
+            for (int64_t row = 0; row < kBatchSize; ++row) {
+                ASSERT_EQ(3 + row, ids.Value(row));
+                ASSERT_EQ(value, text.GetString(row));
+            }
+        }
+        first.reset();
+        ASSERT_EQ(0, read_pool->bytes_allocated());
     }
+}
 
-    std::vector<int64_t> expected;
-    for (int64_t i = 0; i < 15; ++i) {
-        expected.push_back(i);
+TEST_F(PageFilteredRowGroupReaderTest, StreamingDefersLaterPageDecodeErrors) {
+    const std::string file_name = dir_->Str() + 
"/streaming_corrupt_tail.parquet";
+    auto data = MakeTwoColumnData(100);
+    WriteTestFile(file_name, data, 10, 100);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> input, 
fs_->Open(file_name));
+    ASSERT_OK_AND_ASSIGN(int64_t length, input->Length());
+    auto stream = std::make_shared<ArrowInputStreamAdapter>(input, length, 
arrow_pool_);
+    std::unique_ptr<::parquet::arrow::FileReader> original;
+    ASSERT_TRUE(::parquet::arrow::OpenFile(stream, arrow_pool_.get(), 
&original).ok());
+    auto index = 
original->parquet_reader()->GetPageIndexReader()->RowGroup(0)->GetOffsetIndex(1);
+    ASSERT_NE(nullptr, index);
+    ASSERT_EQ(10, index->page_locations().size());
+    const auto last_page = index->page_locations().back();
+    ASSERT_EQ(90, last_page.first_row_index);
+    ASSERT_GE(last_page.offset, 0);
+    ASSERT_LE(last_page.offset + last_page.compressed_page_size, length);
+    std::string contents(length, '\0');
+    ASSERT_OK_AND_ASSIGN(int64_t read, input->Read(contents.data(), length, 
0));
+    ASSERT_EQ(length, read);
+    std::fill(contents.begin() + last_page.offset,
+              contents.begin() + last_page.offset + 
last_page.compressed_page_size,
+              static_cast<char>(0xff));
+    auto buffer = std::make_shared<arrow::Buffer>(reinterpret_cast<const 
uint8_t*>(contents.data()),
+                                                  contents.size());
+    auto corrupted = std::make_shared<arrow::io::BufferReader>(buffer);
+    std::unique_ptr<::parquet::arrow::FileReader> file_reader;
+    ASSERT_TRUE(::parquet::arrow::OpenFile(corrupted, arrow_pool_.get(), 
&file_reader).ok());
+    auto indexes = 
file_reader->parquet_reader()->GetPageIndexReader()->RowGroup(0);
+    TargetRowGroup target(0, true, RowRanges::CreateSingle(100));
+    const auto ranges = PageFilteredRowGroupReader::ComputePageRanges(
+        target, {0, 1}, indexes, file_reader->parquet_reader());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<arrow::RecordBatchReader> batches,
+                         PageFilteredRowGroupReader::ReadFilteredRowGroup(
+                             target, {0, 1}, 
arrow::io::CacheOptions::Defaults(), false, ranges, 5,
+                             indexes, arrow_pool_, file_reader.get()));
+    int64_t rows = 0;
+    arrow::Status status;
+    std::shared_ptr<arrow::RecordBatch> first;
+    for (int32_t i = 0; i < 20; ++i) {
+        std::shared_ptr<arrow::RecordBatch> batch = first;
+        status = batches->ReadNext(&batch);
+        if (!status.ok()) {
+            ASSERT_EQ(nullptr, batch);
+            break;
+        }
+        ASSERT_NE(nullptr, batch);
+        ASSERT_TRUE(batch->ValidateFull().ok());
+        ASSERT_EQ(5, batch->num_rows());
+        ASSERT_EQ(2, batch->num_columns());
+        for (int32_t column = 0; column < batch->num_columns(); ++column) {
+            
ASSERT_TRUE(batch->column(column)->Equals(data->field(column)->Slice(rows, 5)));
+        }
+        if (!first) {
+            first = batch;
+        }
+        rows += batch->num_rows();
     }
-    for (int64_t i = 77; i < 87; ++i) {
-        expected.push_back(i);
+    ASSERT_EQ(90, rows);
+    ASSERT_FALSE(status.ok());
+    std::shared_ptr<arrow::RecordBatch> batch = first;
+    ASSERT_EQ(status, batches->ReadNext(&batch));
+    ASSERT_EQ(nullptr, batch);
+    ASSERT_TRUE(batches->Close().ok());
+    batches.reset();
+    file_reader.reset();
+    ASSERT_NE(nullptr, first);
+    ASSERT_TRUE(first->ValidateFull().ok());
+    for (int32_t column = 0; column < first->num_columns(); ++column) {
+        
ASSERT_TRUE(first->column(column)->Equals(data->field(column)->Slice(0, 5)));
     }

Review Comment:
   Thanks for adding streaming decoding for page-filtered reads!
   
   Is there a test that exercises a decoding window where columns produce 
multiple chunks with different boundaries, so that `TableRecordBatchReader` 
aligns and returns multiple sub-batches, then the reader advances to the next 
decoding window?
   
   If this is already covered, could you point me to the specific test? 
Otherwise, could you please add a focused regression test for this path? Thanks!



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to