SteNicholas commented on code in PR #513:
URL: https://github.com/apache/paimon-cpp/pull/513#discussion_r4495093451
##########
src/paimon/format/orc/orc_file_batch_reader.cpp:
##########
@@ -203,18 +213,41 @@ Status OrcFileBatchReader::SetReadSchema(::ArrowSchema*
read_schema,
target_column_ids_ = target_column_ids;
previous_batch_row_count_ = 0;
PAIMON_RETURN_NOT_OK(reader_->SetReadSchema(target_type,
row_reader_options));
+ pre_buffered_row_range_.reset();
return Status::OK();
}
Status OrcFileBatchReader::SeekToRow(uint64_t row_number) {
+ // ORC may read payload while seeking within a stripe. Warm the target
before that I/O.
+ PAIMON_RETURN_NOT_OK(PrepareRangesForRow(row_number));
return reader_->SeekToRow(row_number);
}
Result<std::vector<std::pair<uint64_t, uint64_t>>>
OrcFileBatchReader::PreBufferRange() {
- return reader_->PreBufferRange(target_column_ids_);
+ using ByteRanges = std::vector<std::pair<uint64_t, uint64_t>>;
+ std::pair<uint64_t, uint64_t> row_range;
+ PAIMON_ASSIGN_OR_RAISE(
+ ByteRanges ranges,
+ reader_->PreBufferRange(target_column_ids_,
+
range_preparation_row_.value_or(reader_->GetNextRowToRead()),
+ &row_range));
+ pre_buffered_row_range_ = row_range;
+ return ranges;
+}
+
+Status OrcFileBatchReader::PrepareRangesForRow(uint64_t row_number) {
+ if (read_ahead_ && (!pre_buffered_row_range_ || row_number <
pre_buffered_row_range_->first ||
+ row_number >= pre_buffered_row_range_->second)) {
+ range_preparation_row_ = row_number;
+ Status status = read_ahead_->PrepareRanges(this);
+ range_preparation_row_.reset();
+ return status;
+ }
+ return Status::OK();
}
Result<BatchReader::ReadBatch> OrcFileBatchReader::NextBatch() {
+ PAIMON_RETURN_NOT_OK(PrepareRangesForRow(reader_->GetNextRowToRead()));
Review Comment:
**Predicate-driven stripe skipping can read the target payload twice.**
`GetNextRowToRead()` still denotes the pre-filter position here, but ORC's
`startNextStripe()` can advance to a matching stripe inside `reader_->Next()`
without calling our `SeekToRow()`. In a three-stripe file containing rows 0–11,
with `a >= 8`, batch size 2 and the block cache disabled, I reproduced the
first `NextBatch()` reading all 7 bytes of stripe 2's payload directly (zero
payload bytes from the range cache). The second `NextBatch()` then fetched
those same 7 bytes through `PrepareRangesForRow(10)`, after ORC had already
consumed them. The explicit-seek fix does not cover this path. For larger
stripes this adds redundant storage I/O and misses preparation before the first
matching batch. Please coordinate preparation with the stripe actually selected
by ORC before its payload read, or otherwise prevent fetching already-consumed
payload again. Extend the predicate portion of `DiscoverStripeRangesOnDe
mand` to assert physical reads/cache hits as well as returned rows.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]