alamb commented on code in PR #11334:
URL: https://github.com/apache/arrow-rs/pull/11334#discussion_r4233216625
##########
parquet/src/arrow/push_decoder/scan_plan/mod.rs:
##########
@@ -506,58 +493,75 @@ struct RowGroupContext {
expanded_selection: Option<RowSelection>,
/// What each decoding stage fetches.
stages: Arc<StageSchedule>,
+ /// The file metadata, for the byte ranges of the column chunks.
+ metadata: Arc<ParquetMetaData>,
/// The file's page index, if it has offset indexes.
page_index: Option<Arc<dyn PageIndexProvider>>,
}
/// The merge state of one stage.
+///
+/// A cursor of a column chunk is one heap entry: the sort key of its next
+/// range, the column and the position of that range. The other parts of the
+/// range are computed again when the entry is popped. Thus a stage stores
+/// no per-column state other than its heap entry, unless the selection
+/// removes pages.
#[derive(Debug, Clone)]
struct StageRanges {
stage: ScanStage,
conditional: bool,
- cursors: Vec<ColumnCursor>,
- /// The sort key of the next range of each cursor that has one, smallest
- /// first.
- heap: BinaryHeap<Reverse<(RangeOrder, usize)>>,
+ /// Leaf column indexes, in file order. A [`HeapEntry`] refers to a
+ /// column by its position in this list.
+ columns: Vec<usize>,
+ /// For each column in `columns`, the indexes of the data pages that the
+ /// decoder reads, if the selection removes pages. `None` if no column of
+ /// the stage has such a list.
+ selected: Option<Vec<Option<Vec<usize>>>>,
+ /// The next range of each cursor that has one, smallest first.
+ heap: BinaryHeap<Reverse<HeapEntry>>,
}
/// How the heap orders the next ranges of different column chunks: by first
/// row, then more rows first (a dictionary page), then by file offset. The
/// ranges of one column chunk keep file order, also a page with no planned
/// rows that the predicate cache reads.
+///
+/// The key is `(first row, Reverse(last row), range start)`.
type RangeOrder = (u64, Reverse<u64>, u64);
-fn range_order(range: &PlannedRange) -> RangeOrder {
- (range.first_row, Reverse(range.last_row), range.range.start)
-}
+/// A heap entry: the sort key of the next range of a cursor, the position
+/// of the column in [`StageRanges::columns`], and the [`HeadPosition`] of
+/// that range. The column position breaks ties, so ranges with equal keys
+/// stay in file order of their columns.
+type HeapEntry = (RangeOrder, u32, HeadPosition);
-/// The ranges of one column chunk that are not yet returned.
-#[derive(Debug, Clone)]
-struct ColumnCursor {
- column_idx: usize,
- /// The next range of this column chunk.
- head: Option<PlannedRange>,
- /// Data pages the decoder reads.
- pages: PageSet,
- /// Position in `pages` of the next data page after `head`.
- next_page: usize,
+/// Which range of a column chunk is next: [`CHUNK`], [`DICTIONARY`], or the
+/// position of a data page in the pages that the decoder reads.
+type HeadPosition = u32;
+
+/// The next range is the whole column chunk. The column has no offset index.
+const CHUNK: HeadPosition = u32::MAX;
+
+/// The next range is the dictionary page.
+const DICTIONARY: HeadPosition = u32::MAX - 1;
+
+/// Converts a column or page position to a heap entry field.
+fn position(position: usize) -> u32 {
+ u32::try_from(position).expect("fewer than u32::MAX - 1 columns and pages")
}
/// The data pages of a column chunk that the decoder reads, in page order.
-#[derive(Debug, Clone)]
-enum PageSet {
- /// No offset index, so the column chunk is one range.
- None,
- /// Every page: there is no selection.
+#[derive(Debug, Clone, Copy)]
+enum PageSet<'a> {
+ /// Every page of the offset index, which has this many pages.
All(usize),
/// The pages at these indexes in the offset index.
- Selected(Vec<usize>),
+ Selected(&'a [usize]),
Review Comment:
nice
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
