zeroshade commented on code in PR #1632:
URL: https://github.com/apache/arrow-go/pull/1632#discussion_r4635326959


##########
parquet/pqarrow/file_reader.go:
##########
@@ -802,8 +824,8 @@ func (c *columnIterator) FindChunkForRow(rowIdx int64) 
(file.PageReader, int64,
                return nil, 0, nil
        }
 
-       if rowIdx < 0 || rowIdx > c.rdr.NumRows() {
-               return nil, 0, fmt.Errorf("invalid row index %d, file only has 
%d rows", rowIdx, c.rdr.NumRows())
+       if rowIdx < 0 {
+               return nil, 0, fmt.Errorf("%w: negative row index %d", 
arrow.ErrInvalid, rowIdx)

Review Comment:
   The same applies here. The removed check compared against the file's row 
count instead of the requested row groups' total, so on main, 
`GetRecordReader(ctx, nil, []int{0, 0, 0, 1})` on an 8-row file rejects 
`SeekToRow(9)` with `invalid row index 9, file only has 8 rows`, even though 16 
rows were requested. With this change it works. Please mention this as a 
user-facing fix too.



##########
parquet/pqarrow/file_reader.go:
##########
@@ -856,11 +878,23 @@ type recordReader struct {
        cur          arrow.RecordBatch
        err          error
        mem          memory.Allocator
+       selection    *rowRangeSelection
 
        refCount atomic.Int64
 }
 
 func (r *recordReader) SeekToRow(row int64) error {
+       if r.selection != nil {
+               return r.seekToSelectedRow(row)
+       }
+       if err := r.seekToInputRow(row); err != nil {
+               return err
+       }
+       r.err = nil

Review Comment:
   This changes the existing `GetRecordReader` behaviour, and the change is a 
fix. On main, after reading to the end, `SeekToRow(0)` followed by `Next()` 
returns false, because the end-of-stream error stays set. With this change, 
reading resumes. Please list it under user-facing changes in the description.



##########
parquet/pqarrow/row_ranges.go:
##########
@@ -0,0 +1,164 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pqarrow
+
+import (
+       "context"
+       "fmt"
+       "io"
+
+       "github.com/apache/arrow-go/v18/arrow"
+)
+
+// RowRange is a half-open interval of rows, including Start and excluding End.
+type RowRange struct {
+       Start, End int64
+}
+
+// GetRecordReaderWithRowRanges reads only the requested row ranges from the
+// selected row groups and columns. Row offsets are relative to the requested
+// row groups in their supplied order. Ranges must be non-empty, sorted by
+// start, and non-overlapping.
+//
+// As with GetRecordReader, nil colIndices or rowGroups selects all columns or
+// row groups, respectively. Nil ranges selects all rows and an empty, non-nil
+// slice selects no rows. Batches may be shorter than the configured batch size
+// at range boundaries. For projections containing repeated columns, gaps are
+// read sequentially to preserve row alignment across page boundaries.
+func (fr *FileReader) GetRecordReaderWithRowRanges(ctx context.Context, 
colIndices, rowGroups []int, ranges []RowRange) (RecordReader, error) {
+       return fr.getRecordReader(ctx, colIndices, rowGroups, ranges)
+}
+
+type rowRangeSelection struct {
+       ranges             []RowRange
+       index              int
+       nextRow            int64
+       hasRepeatedColumns bool
+}
+
+func newRowRangeSelection(ranges []RowRange, numRows int64) 
(*rowRangeSelection, error) {
+       if ranges == nil {
+               return nil, nil
+       }
+       var merged []RowRange
+       for i, rows := range ranges {
+               if rows.Start < 0 || rows.End <= rows.Start || rows.End > 
numRows {
+                       return nil, fmt.Errorf("%w: row range %d [%d, %d) 
exceeds %d rows or is empty", arrow.ErrInvalid, i, rows.Start, rows.End, 
numRows)
+               }
+               if i > 0 {
+                       previous := &merged[len(merged)-1]
+                       if rows.Start < previous.End {
+                               return nil, fmt.Errorf("%w: row range %d 
overlaps or precedes a previous range", arrow.ErrInvalid, i)
+                       }
+                       if rows.Start == previous.End {
+                               previous.End = rows.End
+                               continue
+                       }
+               }
+               merged = append(merged, rows)
+       }
+       if len(merged) == 1 && merged[0] == (RowRange{0, numRows}) {
+               return nil, nil
+       }
+       return &rowRangeSelection{ranges: merged}, nil
+}
+
+func (r *recordReader) seekToSelectedRow(row int64) error {
+       if row < 0 {
+               return fmt.Errorf("%w: negative row index %d", arrow.ErrIndex, 
row)
+       }
+       remaining := row
+       for i, rows := range r.selection.ranges {
+               length := rows.End - rows.Start
+               if remaining < length {
+                       inputRow := rows.Start + remaining
+                       if err := r.seekToRangeRow(inputRow); err != nil {
+                               r.err = err
+                               return err
+                       }
+                       r.selection.index = i
+                       r.selection.nextRow = inputRow
+                       r.err = nil
+                       return nil
+               }
+               remaining -= length
+       }
+       return fmt.Errorf("%w: row index %d exceeds %d selected rows", 
arrow.ErrIndex, row, row-remaining)
+}
+
+func (r *recordReader) seekToRangeRow(row int64) error {
+       if !r.selection.hasRepeatedColumns {
+               return r.seekToInputRow(row)
+       }
+       if err := r.seekToInputRow(0); err != nil {
+               return err
+       }
+       return r.readThroughGap(row)
+}
+
+// Arbitrary seeks into repeated Data Page V1 columns without offset indexes
+// can lose row alignment (GH-1522). Read through gaps in repeated projections.
+func (r *recordReader) readThroughGap(rows int64) error {

Review Comment:
   This works, but it costs a lot. Every row in a gap is decoded into Arrow 
batches and thrown away, and `seekToRangeRow` restarts from row 0 on every 
seek, so projections with a repeated column get almost none of the benefit. It 
also applies when #1522 doesn't, for example to V2 pages or columns with an 
offset index.
   
   #1601 has already fixed part of #1522. #1608 fixes the rest, including seeks 
on V1 repeated columns without an offset index. Once it merges, these seeks 
return the right rows. Could you link #1608 in this comment and remove the 
workaround once it lands, either here after a rebase or in a follow-up? Until 
then, keeping it is the right choice, since main still returns wrong rows for 
those seeks.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to