zeroshade commented on code in PR #1640:
URL: https://github.com/apache/arrow-go/pull/1640#discussion_r4645133022


##########
parquet/pqarrow/file_reader_test.go:
##########
@@ -612,6 +612,90 @@ func TestRecordReaderSeekToCorruptPage(t *testing.T) {
        assert.ErrorContains(t, rr.Err(), "invalid checksum")
 }
 
+func TestRecordReaderSeekToRowRepeatedV1WithoutPageIndex(t *testing.T) {
+       mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+       defer mem.AssertSize(t, 0)
+
+       const nrows = 300
+       sb := array.NewListBuilder(mem, arrow.PrimitiveTypes.Int32)
+       defer sb.Release()
+       vb := sb.ValueBuilder().(*array.Int32Builder)
+       for i := 0; i < nrows; i++ {
+               sb.Append(true)
+               vb.AppendValues([]int32{int32(3 * i), int32(3*i + 1), int32(3*i 
+ 2)}, nil)
+       }
+       col := sb.NewListArray()
+       defer col.Release()
+
+       lstField := arrow.Field{Name: "lst", Type: 
arrow.ListOf(arrow.PrimitiveTypes.Int32), Nullable: true}
+       column := arrow.NewColumnFromArr(lstField, col)
+       defer column.Release()
+
+       tbl := array.NewTable(
+               arrow.NewSchema([]arrow.Field{lstField}, nil),
+               []arrow.Column{column}, nrows)
+       defer tbl.Release()
+
+       for _, pageVersion := range 
[]parquet.DataPageVersion{parquet.DataPageV1, parquet.DataPageV2} {
+               t.Run(fmt.Sprintf("page-v%d", int(pageVersion)+1), func(t 
*testing.T) {
+                       var buf bytes.Buffer
+                       wrProps := parquet.NewWriterProperties(
+                               parquet.WithDataPageVersion(pageVersion),
+                               parquet.WithDictionaryDefault(false),
+                               parquet.WithDataPageSize(256),
+                               parquet.WithBatchSize(16),
+                               parquet.WithPageIndexEnabled(false),
+                       )
+                       require.NoError(t, pqarrow.WriteTable(tbl, &buf, nrows, 
wrProps, pqarrow.NewArrowWriterProperties()))
+
+                       for _, seekRow := range []int64{0, 1, 150, 299} {
+                               t.Run(fmt.Sprintf("seek=%d", seekRow), func(t 
*testing.T) {
+                                       pf, err := 
file.NewParquetReader(bytes.NewReader(buf.Bytes()), 
file.WithReadProps(parquet.NewReaderProperties(mem)))
+                                       require.NoError(t, err)
+                                       defer pf.Close()
+
+                                       reader, err := 
pqarrow.NewFileReader(pf, pqarrow.ArrowReadProperties{BatchSize: 1024}, mem)
+                                       require.NoError(t, err)
+
+                                       rr, err := 
reader.GetRecordReader(context.Background(), nil, nil)
+                                       require.NoError(t, err)
+                                       defer rr.Release()
+
+                                       require.NoError(t, 
rr.SeekToRow(seekRow))
+
+                                       totalRows := 0
+                                       var firstRow []int32
+                                       for {
+                                               rec, err := rr.Read()
+                                               if errors.Is(err, io.EOF) {
+                                                       break
+                                               }
+                                               require.NoError(t, err)
+
+                                               lst := 
rec.Column(0).(*array.List)
+                                               for i := 0; i < lst.Len(); i++ {
+                                                       if totalRows == 0 {
+                                                               s, e := 
lst.ValueOffsets(0)
+                                                               v := 
lst.ListValues().(*array.Int32)
+                                                               for j := s; j < 
e; j++ {
+                                                                       
firstRow = append(firstRow, v.Value(int(j)))
+                                                               }
+                                                       }
+                                                       totalRows++
+                                               }
+                                               rec.Release()

Review Comment:
   `Read()` returns the reader's current batch, and the reader releases it on 
the next call, so this is a second release. CI runs with `-tags assert,test` 
(`ci/scripts/test.sh`), and with those tags this panics with `too many 
releases` in `page-v1/seek=0`. Without the `assert` tag it passes. Please 
remove this `Release`.



##########
parquet/file/page_reader.go:
##########
@@ -948,6 +949,35 @@ func (p *serializedPageReader) SeekToPageWithRow(rowIdx 
int64) error {
        return p.err
 }
 
+// countRowsV1 returns the number of records in a V1 data page by counting
+// the level-0 repetition levels in the page's level region, which the
+// DataPageV1 layout puts first. The levels stay encoded on the page, so this
+// decodes just the repetition run and leaves the buffer untouched.
+func (p *serializedPageReader) countRowsV1(levels []byte, repEnc 
format.Encoding, numValues int32) (int64, error) {
+       var (
+               dec   encoding.LevelDecoder
+               batch [4096]int16

Review Comment:
   This runs for every V1 page of a repeated column, on ordinary sequential 
reads as well as seeks. Because `batch[:...]` is passed through `Decode`, the 8 
KiB array is most likely heap-allocated on each call. Reading a 512K-row 
`list<int32>` V1 column showed no measurable time change, but allocations went 
up 5.3% (+18 per read). A scratch buffer kept on `serializedPageReader` would 
avoid that.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to