zeroshade commented on code in PR #1640:
URL: https://github.com/apache/arrow-go/pull/1640#discussion_r4645133022
##########
parquet/pqarrow/file_reader_test.go:
##########
@@ -612,6 +612,90 @@ func TestRecordReaderSeekToCorruptPage(t *testing.T) {
assert.ErrorContains(t, rr.Err(), "invalid checksum")
}
+func TestRecordReaderSeekToRowRepeatedV1WithoutPageIndex(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ const nrows = 300
+ sb := array.NewListBuilder(mem, arrow.PrimitiveTypes.Int32)
+ defer sb.Release()
+ vb := sb.ValueBuilder().(*array.Int32Builder)
+ for i := 0; i < nrows; i++ {
+ sb.Append(true)
+ vb.AppendValues([]int32{int32(3 * i), int32(3*i + 1), int32(3*i
+ 2)}, nil)
+ }
+ col := sb.NewListArray()
+ defer col.Release()
+
+ lstField := arrow.Field{Name: "lst", Type:
arrow.ListOf(arrow.PrimitiveTypes.Int32), Nullable: true}
+ column := arrow.NewColumnFromArr(lstField, col)
+ defer column.Release()
+
+ tbl := array.NewTable(
+ arrow.NewSchema([]arrow.Field{lstField}, nil),
+ []arrow.Column{column}, nrows)
+ defer tbl.Release()
+
+ for _, pageVersion := range
[]parquet.DataPageVersion{parquet.DataPageV1, parquet.DataPageV2} {
+ t.Run(fmt.Sprintf("page-v%d", int(pageVersion)+1), func(t
*testing.T) {
+ var buf bytes.Buffer
+ wrProps := parquet.NewWriterProperties(
+ parquet.WithDataPageVersion(pageVersion),
+ parquet.WithDictionaryDefault(false),
+ parquet.WithDataPageSize(256),
+ parquet.WithBatchSize(16),
+ parquet.WithPageIndexEnabled(false),
+ )
+ require.NoError(t, pqarrow.WriteTable(tbl, &buf, nrows,
wrProps, pqarrow.NewArrowWriterProperties()))
+
+ for _, seekRow := range []int64{0, 1, 150, 299} {
+ t.Run(fmt.Sprintf("seek=%d", seekRow), func(t
*testing.T) {
+ pf, err :=
file.NewParquetReader(bytes.NewReader(buf.Bytes()),
file.WithReadProps(parquet.NewReaderProperties(mem)))
+ require.NoError(t, err)
+ defer pf.Close()
+
+ reader, err :=
pqarrow.NewFileReader(pf, pqarrow.ArrowReadProperties{BatchSize: 1024}, mem)
+ require.NoError(t, err)
+
+ rr, err :=
reader.GetRecordReader(context.Background(), nil, nil)
+ require.NoError(t, err)
+ defer rr.Release()
+
+ require.NoError(t,
rr.SeekToRow(seekRow))
+
+ totalRows := 0
+ var firstRow []int32
+ for {
+ rec, err := rr.Read()
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ require.NoError(t, err)
+
+ lst :=
rec.Column(0).(*array.List)
+ for i := 0; i < lst.Len(); i++ {
+ if totalRows == 0 {
+ s, e :=
lst.ValueOffsets(0)
+ v :=
lst.ListValues().(*array.Int32)
+ for j := s; j <
e; j++ {
+
firstRow = append(firstRow, v.Value(int(j)))
+ }
+ }
+ totalRows++
+ }
+ rec.Release()
Review Comment:
`Read()` returns the reader's current batch, and the reader releases it on
the next call, so this is a second release. CI runs with `-tags assert,test`
(`ci/scripts/test.sh`), and with those tags this panics with `too many
releases` in `page-v1/seek=0`. Without the `assert` tag it passes. Please
remove this `Release`.
##########
parquet/file/page_reader.go:
##########
@@ -948,6 +949,35 @@ func (p *serializedPageReader) SeekToPageWithRow(rowIdx
int64) error {
return p.err
}
+// countRowsV1 returns the number of records in a V1 data page by counting
+// the level-0 repetition levels in the page's level region, which the
+// DataPageV1 layout puts first. The levels stay encoded on the page, so this
+// decodes just the repetition run and leaves the buffer untouched.
+func (p *serializedPageReader) countRowsV1(levels []byte, repEnc
format.Encoding, numValues int32) (int64, error) {
+ var (
+ dec encoding.LevelDecoder
+ batch [4096]int16
Review Comment:
This runs for every V1 page of a repeated column, on ordinary sequential
reads as well as seeks. Because `batch[:...]` is passed through `Decode`, the 8
KiB array is most likely heap-allocated on each call. Reading a 512K-row
`list<int32>` V1 column showed no measurable time change, but allocations went
up 5.3% (+18 per read). A scratch buffer kept on `serializedPageReader` would
avoid that.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]