hcrosse commented on code in PR #1647:
URL: https://github.com/apache/arrow-go/pull/1647#discussion_r4645966235
##########
parquet/file/page_reader.go:
##########
@@ -805,26 +833,24 @@ func (p *serializedPageReader) GetDictionaryPage()
(*DictionaryPage, error) {
return nil, errors.New("parquet: invalid page header
(negative number of values)")
}
+ // A failure below would leave the cache pointing at the
resized buffer.
Review Comment:
Extracted `decodeDictPage` and called it from both `GetDictionaryPage` and
`Next` so the cache invalidation before resize stays in one place.
##########
parquet/file/page_reader_internal_test.go:
##########
@@ -395,3 +401,49 @@ func
TestGetDictionaryPageWithoutRecordedOffsetReadsOnlyTheDictionary(t *testing
require.EqualValues(t, dictValues, got.NumValues())
require.Less(t, *read, 2*len(dict), "read %d of %d chunk bytes for a %d
byte dictionary page", *read, len(chunk), len(dict))
}
+
+func TestSeekToRowDictionaryAtDataPageOffsetReadsDictionaryOnce(t *testing.T) {
+ // RLE_DICTIONARY indices 0,1,1,0,0,0,1,0 at bit width 1
+ indices := []byte{1, 3, 0x46}
+ chunk := append(
+ serializePage(t, dictPageSpec(10, 20)),
+ serializePage(t, dataPageSpec(8,
format.Encoding_RLE_DICTIONARY, indices))...,
+ )
+
+ p, outerRead := unencryptedChunkReader(t, 8, chunk)
+ cr := requiredInt32Column(t, p)
+ t.Cleanup(func() { require.NoError(t, cr.Close()) })
+
+ require.NoError(t, cr.SeekToRow(0))
+ vals := make([]int32, 8)
+ _, n, err := cr.ReadBatch(8, vals, nil, nil)
+ require.NoError(t, err)
+ require.Equal(t, []int32{10, 20, 20, 10, 10, 10, 20, 10}, vals[:n])
+ require.Zero(t, *outerRead, "dictionary page was re-read from the chunk
stream after seek already decoded it")
+}
+
+// A dictionary page that fails to decode must invalidate the cached
dictionary,
+// or a later GetDictionaryPage reports the resized dictionary buffer under the
+// header of the page that did decode.
+func TestNextFailedDictionaryPageInvalidatesCachedDictionary(t *testing.T) {
+ first := serializePage(t, dictPageSpec(10, 20))
+ // The second page claims 16 uncompressed bytes but carries 8, which
the reader
+ // rejects after resizing and partly overwriting the dictionary buffer.
+ second := dictPageSpec(4, 5)
+ chunk := append(first, serializePageWithSizes(t, second,
int32(len(second.body)), 16)...)
+
+ p, _ := unencryptedChunkReader(t, 8, chunk)
+ require.True(t, p.Next())
+ require.True(t, p.cachedDict.valid)
Review Comment:
Dropped both `cachedDict.valid` assertions and confirmed the test still
fails on `dict.Data()` if the cache clear is removed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]