zeroshade commented on code in PR #1647:
URL: https://github.com/apache/arrow-go/pull/1647#discussion_r4645778371
##########
parquet/file/page_reader_internal_test.go:
##########
@@ -395,3 +401,49 @@ func
TestGetDictionaryPageWithoutRecordedOffsetReadsOnlyTheDictionary(t *testing
require.EqualValues(t, dictValues, got.NumValues())
require.Less(t, *read, 2*len(dict), "read %d of %d chunk bytes for a %d
byte dictionary page", *read, len(chunk), len(dict))
}
+
+func TestSeekToRowDictionaryAtDataPageOffsetReadsDictionaryOnce(t *testing.T) {
+ // RLE_DICTIONARY indices 0,1,1,0,0,0,1,0 at bit width 1
+ indices := []byte{1, 3, 0x46}
+ chunk := append(
+ serializePage(t, dictPageSpec(10, 20)),
+ serializePage(t, dataPageSpec(8,
format.Encoding_RLE_DICTIONARY, indices))...,
+ )
+
+ p, outerRead := unencryptedChunkReader(t, 8, chunk)
+ cr := requiredInt32Column(t, p)
+ t.Cleanup(func() { require.NoError(t, cr.Close()) })
+
+ require.NoError(t, cr.SeekToRow(0))
+ vals := make([]int32, 8)
+ _, n, err := cr.ReadBatch(8, vals, nil, nil)
+ require.NoError(t, err)
+ require.Equal(t, []int32{10, 20, 20, 10, 10, 10, 20, 10}, vals[:n])
+ require.Zero(t, *outerRead, "dictionary page was re-read from the chunk
stream after seek already decoded it")
+}
+
+// A dictionary page that fails to decode must invalidate the cached
dictionary,
+// or a later GetDictionaryPage reports the resized dictionary buffer under the
+// header of the page that did decode.
+func TestNextFailedDictionaryPageInvalidatesCachedDictionary(t *testing.T) {
+ first := serializePage(t, dictPageSpec(10, 20))
+ // The second page claims 16 uncompressed bytes but carries 8, which
the reader
+ // rejects after resizing and partly overwriting the dictionary buffer.
+ second := dictPageSpec(4, 5)
+ chunk := append(first, serializePageWithSizes(t, second,
int32(len(second.body)), 16)...)
+
+ p, _ := unencryptedChunkReader(t, 8, chunk)
+ require.True(t, p.Next())
+ require.True(t, p.cachedDict.valid)
Review Comment:
nit: These `cachedDict.valid` assertions (here and L441) check private
state. If the cache weren't cleared, the `GetDictionaryPage` assertions at the
end would already fail, because `Data()` would return the 16-byte resized
buffer. Consider dropping these two lines so the test only checks what a caller
can see.
##########
parquet/file/page_reader.go:
##########
@@ -805,26 +833,24 @@ func (p *serializedPageReader) GetDictionaryPage()
(*DictionaryPage, error) {
return nil, errors.New("parquet: invalid page header
(negative number of values)")
}
+ // A failure below would leave the cache pointing at the
resized buffer.
Review Comment:
nit: This block (clear the cache, resize, decompress, check the length, fill
the cache) is repeated in `Next()` at around L1101. Whether the cache is safe
depends on clearing it before the resize, so it would be good to keep that rule
in one place. For example, `func (p *serializedPageReader) decodeDictPage(rd
io.Reader, h *format.DictionaryPageHeader, lenCompressed, lenUncompressed int)
error`, used by both paths.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]