This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 970a12b5 fix(parquet/encoding): reject truncated plain boolean data
(#1031)
970a12b5 is described below
commit 970a12b5ee93b67119b03bca22bcfc9635a27f0a
Author: Minh Vu <[email protected]>
AuthorDate: Tue Jul 28 17:07:02 2026 +0200
fix(parquet/encoding): reject truncated plain boolean data (#1031)
### Rationale for this change
The plain Boolean decoder trusted that each requested physical value had
a corresponding payload bit. A truncated page could therefore panic
during `Decode`, `Discard`, or `DecodeToBitmap` when the decoder indexed
beyond the input buffer.
The value count passed to `SetData` is only an upper bound because it
includes null rows, so payload bounds must be checked when physical
values are consumed.
### What changes are included in this PR?
* Reject negative Boolean value counts in `SetData`.
* Bound `Decode`, `Discard`, and `DecodeToBitmap` against the payload
bits currently available.
* Return an error wrapping `io.ErrUnexpectedEOF` without consuming
decoder state.
* Continue accepting nullable pages whose level count exceeds their
physical Boolean count.
### Are these changes tested?
Yes. The regression test covers all three truncated consumption paths
and a nullable-style page with 100 levels and four physical values. The
full `parquet/...` test tree passes, including the nullable V1/V2,
spaced-bitmap, and Parquet-to-Arrow paths.
---
parquet/internal/encoding/boolean_decoder.go | 20 ++++++++++++++++
parquet/internal/encoding/encoding_test.go | 36 ++++++++++++++++++++++++++++
2 files changed, 56 insertions(+)
diff --git a/parquet/internal/encoding/boolean_decoder.go
b/parquet/internal/encoding/boolean_decoder.go
index b644baf9..84924742 100644
--- a/parquet/internal/encoding/boolean_decoder.go
+++ b/parquet/internal/encoding/boolean_decoder.go
@@ -43,6 +43,9 @@ func (PlainBooleanDecoder) Type() parquet.Type {
}
func (dec *PlainBooleanDecoder) SetData(nvals int, data []byte) error {
+ if nvals < 0 {
+ return fmt.Errorf("parquet: invalid number of boolean values:
%d", nvals)
+ }
if err := dec.decoder.SetData(nvals, data); err != nil {
return err
}
@@ -50,8 +53,19 @@ func (dec *PlainBooleanDecoder) SetData(nvals int, data
[]byte) error {
return nil
}
+func (dec *PlainBooleanDecoder) ensureBitsAvailable(n int) error {
+ available := int64(len(dec.data))*8 - int64(dec.bitOffset)
+ if int64(n) > available {
+ return fmt.Errorf("parquet: boolean data has %d bits available,
need %d: %w", available, n, io.ErrUnexpectedEOF)
+ }
+ return nil
+}
+
func (dec *PlainBooleanDecoder) Discard(n int) (int, error) {
n = min(n, dec.nvals)
+ if err := dec.ensureBitsAvailable(n); err != nil {
+ return 0, err
+ }
dec.nvals -= n
if dec.bitOffset+n < 8 {
@@ -77,6 +91,9 @@ func (dec *PlainBooleanDecoder) Discard(n int) (int, error) {
// Returns the number of values decoded
func (dec *PlainBooleanDecoder) Decode(out []bool) (int, error) {
max := shared_utils.Min(len(out), dec.nvals)
+ if err := dec.ensureBitsAvailable(max); err != nil {
+ return 0, err
+ }
// attempts to read all remaining bool values from the current data byte
unalignedExtract := func(i int) int {
@@ -127,6 +144,9 @@ func (dec *PlainBooleanDecoder) DecodeToBitmap(out []byte,
outOffset int64, leng
if max == 0 {
return 0, nil
}
+ if err := dec.ensureBitsAvailable(max); err != nil {
+ return 0, err
+ }
// Check if we're aligned and can do a fast copy
if dec.bitOffset == 0 && outOffset%8 == 0 {
diff --git a/parquet/internal/encoding/encoding_test.go
b/parquet/internal/encoding/encoding_test.go
index 8242e56e..7a73df08 100644
--- a/parquet/internal/encoding/encoding_test.go
+++ b/parquet/internal/encoding/encoding_test.go
@@ -1089,6 +1089,42 @@ func TestBooleanPlainDecoderAfterFlushing(t *testing.T) {
assert.Equal(t, decSlice[0], false)
}
+func TestBooleanPlainDecoderRejectsTruncatedData(t *testing.T) {
+ descr := schema.NewColumn(schema.NewBooleanNode("bool",
parquet.Repetitions.Required, -1), 0, 0)
+ type booleanBitmapDecoder interface {
+ encoding.BooleanDecoder
+ DecodeToBitmap([]byte, int64, int) (int, error)
+ }
+ newDecoder := func() booleanBitmapDecoder {
+ dec := encoding.NewDecoder(parquet.Types.Boolean,
parquet.Encodings.Plain, descr, memory.DefaultAllocator).(booleanBitmapDecoder)
+ require.NoError(t, dec.SetData(9, []byte{0xff}))
+ return dec
+ }
+
+ dec := newDecoder()
+ n, err := dec.Decode(make([]bool, 9))
+ assert.Zero(t, n)
+ assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
+
+ dec = newDecoder()
+ n, err = dec.Discard(9)
+ assert.Zero(t, n)
+ assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
+
+ dec = newDecoder()
+ n, err = dec.DecodeToBitmap(make([]byte, 2), 0, 9)
+ assert.Zero(t, n)
+ assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
+
+ dec = encoding.NewDecoder(parquet.Types.Boolean,
parquet.Encodings.Plain, descr, memory.DefaultAllocator).(booleanBitmapDecoder)
+ require.NoError(t, dec.SetData(100, []byte{0x0f}))
+ n, err = dec.Decode(make([]bool, 4))
+ assert.Equal(t, 4, n)
+ assert.NoError(t, err)
+
+ assert.Error(t, dec.SetData(-1, nil))
+}
+
func TestBooleanPlainEncoderPutBitmap(t *testing.T) {
descr := schema.NewColumn(schema.NewBooleanNode("bool",
parquet.Repetitions.Optional, -1), 0, 0)
enc := encoding.NewEncoder(parquet.Types.Boolean,
parquet.Encodings.Plain, false, descr, memory.DefaultAllocator)
