This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 7b3abb64 fix(parquet/variant): reject truncated scalar values (#1017)
7b3abb64 is described below
commit 7b3abb64dba8a9a2b6366600894d06b2eaeda925
Author: Minh Vu <[email protected]>
AuthorDate: Mon Jul 27 19:08:59 2026 +0200
fix(parquet/variant): reject truncated scalar values (#1017)
### Rationale for this change
NewWithMetadata only checks that a variant value is non-empty. Truncated
or unknown scalar encodings can therefore be accepted and later panic or
read invalid data when their type or value is accessed.
### What changes are included in this PR?
* Validate the encoded width of fixed-size primitive values and short
strings.
* Validate binary and string length prefixes and their payload sizes.
* Reject unknown primitive type codes while keeping compound values
lazily decoded.
### Are these changes tested?
Yes. The tests cover unknown primitive types, truncated fixed-width
values, truncated binary prefixes and payloads, and truncated short
strings. The parquet/variant and parquet/pqarrow packages pass,
including the assertion build for parquet/variant.
---
parquet/variant/variant.go | 57 +++++++++++++++++++++++++++++++++++++++++
parquet/variant/variant_test.go | 23 +++++++++++++++++
2 files changed, 80 insertions(+)
diff --git a/parquet/variant/variant.go b/parquet/variant/variant.go
index 25eacd43..8e31969c 100644
--- a/parquet/variant/variant.go
+++ b/parquet/variant/variant.go
@@ -501,10 +501,67 @@ func NewWithMetadata(meta Metadata, value []byte) (Value,
error) {
if len(value) == 0 {
return Value{}, errors.New("invalid variant value: empty")
}
+ if err := validateScalarValue(value); err != nil {
+ return Value{}, err
+ }
return Value{value: value, meta: meta}, nil
}
+func validateScalarValue(value []byte) error {
+ if basicTypeFromHeader(value[0]) == BasicShortString {
+ want := 1 + int(value[0]>>basicTypeBits)
+ if len(value) < want {
+ return fmt.Errorf("invalid variant value: short string
requires %d bytes, got %d", want, len(value))
+ }
+ return nil
+ }
+ if basicTypeFromHeader(value[0]) != BasicPrimitive {
+ return nil
+ }
+
+ primitiveType := primitiveTypeFromHeader(value[0])
+ want := 0
+ switch primitiveType {
+ case PrimitiveNull, PrimitiveBoolTrue, PrimitiveBoolFalse:
+ want = 1
+ case PrimitiveInt8:
+ want = 2
+ case PrimitiveInt16:
+ want = 3
+ case PrimitiveInt32, PrimitiveDate, PrimitiveFloat:
+ want = 5
+ case PrimitiveInt64, PrimitiveDouble, PrimitiveTimeMicrosNTZ,
+ PrimitiveTimestampMicros, PrimitiveTimestampMicrosNTZ,
+ PrimitiveTimestampNanos, PrimitiveTimestampNanosNTZ:
+ want = 9
+ case PrimitiveDecimal4:
+ want = 6
+ case PrimitiveDecimal8:
+ want = 10
+ case PrimitiveDecimal16:
+ want = 18
+ case PrimitiveUUID:
+ want = 17
+ case PrimitiveBinary, PrimitiveString:
+ if len(value) < 5 {
+ return fmt.Errorf("invalid variant value: %s length
prefix requires 5 bytes, got %d", primitiveType, len(value))
+ }
+ dataLen := uint64(binary.LittleEndian.Uint32(value[1:5]))
+ if dataLen > uint64(len(value)-5) {
+ return fmt.Errorf("invalid variant value: %s data
requires %d bytes, got %d", primitiveType, dataLen, len(value)-5)
+ }
+ return nil
+ default:
+ return fmt.Errorf("invalid variant value: unknown primitive
type %d", primitiveType)
+ }
+
+ if len(value) < want {
+ return fmt.Errorf("invalid variant value: %s requires %d bytes,
got %d", primitiveType, want, len(value))
+ }
+ return nil
+}
+
// New creates a Value by parsing both the metadata and value bytes.
func New(meta, value []byte) (Value, error) {
m, err := NewMetadata(meta)
diff --git a/parquet/variant/variant_test.go b/parquet/variant/variant_test.go
index 649a719c..37bc04c8 100644
--- a/parquet/variant/variant_test.go
+++ b/parquet/variant/variant_test.go
@@ -620,6 +620,29 @@ func TestInvalidValue(t *testing.T) {
}
}
+func TestInvalidPrimitiveValue(t *testing.T) {
+ tests := []struct {
+ name string
+ value []byte
+ }{
+ {name: "unknown primitive", value: []byte{byte(63 << 2)}},
+ {name: "truncated int32", value:
[]byte{byte(variant.PrimitiveInt32 << 2)}},
+ {name: "truncated binary length", value:
[]byte{byte(variant.PrimitiveBinary << 2), 1}},
+ {name: "truncated binary data", value:
[]byte{byte(variant.PrimitiveBinary << 2), 2, 0, 0, 0, 1}},
+ {name: "truncated short string", value: []byte{byte(3<<2) |
byte(variant.BasicShortString), 'a'}},
+ }
+
+ meta, err := variant.NewMetadata(variant.EmptyMetadataBytes[:])
+ require.NoError(t, err)
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ _, err := variant.NewWithMetadata(meta, tt.value)
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "invalid variant value")
+ })
+ }
+}
+
func TestInvalidObjectAccess(t *testing.T) {
v := loadVariant(t, "object_primitive")
obj := v.Value().(variant.ObjectValue)
