This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new f6708819 fix(parquet/metadata): reject malformed fixed-width
statistics (#1010)
f6708819 is described below
commit f670881919035fa6a07ea10ac70ddf3330eba31c
Author: Minh Vu <[email protected]>
AuthorDate: Mon Jul 27 22:30:38 2026 +0200
fix(parquet/metadata): reject malformed fixed-width statistics (#1010)
### Rationale for this change
Fixed-width Parquet statistics must use the exact PLAIN-encoded physical
width. Padding a shorter value invents a different bound, while
formatting the raw bytes can panic. Variable-length BYTE_ARRAY bounds
should remain unchanged.
### What changes are included in this PR?
- Ignore min/max when a present fixed-width statistic does not match the
declared physical width.
- Preserve variable-length BYTE_ARRAY values exactly as encoded.
- Return malformed scalar bytes unchanged from `GetStatValue` instead of
padding or indexing past them.
- Regenerate the typed statistics implementation from its template.
### Are these changes tested?
Yes. Focused tests cover malformed numeric and FIXED_LEN_BYTE_ARRAY
values, short scalar formatting, and BYTE_ARRAY preservation. The
metadata and Parquet file test suites pass.
---
parquet/metadata/statistics.go | 30 +++++++
parquet/metadata/statistics_test.go | 36 ++++++++
parquet/metadata/statistics_types.gen.go | 113 +++++++++++++++++---------
parquet/metadata/statistics_types.gen.go.tmpl | 14 ++--
4 files changed, 148 insertions(+), 45 deletions(-)
diff --git a/parquet/metadata/statistics.go b/parquet/metadata/statistics.go
index 0aa9ef46..1ef11653 100644
--- a/parquet/metadata/statistics.go
+++ b/parquet/metadata/statistics.go
@@ -605,7 +605,37 @@ func (FixedLenByteArrayStatistics) cleanStat(minMax
minmaxPairFixedLenByteArray)
return &minMax
}
+func fixedStatByteWidth(typ parquet.Type) (int, bool) {
+ switch typ {
+ case parquet.Types.Boolean:
+ return 1, true
+ case parquet.Types.Int32, parquet.Types.Float:
+ return 4, true
+ case parquet.Types.Int64, parquet.Types.Double:
+ return 8, true
+ case parquet.Types.Int96:
+ return 12, true
+ default:
+ return 0, false
+ }
+}
+
+func validEncodedStat(descr *schema.Column, val []byte) bool {
+ if descr.PhysicalType() == parquet.Types.ByteArray {
+ return true
+ }
+ if descr.PhysicalType() == parquet.Types.FixedLenByteArray {
+ return len(val) == int(descr.TypeLength())
+ }
+ width, ok := fixedStatByteWidth(descr.PhysicalType())
+ return ok && len(val) == width
+}
+
func GetStatValue(typ parquet.Type, val []byte) interface{} {
+ if width, ok := fixedStatByteWidth(typ); ok && len(val) != width {
+ return val
+ }
+
switch typ {
case parquet.Types.Boolean:
return val[0] != 0
diff --git a/parquet/metadata/statistics_test.go
b/parquet/metadata/statistics_test.go
index f7fcbda6..4aa1607a 100644
--- a/parquet/metadata/statistics_test.go
+++ b/parquet/metadata/statistics_test.go
@@ -858,3 +858,39 @@ func TestByteArrayStatisticsFromEncodedOwnsMinMax(t
*testing.T) {
assert.Equal(t, "aaa", string(stats.Min()))
assert.Equal(t, "zzz", string(stats.Max()))
}
+
+func TestMalformedFixedWidthStatisticsAreIgnored(t *testing.T) {
+ descr := schema.NewColumn(schema.NewInt32Node("i32",
parquet.Repetitions.Required, -1), 0, 0)
+ stats := metadata.NewStatisticsFromEncoded(descr,
memory.DefaultAllocator, 1,
+ &encodedStatProvider{min: []byte{0x34, 0x12}, max:
[]byte{0x78}}).(*metadata.Int32Statistics)
+
+ assert.False(t, stats.HasMinMax())
+ assert.Equal(t, []byte{0x34, 0x12},
metadata.GetStatValue(parquet.Types.Int32, []byte{0x34, 0x12}))
+ assert.Equal(t, []byte(nil),
metadata.GetStatValue(parquet.Types.Boolean, nil))
+
+ for _, typ := range []parquet.Type{
+ parquet.Types.Boolean,
+ parquet.Types.Int32,
+ parquet.Types.Int64,
+ parquet.Types.Int96,
+ parquet.Types.Float,
+ parquet.Types.Double,
+ } {
+ assert.NotPanics(t, func() {
+ metadata.GetStatValue(typ, []byte{1})
+ })
+ }
+}
+
+func TestMalformedFixedLenByteArrayStatisticsAreIgnored(t *testing.T) {
+ descr := schema.NewColumn(schema.NewFixedLenByteArrayNode("flba",
parquet.Repetitions.Required, 8, -1), 0, 0)
+ stats := metadata.NewStatisticsFromEncoded(descr,
memory.DefaultAllocator, 1,
+ &encodedStatProvider{min: []byte("ab"), max:
[]byte("xy")}).(*metadata.FixedLenByteArrayStatistics)
+
+ assert.False(t, stats.HasMinMax())
+}
+
+func TestGetStatValuePreservesByteArrays(t *testing.T) {
+ assert.Equal(t, []byte("ab"),
metadata.GetStatValue(parquet.Types.ByteArray, []byte("ab")))
+ assert.Equal(t, []byte("xy"),
metadata.GetStatValue(parquet.Types.FixedLenByteArray, []byte("xy")))
+}
diff --git a/parquet/metadata/statistics_types.gen.go
b/parquet/metadata/statistics_types.gen.go
index e450fb7c..aa4099a5 100644
--- a/parquet/metadata/statistics_types.gen.go
+++ b/parquet/metadata/statistics_types.gen.go
@@ -79,11 +79,16 @@ func NewInt32StatisticsFromEncoded(descr *schema.Column,
mem memory.Allocator, n
}
encodedMin := encoded.GetMin()
- if encodedMin != nil && len(encodedMin) > 0 {
+ encodedMax := encoded.GetMax()
+ if (encoded.IsSetMin() && !validEncodedStat(descr, encodedMin)) ||
+ (encoded.IsSetMax() && !validEncodedStat(descr, encodedMax)) {
+ return ret
+ }
+
+ if encoded.IsSetMin() {
ret.min = ret.plainDecode(encodedMin)
}
- encodedMax := encoded.GetMax()
- if encodedMax != nil && len(encodedMax) > 0 {
+ if encoded.IsSetMax() {
ret.max = ret.plainDecode(encodedMax)
}
ret.hasMinMax = encoded.IsSetMax() || encoded.IsSetMin()
@@ -105,7 +110,6 @@ func (s *Int32Statistics) plainEncode(src int32) []byte {
func (s *Int32Statistics) plainDecode(src []byte) int32 {
var buf [1]int32
-
decoder := encoding.NewDecoder(s.descr.PhysicalType(),
parquet.Encodings.Plain, s.descr, s.mem)
decoder.SetData(1, src)
decoder.(encoding.Int32Decoder).Decode(buf[:])
@@ -379,11 +383,16 @@ func NewInt64StatisticsFromEncoded(descr *schema.Column,
mem memory.Allocator, n
}
encodedMin := encoded.GetMin()
- if encodedMin != nil && len(encodedMin) > 0 {
+ encodedMax := encoded.GetMax()
+ if (encoded.IsSetMin() && !validEncodedStat(descr, encodedMin)) ||
+ (encoded.IsSetMax() && !validEncodedStat(descr, encodedMax)) {
+ return ret
+ }
+
+ if encoded.IsSetMin() {
ret.min = ret.plainDecode(encodedMin)
}
- encodedMax := encoded.GetMax()
- if encodedMax != nil && len(encodedMax) > 0 {
+ if encoded.IsSetMax() {
ret.max = ret.plainDecode(encodedMax)
}
ret.hasMinMax = encoded.IsSetMax() || encoded.IsSetMin()
@@ -405,7 +414,6 @@ func (s *Int64Statistics) plainEncode(src int64) []byte {
func (s *Int64Statistics) plainDecode(src []byte) int64 {
var buf [1]int64
-
decoder := encoding.NewDecoder(s.descr.PhysicalType(),
parquet.Encodings.Plain, s.descr, s.mem)
decoder.SetData(1, src)
decoder.(encoding.Int64Decoder).Decode(buf[:])
@@ -679,11 +687,16 @@ func NewInt96StatisticsFromEncoded(descr *schema.Column,
mem memory.Allocator, n
}
encodedMin := encoded.GetMin()
- if encodedMin != nil && len(encodedMin) > 0 {
+ encodedMax := encoded.GetMax()
+ if (encoded.IsSetMin() && !validEncodedStat(descr, encodedMin)) ||
+ (encoded.IsSetMax() && !validEncodedStat(descr, encodedMax)) {
+ return ret
+ }
+
+ if encoded.IsSetMin() {
ret.min = ret.plainDecode(encodedMin)
}
- encodedMax := encoded.GetMax()
- if encodedMax != nil && len(encodedMax) > 0 {
+ if encoded.IsSetMax() {
ret.max = ret.plainDecode(encodedMax)
}
ret.hasMinMax = encoded.IsSetMax() || encoded.IsSetMin()
@@ -705,7 +718,6 @@ func (s *Int96Statistics) plainEncode(src parquet.Int96)
[]byte {
func (s *Int96Statistics) plainDecode(src []byte) parquet.Int96 {
var buf [1]parquet.Int96
-
decoder := encoding.NewDecoder(s.descr.PhysicalType(),
parquet.Encodings.Plain, s.descr, s.mem)
decoder.SetData(1, src)
decoder.(encoding.Int96Decoder).Decode(buf[:])
@@ -957,11 +969,16 @@ func NewFloat32StatisticsFromEncoded(descr
*schema.Column, mem memory.Allocator,
}
encodedMin := encoded.GetMin()
- if encodedMin != nil && len(encodedMin) > 0 {
+ encodedMax := encoded.GetMax()
+ if (encoded.IsSetMin() && !validEncodedStat(descr, encodedMin)) ||
+ (encoded.IsSetMax() && !validEncodedStat(descr, encodedMax)) {
+ return ret
+ }
+
+ if encoded.IsSetMin() {
ret.min = ret.plainDecode(encodedMin)
}
- encodedMax := encoded.GetMax()
- if encodedMax != nil && len(encodedMax) > 0 {
+ if encoded.IsSetMax() {
ret.max = ret.plainDecode(encodedMax)
}
ret.hasMinMax = encoded.IsSetMax() || encoded.IsSetMin()
@@ -983,7 +1000,6 @@ func (s *Float32Statistics) plainEncode(src float32)
[]byte {
func (s *Float32Statistics) plainDecode(src []byte) float32 {
var buf [1]float32
-
decoder := encoding.NewDecoder(s.descr.PhysicalType(),
parquet.Encodings.Plain, s.descr, s.mem)
decoder.SetData(1, src)
decoder.(encoding.Float32Decoder).Decode(buf[:])
@@ -1249,11 +1265,16 @@ func NewFloat64StatisticsFromEncoded(descr
*schema.Column, mem memory.Allocator,
}
encodedMin := encoded.GetMin()
- if encodedMin != nil && len(encodedMin) > 0 {
+ encodedMax := encoded.GetMax()
+ if (encoded.IsSetMin() && !validEncodedStat(descr, encodedMin)) ||
+ (encoded.IsSetMax() && !validEncodedStat(descr, encodedMax)) {
+ return ret
+ }
+
+ if encoded.IsSetMin() {
ret.min = ret.plainDecode(encodedMin)
}
- encodedMax := encoded.GetMax()
- if encodedMax != nil && len(encodedMax) > 0 {
+ if encoded.IsSetMax() {
ret.max = ret.plainDecode(encodedMax)
}
ret.hasMinMax = encoded.IsSetMax() || encoded.IsSetMin()
@@ -1275,7 +1296,6 @@ func (s *Float64Statistics) plainEncode(src float64)
[]byte {
func (s *Float64Statistics) plainDecode(src []byte) float64 {
var buf [1]float64
-
decoder := encoding.NewDecoder(s.descr.PhysicalType(),
parquet.Encodings.Plain, s.descr, s.mem)
decoder.SetData(1, src)
decoder.(encoding.Float64Decoder).Decode(buf[:])
@@ -1541,11 +1561,16 @@ func NewBooleanStatisticsFromEncoded(descr
*schema.Column, mem memory.Allocator,
}
encodedMin := encoded.GetMin()
- if encodedMin != nil && len(encodedMin) > 0 {
+ encodedMax := encoded.GetMax()
+ if (encoded.IsSetMin() && !validEncodedStat(descr, encodedMin)) ||
+ (encoded.IsSetMax() && !validEncodedStat(descr, encodedMax)) {
+ return ret
+ }
+
+ if encoded.IsSetMin() {
ret.min = ret.plainDecode(encodedMin)
}
- encodedMax := encoded.GetMax()
- if encodedMax != nil && len(encodedMax) > 0 {
+ if encoded.IsSetMax() {
ret.max = ret.plainDecode(encodedMax)
}
ret.hasMinMax = encoded.IsSetMax() || encoded.IsSetMin()
@@ -1567,7 +1592,6 @@ func (s *BooleanStatistics) plainEncode(src bool) []byte {
func (s *BooleanStatistics) plainDecode(src []byte) bool {
var buf [1]bool
-
decoder := encoding.NewDecoder(s.descr.PhysicalType(),
parquet.Encodings.Plain, s.descr, s.mem)
decoder.SetData(1, src)
decoder.(encoding.BooleanDecoder).Decode(buf[:])
@@ -1898,13 +1922,18 @@ func NewByteArrayStatisticsFromEncoded(descr
*schema.Column, mem memory.Allocato
}
encodedMin := encoded.GetMin()
- if encodedMin != nil && len(encodedMin) > 0 {
+ encodedMax := encoded.GetMax()
+ if (encoded.IsSetMin() && !validEncodedStat(descr, encodedMin)) ||
+ (encoded.IsSetMax() && !validEncodedStat(descr, encodedMax)) {
+ return ret
+ }
+
+ if encoded.IsSetMin() {
// Copy into a statistics-owned buffer so the stored min does
not alias the
// encoded metadata buffer; SetMinMax reuses this buffer on
later updates.
ret.min = append(ret.min[:0], ret.plainDecode(encodedMin)...)
}
- encodedMax := encoded.GetMax()
- if encodedMax != nil && len(encodedMax) > 0 {
+ if encoded.IsSetMax() {
ret.max = append(ret.max[:0], ret.plainDecode(encodedMax)...)
}
ret.hasMinMax = encoded.IsSetMax() || encoded.IsSetMin()
@@ -2220,13 +2249,18 @@ func NewFixedLenByteArrayStatisticsFromEncoded(descr
*schema.Column, mem memory.
}
encodedMin := encoded.GetMin()
- if encodedMin != nil && len(encodedMin) > 0 {
+ encodedMax := encoded.GetMax()
+ if (encoded.IsSetMin() && !validEncodedStat(descr, encodedMin)) ||
+ (encoded.IsSetMax() && !validEncodedStat(descr, encodedMax)) {
+ return ret
+ }
+
+ if encoded.IsSetMin() {
// Copy into a statistics-owned buffer so the stored min does
not alias the
// encoded metadata buffer; SetMinMax reuses this buffer on
later updates.
ret.min = append(ret.min[:0], ret.plainDecode(encodedMin)...)
}
- encodedMax := encoded.GetMax()
- if encodedMax != nil && len(encodedMax) > 0 {
+ if encoded.IsSetMax() {
ret.max = append(ret.max[:0], ret.plainDecode(encodedMax)...)
}
ret.hasMinMax = encoded.IsSetMax() || encoded.IsSetMin()
@@ -2247,12 +2281,7 @@ func (s *FixedLenByteArrayStatistics) plainEncode(src
parquet.FixedLenByteArray)
}
func (s *FixedLenByteArrayStatistics) plainDecode(src []byte)
parquet.FixedLenByteArray {
- var buf [1]parquet.FixedLenByteArray
-
- decoder := encoding.NewDecoder(s.descr.PhysicalType(),
parquet.Encodings.Plain, s.descr, s.mem)
- decoder.SetData(1, src)
- decoder.(encoding.FixedLenByteArrayDecoder).Decode(buf[:])
- return buf[0]
+ return src
}
func (s *FixedLenByteArrayStatistics) minval(a, b parquet.FixedLenByteArray)
parquet.FixedLenByteArray {
@@ -2552,13 +2581,18 @@ func NewFloat16StatisticsFromEncoded(descr
*schema.Column, mem memory.Allocator,
}
encodedMin := encoded.GetMin()
- if encodedMin != nil && len(encodedMin) > 0 {
+ encodedMax := encoded.GetMax()
+ if (encoded.IsSetMin() && !validEncodedStat(descr, encodedMin)) ||
+ (encoded.IsSetMax() && !validEncodedStat(descr, encodedMax)) {
+ return ret
+ }
+
+ if encoded.IsSetMin() {
// Copy into a statistics-owned buffer so the stored min does
not alias the
// encoded metadata buffer; SetMinMax reuses this buffer on
later updates.
ret.min = append(ret.min[:0], ret.plainDecode(encodedMin)...)
}
- encodedMax := encoded.GetMax()
- if encodedMax != nil && len(encodedMax) > 0 {
+ if encoded.IsSetMax() {
ret.max = append(ret.max[:0], ret.plainDecode(encodedMax)...)
}
ret.hasMinMax = encoded.IsSetMax() || encoded.IsSetMin()
@@ -2580,7 +2614,6 @@ func (s *Float16Statistics) plainEncode(src
parquet.FixedLenByteArray) []byte {
func (s *Float16Statistics) plainDecode(src []byte) parquet.FixedLenByteArray {
var buf [1]parquet.FixedLenByteArray
-
decoder := encoding.NewDecoder(s.descr.PhysicalType(),
parquet.Encodings.Plain, s.descr, s.mem)
decoder.SetData(1, src)
decoder.(encoding.FixedLenByteArrayDecoder).Decode(buf[:])
diff --git a/parquet/metadata/statistics_types.gen.go.tmpl
b/parquet/metadata/statistics_types.gen.go.tmpl
index 0f0bc377..0938d4ed 100644
--- a/parquet/metadata/statistics_types.gen.go.tmpl
+++ b/parquet/metadata/statistics_types.gen.go.tmpl
@@ -86,7 +86,13 @@ func New{{.Name}}StatisticsFromEncoded(descr *schema.Column,
mem memory.Allocato
}
encodedMin := encoded.GetMin()
- if encodedMin != nil && len(encodedMin) > 0 {
+ encodedMax := encoded.GetMax()
+ if (encoded.IsSetMin() && !validEncodedStat(descr, encodedMin)) ||
+ (encoded.IsSetMax() && !validEncodedStat(descr, encodedMax)) {
+ return ret
+ }
+
+ if encoded.IsSetMin() {
{{- if or (eq .name "parquet.ByteArray") (eq .name
"parquet.FixedLenByteArray")}}
// Copy into a statistics-owned buffer so the stored min does not alias the
// encoded metadata buffer; SetMinMax reuses this buffer on later updates.
@@ -95,8 +101,7 @@ func New{{.Name}}StatisticsFromEncoded(descr *schema.Column,
mem memory.Allocato
ret.min = ret.plainDecode(encodedMin)
{{- end}}
}
- encodedMax := encoded.GetMax()
- if encodedMax != nil && len(encodedMax) > 0 {
+ if encoded.IsSetMax() {
{{- if or (eq .name "parquet.ByteArray") (eq .name
"parquet.FixedLenByteArray")}}
ret.max = append(ret.max[:0], ret.plainDecode(encodedMax)...)
{{- else}}
@@ -127,11 +132,10 @@ func (s *{{.Name}}Statistics) plainEncode(src {{.name}})
[]byte {
}
func (s *{{.Name}}Statistics) plainDecode(src []byte) {{.name}} {
-{{- if eq .Name "ByteArray"}}
+{{- if or (eq .Name "ByteArray") (eq .Name "FixedLenByteArray")}}
return src
{{- else}}
var buf [1]{{.name}}
-
decoder := encoding.NewDecoder(s.descr.PhysicalType(),
parquet.Encodings.Plain, s.descr, s.mem)
decoder.SetData(1, src)
decoder.(encoding.{{if
.logical}}{{.physical}}{{else}}{{.Name}}{{end}}Decoder).Decode(buf[:])
