This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 50fb0781 fix(parquet/metadata): count non-null values from the
validity bitmap in UpdateSpaced (#1367)
50fb0781 is described below
commit 50fb07811d3626f175e66ceba020c775594768b7
Author: Animesh Kumar <[email protected]>
AuthorDate: Tue Oct 13 19:41:54 2026 +0530
fix(parquet/metadata): count non-null values from the validity bitmap in
UpdateSpaced (#1367)
### Rationale for this change
Fixes #1366. Writing a `list<timestamp>` column whose only row is a null
list with statistics enabled produced `min = MaxInt64` and `max =
MinInt64` with `HasMinMax() == true`, so DuckDB failed to read the file.
`UpdateSpaced` computed the non-null count as `len(values) - numNull`.
For a nested column the spaced values only cover the leaf slots, but
`numNull` also counts the null list, so the count was `0 - 1 = -1`. That
skipped the `notnull == 0` early return and set min/max from no values.
The same miscount also hid real values: a null list next to `[5]` gave
`1 - 1 = 0`, so the `5` never reached min/max.
### What changes are included in this PR?
`UpdateSpaced` counts the non-null values with `bitutil.CountSetBits`
over `validBits`, the bitmap it already uses for `getMinMaxSpaced`. The
change is in `statistics_types.gen.go.tmpl`, and
`statistics_types.gen.go` is regenerated with `go generate`.
### Are these changes tested?
Yes. `TestWriteArrowNestedNullListStatistics` writes a column with only
a null list and one with a null list followed by `[5]`, on both data
page versions. It checks the null count, the number of values and that
min/max are absent or `5`. All four cases fail on main and pass with the
fix.
`go test ./parquet/... ./arrow/compute/...` passes with the testing
submodules, and golangci-lint reports no issues.
### Are there any user-facing changes?
Nested columns with null lists no longer get invalid min/max statistics,
and their non-null values are included in min/max.
---
parquet/metadata/statistics_types.gen.go | 18 ++++-----
parquet/metadata/statistics_types.gen.go.tmpl | 3 +-
parquet/pqarrow/encode_arrow_test.go | 54 +++++++++++++++++++++++++++
3 files changed, 65 insertions(+), 10 deletions(-)
diff --git a/parquet/metadata/statistics_types.gen.go
b/parquet/metadata/statistics_types.gen.go
index aa4099a5..f6e93ef4 100644
--- a/parquet/metadata/statistics_types.gen.go
+++ b/parquet/metadata/statistics_types.gen.go
@@ -242,7 +242,7 @@ func (s *Int32Statistics) Update(values []int32, numNull
int64) {
// and skip null values.
func (s *Int32Statistics) UpdateSpaced(values []int32, validBits []byte,
validBitsOffset, numNull int64) {
s.IncNulls(numNull)
- notnull := int64(len(values)) - numNull
+ notnull := int64(bitutil.CountSetBits(validBits, int(validBitsOffset),
len(values)))
s.nvalues += notnull
if notnull == 0 {
@@ -546,7 +546,7 @@ func (s *Int64Statistics) Update(values []int64, numNull
int64) {
// and skip null values.
func (s *Int64Statistics) UpdateSpaced(values []int64, validBits []byte,
validBitsOffset, numNull int64) {
s.IncNulls(numNull)
- notnull := int64(len(values)) - numNull
+ notnull := int64(bitutil.CountSetBits(validBits, int(validBitsOffset),
len(values)))
s.nvalues += notnull
if notnull == 0 {
@@ -835,7 +835,7 @@ func (s *Int96Statistics) Update(values []parquet.Int96,
numNull int64) {
// and skip null values.
func (s *Int96Statistics) UpdateSpaced(values []parquet.Int96, validBits
[]byte, validBitsOffset, numNull int64) {
s.IncNulls(numNull)
- notnull := int64(len(values)) - numNull
+ notnull := int64(bitutil.CountSetBits(validBits, int(validBitsOffset),
len(values)))
s.nvalues += notnull
if notnull == 0 {
@@ -1124,7 +1124,7 @@ func (s *Float32Statistics) Update(values []float32,
numNull int64) {
// and skip null values.
func (s *Float32Statistics) UpdateSpaced(values []float32, validBits []byte,
validBitsOffset, numNull int64) {
s.IncNulls(numNull)
- notnull := int64(len(values)) - numNull
+ notnull := int64(bitutil.CountSetBits(validBits, int(validBitsOffset),
len(values)))
s.nvalues += notnull
if notnull == 0 {
@@ -1420,7 +1420,7 @@ func (s *Float64Statistics) Update(values []float64,
numNull int64) {
// and skip null values.
func (s *Float64Statistics) UpdateSpaced(values []float64, validBits []byte,
validBitsOffset, numNull int64) {
s.IncNulls(numNull)
- notnull := int64(len(values)) - numNull
+ notnull := int64(bitutil.CountSetBits(validBits, int(validBitsOffset),
len(values)))
s.nvalues += notnull
if notnull == 0 {
@@ -1709,7 +1709,7 @@ func (s *BooleanStatistics) Update(values []bool, numNull
int64) {
// and skip null values.
func (s *BooleanStatistics) UpdateSpaced(values []bool, validBits []byte,
validBitsOffset, numNull int64) {
s.IncNulls(numNull)
- notnull := int64(len(values)) - numNull
+ notnull := int64(bitutil.CountSetBits(validBits, int(validBitsOffset),
len(values)))
s.nvalues += notnull
if notnull == 0 {
@@ -2084,7 +2084,7 @@ func (s *ByteArrayStatistics) Update(values
[]parquet.ByteArray, numNull int64)
// and skip null values.
func (s *ByteArrayStatistics) UpdateSpaced(values []parquet.ByteArray,
validBits []byte, validBitsOffset, numNull int64) {
s.IncNulls(numNull)
- notnull := int64(len(values)) - numNull
+ notnull := int64(bitutil.CountSetBits(validBits, int(validBitsOffset),
len(values)))
s.nvalues += notnull
if notnull == 0 {
@@ -2418,7 +2418,7 @@ func (s *FixedLenByteArrayStatistics) Update(values
[]parquet.FixedLenByteArray,
// and skip null values.
func (s *FixedLenByteArrayStatistics) UpdateSpaced(values
[]parquet.FixedLenByteArray, validBits []byte, validBitsOffset, numNull int64) {
s.IncNulls(numNull)
- notnull := int64(len(values)) - numNull
+ notnull := int64(bitutil.CountSetBits(validBits, int(validBitsOffset),
len(values)))
s.nvalues += notnull
if notnull == 0 {
@@ -2761,7 +2761,7 @@ func (s *Float16Statistics) Update(values
[]parquet.FixedLenByteArray, numNull i
// and skip null values.
func (s *Float16Statistics) UpdateSpaced(values []parquet.FixedLenByteArray,
validBits []byte, validBitsOffset, numNull int64) {
s.IncNulls(numNull)
- notnull := int64(len(values)) - numNull
+ notnull := int64(bitutil.CountSetBits(validBits, int(validBitsOffset),
len(values)))
s.nvalues += notnull
if notnull == 0 {
diff --git a/parquet/metadata/statistics_types.gen.go.tmpl
b/parquet/metadata/statistics_types.gen.go.tmpl
index 0938d4ed..eef1a050 100644
--- a/parquet/metadata/statistics_types.gen.go.tmpl
+++ b/parquet/metadata/statistics_types.gen.go.tmpl
@@ -20,6 +20,7 @@ import (
"fmt"
"github.com/apache/arrow-go/v18/arrow"
+ "github.com/apache/arrow-go/v18/arrow/bitutil"
"github.com/apache/arrow-go/v18/parquet"
"github.com/apache/arrow-go/v18/parquet/schema"
"github.com/apache/arrow-go/v18/parquet/internal/utils"
@@ -364,7 +365,7 @@ func (s *{{.Name}}Statistics) Update(values []{{.name}},
numNull int64) {
// and skip null values.
func (s *{{.Name}}Statistics) UpdateSpaced(values []{{.name}}, validBits
[]byte, validBitsOffset, numNull int64) {
s.IncNulls(numNull)
- notnull := int64(len(values)) - numNull
+ notnull := int64(bitutil.CountSetBits(validBits, int(validBitsOffset),
len(values)))
s.nvalues += notnull
if notnull == 0 {
diff --git a/parquet/pqarrow/encode_arrow_test.go
b/parquet/pqarrow/encode_arrow_test.go
index 7f49aeff..ab8d0797 100644
--- a/parquet/pqarrow/encode_arrow_test.go
+++ b/parquet/pqarrow/encode_arrow_test.go
@@ -43,6 +43,7 @@ import (
"github.com/apache/arrow-go/v18/parquet/file"
"github.com/apache/arrow-go/v18/parquet/internal/encoding"
"github.com/apache/arrow-go/v18/parquet/internal/testutils"
+ "github.com/apache/arrow-go/v18/parquet/metadata"
"github.com/apache/arrow-go/v18/parquet/pqarrow"
"github.com/apache/arrow-go/v18/parquet/schema"
"github.com/google/uuid"
@@ -3082,3 +3083,56 @@ func
TestByteArrayStatisticsStreamingReleaseBetweenBatches(t *testing.T) {
assert.Equal(t, "aaa", string(stats.EncodeMin()), "min corrupted by
released batch buffer")
assert.Equal(t, "zzz", string(stats.EncodeMax()), "max corrupted by
released batch buffer")
}
+
+func TestWriteArrowNestedNullListStatistics(t *testing.T) {
+ tests := []struct {
+ name string
+ values []int64
+ hasMinMax bool
+ }{
+ {name: "null-list-only"},
+ {name: "null-list-and-value", values: []int64{5}, hasMinMax:
true},
+ }
+ for _, pageVersion := range
[]parquet.DataPageVersion{parquet.DataPageV1, parquet.DataPageV2} {
+ for _, tt := range tests {
+ t.Run(fmt.Sprintf("page-%d/%s", pageVersion, tt.name),
func(t *testing.T) {
+ builder :=
array.NewListBuilder(memory.DefaultAllocator, &arrow.TimestampType{Unit:
arrow.Millisecond})
+ defer builder.Release()
+ values :=
builder.ValueBuilder().(*array.TimestampBuilder)
+ builder.AppendNull()
+ if len(tt.values) > 0 {
+ builder.Append(true)
+ for _, v := range tt.values {
+
values.Append(arrow.Timestamp(v))
+ }
+ }
+ arr := builder.NewListArray()
+ defer arr.Release()
+ field := arrow.Field{Name: "events", Type:
arr.DataType(), Nullable: true}
+ column := arrow.NewColumnFromArr(field, arr)
+ defer column.Release()
+ tbl :=
array.NewTable(arrow.NewSchema([]arrow.Field{field}, nil),
[]arrow.Column{column}, int64(arr.Len()))
+ defer tbl.Release()
+
+ props :=
parquet.NewWriterProperties(parquet.WithStats(true),
parquet.WithDataPageVersion(pageVersion))
+ data := writeParquetTable(t, tbl,
tbl.NumRows(), props)
+ reader, err :=
file.NewParquetReader(bytes.NewReader(data))
+ require.NoError(t, err)
+ defer reader.Close()
+ chunk, err :=
reader.MetaData().RowGroup(0).ColumnChunk(0)
+ require.NoError(t, err)
+ stats, err := chunk.Statistics()
+ require.NoError(t, err)
+
+ require.Equal(t, int64(1), stats.NullCount())
+ require.Equal(t, int64(len(tt.values)),
stats.NumValues())
+ require.Equal(t, tt.hasMinMax,
stats.HasMinMax())
+ if tt.hasMinMax {
+ typed :=
stats.(*metadata.Int64Statistics)
+ require.Equal(t, int64(5), typed.Min())
+ require.Equal(t, int64(5), typed.Max())
+ }
+ })
+ }
+ }
+}
