This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 84e4e99d fix(arrow/array): preserve run-end encoded empty values
(#1156)
84e4e99d is described below
commit 84e4e99da0adafd34cf5dbd38f438dcfc63c9f97
Author: Minh Vu <[email protected]>
AuthorDate: Tue Aug 25 17:39:56 2026 +0200
fix(arrow/array): preserve run-end encoded empty values (#1156)
## What
Make RunEndEncodedBuilder.AppendEmptyValue and AppendEmptyValues append
the value builder empty value instead of a null.
## Test
- go test ./arrow/array -run ^TestRunEndEncoded -count=1
---
arrow/array/dictionary.go | 22 +++++++++--
arrow/array/dictionary_test.go | 29 ++++++++++++++
arrow/array/encoded.go | 19 ++++++---
arrow/array/encoded_test.go | 89 ++++++++++++++++++++++++++++++++++++++++++
4 files changed, 149 insertions(+), 10 deletions(-)
diff --git a/arrow/array/dictionary.go b/arrow/array/dictionary.go
index 14c234d2..a03fa457 100644
--- a/arrow/array/dictionary.go
+++ b/arrow/array/dictionary.go
@@ -668,13 +668,27 @@ func (b *dictionaryBuilder) AppendNulls(n int) {
}
func (b *dictionaryBuilder) AppendEmptyValue() {
- b.length += 1
- b.idxBuilder.AppendEmptyValue()
+ b.AppendEmptyValues(1)
}
func (b *dictionaryBuilder) AppendEmptyValues(n int) {
- for i := 0; i < n; i++ {
- b.AppendEmptyValue()
+ if n <= 0 {
+ return
+ }
+
+ if b.dt.ValueType.ID() == arrow.NULL {
+ b.AppendNulls(n)
+ return
+ }
+
+ valueBuilder := NewBuilder(b.mem, b.dt.ValueType)
+ defer valueBuilder.Release()
+ valueBuilder.AppendEmptyValues(n)
+
+ values := valueBuilder.NewArray()
+ defer values.Release()
+ if err := b.AppendArray(values); err != nil {
+ panic(err)
}
}
diff --git a/arrow/array/dictionary_test.go b/arrow/array/dictionary_test.go
index 90f3c0bd..2324b743 100644
--- a/arrow/array/dictionary_test.go
+++ b/arrow/array/dictionary_test.go
@@ -564,6 +564,35 @@ func TestStringDictionaryBuilderInit(t *testing.T) {
assert.True(t, array.Equal(expected, result))
}
+func TestStringDictionaryBuilderAppendEmptyValue(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ dictArr, _, err := array.FromJSON(mem, arrow.BinaryTypes.String,
strings.NewReader(`["existing"]`))
+ require.NoError(t, err)
+ defer dictArr.Release()
+
+ dictType := &arrow.DictionaryType{IndexType: &arrow.Int8Type{},
ValueType: arrow.BinaryTypes.String}
+ bldr := array.NewDictionaryBuilderWithDict(mem, dictType, dictArr)
+ defer bldr.Release()
+
+ bldr.AppendEmptyValue()
+ bldr.AppendEmptyValues(2)
+
+ result := bldr.NewDictionaryArray()
+ defer result.Release()
+
+ dict := result.Dictionary().(*array.String)
+ assert.Equal(t, 2, dict.Len())
+ assert.Equal(t, "existing", dict.Value(0))
+ assert.Empty(t, dict.Value(1))
+ for i := 0; i < result.Len(); i++ {
+ assert.False(t, result.IsNull(i))
+ assert.Equal(t, 1, result.GetValueIndex(i))
+ assert.Empty(t, result.GetOneForMarshal(i))
+ }
+}
+
func TestStringDictionaryBuilderOnlyNull(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
defer mem.AssertSize(t, 0)
diff --git a/arrow/array/encoded.go b/arrow/array/encoded.go
index 219a75cb..bd9a6fb5 100644
--- a/arrow/array/encoded.go
+++ b/arrow/array/encoded.go
@@ -392,9 +392,10 @@ type RunEndEncodedBuilder struct {
maxRunEnd uint64
// currently, mixing AppendValueFromString & UnmarshalOne is unsupported
- lastUnmarshalled interface{}
- unmarshalled bool // tracks if Unmarshal was called (in case
lastUnmarshalled is nil)
- lastStr *string
+ lastUnmarshalled interface{}
+ unmarshalled bool // tracks if Unmarshal was called (in case
lastUnmarshalled is nil)
+ lastValueWasEmpty bool
+ lastStr *string
}
func NewRunEndEncodedBuilder(mem memory.Allocator, runEnds, encoded
arrow.DataType) *RunEndEncodedBuilder {
@@ -449,6 +450,7 @@ func (b *RunEndEncodedBuilder) finishRun() {
b.lastUnmarshalled = nil
b.lastStr = nil
b.unmarshalled = false
+ b.lastValueWasEmpty = false
if b.length == 0 {
return
}
@@ -502,11 +504,16 @@ func (b *RunEndEncodedBuilder) NullN() int {
}
func (b *RunEndEncodedBuilder) AppendEmptyValue() {
- b.AppendNull()
+ b.finishRun()
+ b.values.AppendEmptyValue()
+ b.addLength(1)
+ b.lastValueWasEmpty = true
}
func (b *RunEndEncodedBuilder) AppendEmptyValues(n int) {
- b.AppendNulls(n)
+ for i := 0; i < n; i++ {
+ b.AppendEmptyValue()
+ }
}
func (b *RunEndEncodedBuilder) Reserve(n int) {
@@ -585,7 +592,7 @@ func (b *RunEndEncodedBuilder) UnmarshalOne(dec
*json.Decoder) error {
// make sure we add a new run instead. We can detect that case by
// checking that the number of runEnds matches the number of values
// we have, which means no matter what we have to start a new run
- if reflect.DeepEqual(value, b.lastUnmarshalled) && (value != nil ||
b.runEnds.Len() != b.values.Len()) {
+ if !b.lastValueWasEmpty && reflect.DeepEqual(value, b.lastUnmarshalled)
&& (value != nil || b.runEnds.Len() != b.values.Len()) {
b.ContinueRun(1)
return nil
}
diff --git a/arrow/array/encoded_test.go b/arrow/array/encoded_test.go
index 7269f4fd..512adf85 100644
--- a/arrow/array/encoded_test.go
+++ b/arrow/array/encoded_test.go
@@ -367,6 +367,95 @@ func TestRunEndEncodedBuilder(t *testing.T) {
assert.Equal(t, "Hello", strValues.ValueStr(0))
}
+func TestRunEndEncodedBuilderEmptyValues(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ bldr := array.NewRunEndEncodedBuilder(mem, arrow.PrimitiveTypes.Int16,
arrow.BinaryTypes.String)
+ defer bldr.Release()
+
+ bldr.AppendEmptyValue()
+ bldr.AppendEmptyValues(2)
+
+ arr := bldr.NewRunEndEncodedArray()
+ defer arr.Release()
+
+ values := arr.Values().(*array.String)
+ assert.Equal(t, 3, values.Len())
+ for i := 0; i < values.Len(); i++ {
+ assert.False(t, values.IsNull(i))
+ assert.Empty(t, values.Value(i))
+ }
+}
+
+func TestRunEndEncodedBuilderEmptyValueBeforeUnmarshalNull(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ bldr := array.NewRunEndEncodedBuilder(mem, arrow.PrimitiveTypes.Int16,
arrow.BinaryTypes.String)
+ defer bldr.Release()
+
+ bldr.AppendEmptyValue()
+ dec := json.NewDecoder(strings.NewReader("null"))
+ require.NoError(t, bldr.UnmarshalOne(dec))
+
+ arr := bldr.NewRunEndEncodedArray()
+ defer arr.Release()
+
+ assert.Equal(t, []int16{1, 2},
arr.RunEndsArr().(*array.Int16).Int16Values())
+ values := arr.Values().(*array.String)
+ assert.False(t, values.IsNull(0))
+ assert.Empty(t, values.Value(0))
+ assert.True(t, values.IsNull(1))
+}
+
+func TestRunEndEncodedBuilderEmptyValuesBeforeUnmarshalNulls(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ bldr := array.NewRunEndEncodedBuilder(mem, arrow.PrimitiveTypes.Int16,
arrow.BinaryTypes.String)
+ defer bldr.Release()
+
+ bldr.AppendEmptyValues(2)
+ dec := json.NewDecoder(strings.NewReader("null null"))
+ require.NoError(t, bldr.UnmarshalOne(dec))
+ require.NoError(t, bldr.UnmarshalOne(dec))
+
+ arr := bldr.NewRunEndEncodedArray()
+ defer arr.Release()
+
+ assert.Equal(t, []int16{1, 2, 4},
arr.RunEndsArr().(*array.Int16).Int16Values())
+ values := arr.Values().(*array.String)
+ assert.Equal(t, 3, values.Len())
+ assert.False(t, values.IsNull(0))
+ assert.Empty(t, values.Value(0))
+ assert.False(t, values.IsNull(1))
+ assert.Empty(t, values.Value(1))
+ assert.True(t, values.IsNull(2))
+}
+
+func TestRunEndEncodedBuilderDictionaryEmptyValue(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ encoded := &arrow.DictionaryType{
+ IndexType: arrow.PrimitiveTypes.Int8,
+ ValueType: arrow.BinaryTypes.String,
+ }
+ bldr := array.NewRunEndEncodedBuilder(mem, arrow.PrimitiveTypes.Int16,
encoded)
+ defer bldr.Release()
+
+ bldr.AppendEmptyValue()
+
+ arr := bldr.NewRunEndEncodedArray()
+ defer arr.Release()
+
+ values := arr.Values().(*array.Dictionary)
+ assert.Equal(t, 1, values.Dictionary().Len())
+ assert.Equal(t, 0, values.GetValueIndex(0))
+ assert.Equal(t, "", arr.GetOneForMarshal(0))
+}
+
func TestRunEndEncodedStringRoundTrip(t *testing.T) {
// 1. create array
mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
