This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 0cd57944 feat(parquet/variant): add Metadata.SizeBytes to split a
concatenated variant (#939)
0cd57944 is described below
commit 0cd579447b8c1a56e67e9ae1bd766d5c84e20c04
Author: Neelesh Salian <[email protected]>
AuthorDate: Wed Jul 15 08:17:22 2026 -0700
feat(parquet/variant): add Metadata.SizeBytes to split a concatenated
variant (#939)
### Rationale for this change
A variant bound is stored on disk as the variant metadata bytes
concatenated with the value bytes (metadata ++ value), as Iceberg does
for variant `lower_bound` / `upper_bound`. `Metadata.Bytes()` returns
the backing slice as-is, so a caller that reads such a concatenated
buffer has no way to know where the metadata ends and the
value begins. `NewMetadata` accepts a buffer with trailing bytes but
does not expose the metadata's own length.
iceberg-go needs this to read variant bounds back for scan pruning
(apache/iceberg-go#1016): it must split a stored bound into metadata and
value to reconstruct the variant. Filing it now so it is released ahead
of that work.
### What changes are included in this PR?
- Adds `Metadata.SizeBytes()`, which computes the metadata's own byte
length from its header and offset table (the final offset gives the
string-region length past `valuesStart`). A caller can then split a
buffer at `data[m.SizeBytes():]`.
### Are these changes tested?
Yes. `TestMetadataSizeBytes` covers empty metadata, a keyed object, and
splitting a concatenated metadata++value buffer with a round-trip
through `New`.`TestMetadataSizeBytesMultiByteOffset` covers the
multi-byte offset-width path.
### Are there any user-facing changes?
One new exported method. No change to existing behavior.
---
parquet/variant/variant.go | 19 ++++++++++++
parquet/variant/variant_test.go | 66 +++++++++++++++++++++++++++++++++++++++++
2 files changed, 85 insertions(+)
diff --git a/parquet/variant/variant.go b/parquet/variant/variant.go
index 1fe44cc5..0e8a904b 100644
--- a/parquet/variant/variant.go
+++ b/parquet/variant/variant.go
@@ -231,6 +231,25 @@ func (m Metadata) OffsetSize() uint8 {
// DictionarySize returns the number of keys in the metadata dictionary.
func (m Metadata) DictionarySize() uint32 { return uint32(len(m.keys)) }
+// SizeBytes returns the metadata's own byte length, so a caller can split a
+// buffer that stores metadata concatenated with a value at
data[m.SizeBytes():].
+func (m Metadata) SizeBytes() int {
+ offsetSz := uint32(m.OffsetSize())
+ if uint32(len(m.data)) < uint32(hdrSizeBytes)+offsetSz {
+ return len(m.data)
+ }
+
+ dictSize := readLEU32(m.data[hdrSizeBytes :
uint32(hdrSizeBytes)+offsetSz])
+ // Final offset (index dictSize) is the string-region length; it starts
at valuesStart.
+ lastOffsetPos := uint32(hdrSizeBytes) + offsetSz*(1+dictSize)
+ valuesStart := lastOffsetPos + offsetSz
+ if valuesStart > uint32(len(m.data)) {
+ return len(m.data)
+ }
+
+ return int(valuesStart + readLEU32(m.data[lastOffsetPos:valuesStart]))
+}
+
// KeyAt returns the string key at the given dictionary ID.
// Returns an error if the ID is out of range.
func (m Metadata) KeyAt(id uint32) (string, error) {
diff --git a/parquet/variant/variant_test.go b/parquet/variant/variant_test.go
index 5b76fd3a..b2f55339 100644
--- a/parquet/variant/variant_test.go
+++ b/parquet/variant/variant_test.go
@@ -22,6 +22,7 @@ import (
"math"
"os"
"path/filepath"
+ "strings"
"testing"
"time"
@@ -833,3 +834,68 @@ func TestObjectBinarySearch(t *testing.T) {
assert.Equal(t, int8(i), field.Value.Value())
}
}
+
+func TestMetadataSizeBytes(t *testing.T) {
+ t.Run("empty metadata", func(t *testing.T) {
+ var b variant.Builder
+ require.NoError(t, b.AppendNull())
+ v, err := b.Build()
+ require.NoError(t, err)
+ m := v.Metadata()
+ assert.Equal(t, len(m.Bytes()), m.SizeBytes())
+ assert.Equal(t, len(variant.EmptyMetadataBytes), m.SizeBytes())
+ })
+
+ t.Run("object with keys", func(t *testing.T) {
+ var b variant.Builder
+ start := b.Offset()
+ fields := []variant.FieldEntry{
+ b.NextField(start, "a"),
+ }
+ require.NoError(t, b.AppendInt(1))
+ fields = append(fields, b.NextField(start, "longer_key"))
+ require.NoError(t, b.AppendString("hello"))
+ require.NoError(t, b.FinishObject(start, fields))
+ v, err := b.Build()
+ require.NoError(t, err)
+
+ m := v.Metadata()
+ assert.Equal(t, len(m.Bytes()), m.SizeBytes())
+
+ // Split a concatenated metadata++value buffer using SizeBytes.
+ concat := append(append([]byte{}, m.Bytes()...), v.Bytes()...)
+ parsed, err := variant.NewMetadata(concat)
+ require.NoError(t, err)
+ split := parsed.SizeBytes()
+ assert.Equal(t, m.Bytes(), concat[:split])
+ assert.Equal(t, v.Bytes(), concat[split:])
+
+ rt, err := variant.New(concat[:split], concat[split:])
+ require.NoError(t, err)
+ assert.Equal(t, v.Bytes(), rt.Bytes())
+ })
+}
+
+func TestMetadataSizeBytesMultiByteOffset(t *testing.T) {
+ // A long key forces the metadata offset width above 1 byte, exercising
the
+ // offsetSize>1 arithmetic in SizeBytes.
+ longKey := strings.Repeat("k", 300)
+ var b variant.Builder
+ start := b.Offset()
+ fields := []variant.FieldEntry{b.NextField(start, longKey)}
+ require.NoError(t, b.AppendInt(7))
+ require.NoError(t, b.FinishObject(start, fields))
+ v, err := b.Build()
+ require.NoError(t, err)
+
+ m := v.Metadata()
+ require.Greater(t, m.OffsetSize(), uint8(1), "test must exercise
multi-byte offsets")
+ assert.Equal(t, len(m.Bytes()), m.SizeBytes())
+
+ concat := append(append([]byte{}, m.Bytes()...), v.Bytes()...)
+ parsed, err := variant.NewMetadata(concat)
+ require.NoError(t, err)
+ split := parsed.SizeBytes()
+ assert.Equal(t, m.Bytes(), concat[:split])
+ assert.Equal(t, v.Bytes(), concat[split:])
+}
