This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new dd58761f fix(parquet): serialize column chunk file offsets (#1073)
dd58761f is described below
commit dd58761f7e0fcf9185b06087fc5229dc9b457f3b
Author: Minh Vu <[email protected]>
AuthorDate: Wed Aug 5 20:33:59 2026 +0200
fix(parquet): serialize column chunk file offsets (#1073)
Fixes #891
## Problem
`ColumnChunkMetaDataBuilder.Finish` computed the first page offset for
internal row-group metadata but did not copy it to the serialized Thrift
`ColumnChunk.FileOffset` field. Serialized metadata therefore reported
the default zero value even when the chunk started elsewhere.
## Change
Populate `ColumnChunk.FileOffset` from the computed first page offset:
- dictionary page offset when a dictionary page is present
- data page offset otherwise
## Coverage
The metadata round-trip test verifies dictionary and non-dictionary
column chunks before and after serialization.
## Validation
`go test ./parquet/metadata`
---
parquet/metadata/column_chunk.go | 1 +
parquet/metadata/metadata_test.go | 3 +++
parquet/pqarrow/file_writer_test.go | 4 ++--
3 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/parquet/metadata/column_chunk.go b/parquet/metadata/column_chunk.go
index b387a442..1ccd801b 100644
--- a/parquet/metadata/column_chunk.go
+++ b/parquet/metadata/column_chunk.go
@@ -372,6 +372,7 @@ func (c *ColumnChunkMetaDataBuilder) Finish(info
ChunkMetaInfo, hasDict, dictFal
} else {
c.fileOffset = info.DataPageOffset
}
+ c.chunk.FileOffset = c.fileOffset
c.chunk.MetaData.NumValues = info.NumValues
if info.IndexPageOffset >= 0 {
diff --git a/parquet/metadata/metadata_test.go
b/parquet/metadata/metadata_test.go
index 96b11819..dc84674a 100644
--- a/parquet/metadata/metadata_test.go
+++ b/parquet/metadata/metadata_test.go
@@ -168,9 +168,11 @@ func TestBuildAccess(t *testing.T) {
rg1Col1, err := rg1Access.ColumnChunk(0)
assert.NoError(t, err)
assert.Equal(t, rg1Access.FileOffset(),
rg1Col1.DictionaryPageOffset())
+ assert.Equal(t, rg1Col1.DictionaryPageOffset(),
rg1Col1.FileOffset())
rg1Col2, err := rg1Access.ColumnChunk(1)
assert.NoError(t, err)
+ assert.Equal(t, rg1Col2.DictionaryPageOffset(),
rg1Col2.FileOffset())
assertStatsSet(t, rg1Col1)
assertStatsSet(t, rg1Col2)
assert.Equal(t, statsInt.Min, assertStats(t,
rg1Col1).EncodeMin())
@@ -208,6 +210,7 @@ func TestBuildAccess(t *testing.T) {
rg2Col1, err := rg2Access.ColumnChunk(0)
assert.NoError(t, err)
assert.Equal(t, rg2Access.FileOffset(),
rg2Col1.DataPageOffset())
+ assert.Equal(t, rg2Col1.DataPageOffset(), rg2Col1.FileOffset())
rg2Col2, err := rg2Access.ColumnChunk(1)
assert.NoError(t, err)
diff --git a/parquet/pqarrow/file_writer_test.go
b/parquet/pqarrow/file_writer_test.go
index 7ba4b85b..b3672b0d 100644
--- a/parquet/pqarrow/file_writer_test.go
+++ b/parquet/pqarrow/file_writer_test.go
@@ -172,7 +172,7 @@ func TestFileWriterTotalBytes(t *testing.T) {
// Verify total bytes & compressed bytes are correct
assert.Equal(t, int64(332), writer.TotalCompressedBytes())
- assert.Equal(t, int64(783), writer.TotalBytesWritten())
+ assert.Equal(t, int64(786), writer.TotalBytesWritten())
}
func TestFileWriterTotalBytesBuffered(t *testing.T) {
@@ -206,7 +206,7 @@ func TestFileWriterTotalBytesBuffered(t *testing.T) {
// Verify total bytes & compressed bytes are correct
assert.Equal(t, int64(482), writer.TotalCompressedBytes())
- assert.Equal(t, int64(1115), writer.TotalBytesWritten())
+ assert.Equal(t, int64(1120), writer.TotalBytesWritten())
}
func TestWriteOnClosedFileWriter(t *testing.T) {
