(iceberg-go) branch main updated: fix: Clickhouse does not support "null" as partition spec metadata (#347)

[zeroshade]

This is an automated email from the ASF dual-hosted git repository.

zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git


The following commit(s) were added to refs/heads/main by this push:
     new 5142432  fix: Clickhouse does not support "null" as partition spec 
metadata (#347)
5142432 is described below

commit 5142432b98b90b33c3f6af417de4d40aa3be8946
Author: Arnaud Briche <briche.arn...@gmail.com>
AuthorDate: Thu Mar 20 17:50:54 2025 +0100

    fix: Clickhouse does not support "null" as partition spec metadata (#347)
    
    Hi,
    
    I'm working on [a tool that leverage this
    package](https://github.com/agnosticeng/icepq) to support adding /
    removing / merging Parquet files from ClickHouse with an UDF.
    
    After upgrading this package (and removing a lot of code thx to your
    last commits) I got errors from ClickHouse (again) at query time.
    
    I nailed the issue down to the "partition-spec" metadata of the manifest
    file.
    When there is no field in the partition spec, the current code write a
    "null" JSON document that cause errors with ClickHouse.
    I just replaced "null" with "[]" in this case and things are working ok
    again.
    
    Fixes #344
---
 manifest.go      |  8 +++++++-
 manifest_test.go | 11 +++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/manifest.go b/manifest.go
index 8e2670f..ed921c0 100644
--- a/manifest.go
+++ b/manifest.go
@@ -951,7 +951,13 @@ func (w *ManifestWriter) meta() (map[string][]byte, error) 
{
                return nil, err
        }
 
-       specFieldsJson, err := json.Marshal(w.spec.fields)
+       specFields := w.spec.fields
+
+       if specFields == nil {
+               specFields = []PartitionField{}
+       }
+
+       specFieldsJson, err := json.Marshal(specFields)
        if err != nil {
                return nil, err
        }
diff --git a/manifest_test.go b/manifest_test.go
index d1f8cb6..ccdfb84 100644
--- a/manifest_test.go
+++ b/manifest_test.go
@@ -19,6 +19,7 @@ package iceberg
 
 import (
        "bytes"
+       "io"
        "testing"
        "time"
 
@@ -867,6 +868,16 @@ func (m *ManifestTestSuite) TestManifestEntryBuilder() {
        m.Assert().Equal(0, *data.SortOrderID())
 }
 
+func (m *ManifestTestSuite) TestManifestWriterMeta() {
+       sch := NewSchema(0, NestedField{ID: 0, Name: "test01", Type: 
StringType{}})
+       w, err := NewManifestWriter(2, io.Discard, *UnpartitionedSpec, sch, 1)
+       m.Require().NoError(err)
+       md, err := w.meta()
+       m.Require().NoError(err)
+       m.NotEqual("null", string(md["partition-spec"]))
+       m.Equal("[]", string(md["partition-spec"]))
+}
+
 func TestManifests(t *testing.T) {
        suite.Run(t, new(ManifestTestSuite))
 }