This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new eecb4d28 feat(arrow): map unknown type to null (#1035)
eecb4d28 is described below
commit eecb4d28411d96d2a8ca03f595fac7451ed04e8b
Author: TT <[email protected]>
AuthorDate: Thu May 7 19:17:44 2026 +0200
feat(arrow): map unknown type to null (#1035)
Closes #1003.
Maps Iceberg unknown fields to plain Arrow null fields, and maps Arrow
null fields back to Iceberg unknown.
Checked with `go test ./table -run TestArrow`.
---
table/arrow_utils.go | 6 +++---
table/arrow_utils_test.go | 1 +
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/table/arrow_utils.go b/table/arrow_utils.go
index ad7c3631..583ef254 100644
--- a/table/arrow_utils.go
+++ b/table/arrow_utils.go
@@ -315,6 +315,8 @@ func (c convertToIceberg) Primitive(dt arrow.DataType)
(result iceberg.NestedFie
return c.Primitive(dt.Encoded())
case *arrow.BooleanType:
result.Type = iceberg.PrimitiveTypes.Bool
+ case *arrow.NullType:
+ result.Type = iceberg.PrimitiveTypes.Unknown
case *arrow.Uint8Type, *arrow.Uint16Type, *arrow.Uint32Type,
*arrow.Int8Type, *arrow.Int16Type, *arrow.Int32Type:
result.Type = iceberg.PrimitiveTypes.Int32
@@ -625,9 +627,7 @@ func (c convertToArrow) VisitUUID() arrow.Field {
}
func (c convertToArrow) VisitUnknown() arrow.Field {
- return arrow.Field{
- Type: extensions.NewOpaqueType(arrow.Null, "unknown",
"apache.iceberg"),
- }
+ return arrow.Field{Type: arrow.Null}
}
var _ iceberg.SchemaVisitorPerPrimitiveType[arrow.Field] = convertToArrow{}
diff --git a/table/arrow_utils_test.go b/table/arrow_utils_test.go
index 5e3aea2e..e3f23718 100644
--- a/table/arrow_utils_test.go
+++ b/table/arrow_utils_test.go
@@ -55,6 +55,7 @@ func TestArrowToIceberg(t *testing.T) {
{&arrow.Decimal128Type{Precision: 26, Scale: 20},
iceberg.DecimalTypeOf(26, 20), true, ""},
{&arrow.Decimal256Type{Precision: 8, Scale: 9}, nil, false,
"unsupported arrow type for conversion - decimal256(8, 9)"},
{arrow.FixedWidthTypes.Boolean, iceberg.PrimitiveTypes.Bool,
true, ""},
+ {arrow.Null, iceberg.PrimitiveTypes.Unknown, true, ""},
{arrow.PrimitiveTypes.Int8, iceberg.PrimitiveTypes.Int32,
false, ""},
{arrow.PrimitiveTypes.Uint8, iceberg.PrimitiveTypes.Int32,
false, ""},
{arrow.PrimitiveTypes.Int16, iceberg.PrimitiveTypes.Int32,
false, ""},
