This is an automated email from the ASF dual-hosted git repository.
laskoviymishka pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new 6bb3818ea fix(table): default absent geoarrow CRS metadata to
OGC:CRS84 (#1634)
6bb3818ea is described below
commit 6bb3818ea9d9a00f7e53fc87f84bc781ad974b3a
Author: Tobias Pütz <[email protected]>
AuthorDate: Wed Aug 5 10:26:44 2026 +0200
fix(table): default absent geoarrow CRS metadata to OGC:CRS84 (#1634)
* fix(table): default absent geoarrow CRS metadata to OGC:CRS84
The Parquet geospatial spec makes OGC:CRS84 the default when no CRS is
given, but absent GeoArrow CRS metadata was mapped to a fabricated srid:0,
so a spec-legal file mismatched the table's default geometry/geography
type and failed projection with "cannot promote geometry(srid:0) to
geometry".
Map an absent CRS to the default type, and symmetrically write srid:0 as
an explicit SRID so unknown-CRS columns round-trip instead of being
silently relabelled as the default.
* address review
---
table/arrow_utils.go | 24 +-
table/arrow_utils_internal_test.go | 46 +++-
table/arrow_utils_test.go | 486 ++++++++++++++++++++++++++++++++++---
types.go | 17 +-
4 files changed, 522 insertions(+), 51 deletions(-)
diff --git a/table/arrow_utils.go b/table/arrow_utils.go
index 98ec85989..8cd55a2f6 100644
--- a/table/arrow_utils.go
+++ b/table/arrow_utils.go
@@ -2183,9 +2183,12 @@ func isWKT2CRSString(crs string) bool {
return false
}
+// geoArrowCRSToIcebergCRS maps GeoArrow CRS metadata to an Iceberg CRS string.
+// Absent CRS metadata means the default CRS OGC:CRS84, matching the Parquet
+// geospatial spec and Iceberg's default geometry/geography types.
func geoArrowCRSToIcebergCRS(meta geoarrow.Metadata) (string, error) {
if len(meta.CRS) == 0 {
- return "srid:0", nil
+ return iceberg.DefaultGeoCRS, nil
}
switch {
@@ -2199,8 +2202,8 @@ func geoArrowCRSToIcebergCRS(meta geoarrow.Metadata)
(string, error) {
return "", errors.New("unsupported CRS: empty string
CRS")
}
- if strings.EqualFold(crs, "OGC:CRS84") ||
strings.EqualFold(crs, "EPSG:4326") {
- return "OGC:CRS84", nil
+ if strings.EqualFold(crs, iceberg.DefaultGeoCRS) ||
strings.EqualFold(crs, "EPSG:4326") {
+ return iceberg.DefaultGeoCRS, nil
}
switch meta.CRSType {
@@ -2217,6 +2220,11 @@ func geoArrowCRSToIcebergCRS(meta geoarrow.Metadata)
(string, error) {
return "", errWKT2CRSNotSupported
}
if meta.CRSType == geoarrow.CRSTypeSRID {
+ // Some writers store the identifier already prefixed;
don't prefix twice.
+ if strings.HasPrefix(strings.ToLower(crs), "srid:") {
+ return crs, nil
+ }
+
return "srid:" + crs, nil
}
@@ -2274,8 +2282,8 @@ func geoArrowCRSToIcebergCRS(meta geoarrow.Metadata)
(string, error) {
}
authorityCode := authority + ":" + code
- if strings.EqualFold(authorityCode, "OGC:CRS84") ||
strings.EqualFold(authorityCode, "EPSG:4326") {
- return "OGC:CRS84", nil
+ if strings.EqualFold(authorityCode, iceberg.DefaultGeoCRS) ||
strings.EqualFold(authorityCode, "EPSG:4326") {
+ return iceberg.DefaultGeoCRS, nil
}
return authorityCode, nil
@@ -2340,10 +2348,6 @@ func icebergCRSToGeoArrowMetadata(crs string, props
iceberg.Properties) (geoarro
if strings.HasPrefix(lowerCRS, "srid:") {
id := crs[len("srid:"):]
- if id == "0" {
- return geoarrow.NewMetadata(), nil // srid:0 maps to
omitted GeoArrow CRS
- }
-
raw, _ := json.Marshal(id) //nolint:errcheck // Marshalling a
string can't fail
return geoarrow.Metadata{
@@ -2382,7 +2386,7 @@ func icebergCRSToGeoArrowMetadata(crs string, props
iceberg.Properties) (geoarro
if lowerCRS == "epsg:4326" {
// collapse EPSG:4326 to OGC:CRS84
- raw, _ = json.Marshal("OGC:CRS84") //nolint:errcheck //
Marshalling a string can't fail
+ raw, _ = json.Marshal(iceberg.DefaultGeoCRS) //nolint:errcheck
// Marshalling a string can't fail
} else {
raw, _ = json.Marshal(crs) //nolint:errcheck // Marshalling a
string can't fail
}
diff --git a/table/arrow_utils_internal_test.go
b/table/arrow_utils_internal_test.go
index 809155794..cb949bbdb 100644
--- a/table/arrow_utils_internal_test.go
+++ b/table/arrow_utils_internal_test.go
@@ -482,6 +482,7 @@ func TestIcebergCRSToGeoArrowMetadata(t *testing.T) {
wantCRSType geoarrow.CRSType
}{
{"srid:4326", geoarrow.CRSTypeSRID},
+ {"srid:0", geoarrow.CRSTypeSRID},
{"OGC:CRS84", geoarrow.CRSTypeAuthorityCode},
{"EPSG:4326", geoarrow.CRSTypeAuthorityCode},
{"EPSG:4267", geoarrow.CRSTypeAuthorityCode},
@@ -495,11 +496,16 @@ func TestIcebergCRSToGeoArrowMetadata(t *testing.T) {
}
})
- t.Run("srid:0 maps to an omitted CRS", func(t *testing.T) {
+ // An omitted CRS means OGC:CRS84, so the unknown CRS srid:0 must be
explicit.
+ t.Run("srid:0 round trips as an explicit srid CRS", func(t *testing.T) {
meta, err := icebergCRSToGeoArrowMetadata("srid:0", nil)
require.NoError(t, err)
- assert.Empty(t, meta.CRS)
- assert.Empty(t, meta.CRSType)
+ assert.JSONEq(t, `"0"`, string(meta.CRS))
+ assert.Equal(t, geoarrow.CRSTypeSRID, meta.CRSType)
+
+ crs, err := geoArrowCRSToIcebergCRS(meta)
+ require.NoError(t, err)
+ assert.Equal(t, "srid:0", crs)
})
}
@@ -510,6 +516,27 @@ func TestGeoArrowCRSToIcebergCRS(t *testing.T) {
return raw
}
+ t.Run("absent CRS maps to the default CRS", func(t *testing.T) {
+ got, err := geoArrowCRSToIcebergCRS(geoarrow.Metadata{})
+ require.NoError(t, err)
+ assert.Equal(t, "OGC:CRS84", got)
+ })
+
+ t.Run("absent CRS with a crs_type annotation maps to the default CRS",
func(t *testing.T) {
+ for _, crsType := range []geoarrow.CRSType{
+ geoarrow.CRSTypeSRID,
+ geoarrow.CRSTypeAuthorityCode,
+ geoarrow.CRSTypePROJJSON,
+ geoarrow.CRSTypeWKT22019,
+ } {
+ t.Run(string(crsType), func(t *testing.T) {
+ got, err :=
geoArrowCRSToIcebergCRS(geoarrow.Metadata{CRSType: crsType})
+ require.NoError(t, err)
+ assert.Equal(t, "OGC:CRS84", got)
+ })
+ }
+ })
+
t.Run("accepts long authority code string", func(t *testing.T) {
const crs = "EPSGAuthorityLongName:CODE-12345678901234567890"
@@ -552,6 +579,19 @@ func TestGeoArrowCRSToIcebergCRS(t *testing.T) {
assert.Equal(t, "srid:3857", got)
})
+ t.Run("already prefixed srid is not prefixed twice", func(t *testing.T)
{
+ for _, crs := range []string{"srid:0", "srid:3857",
"SRID:3857"} {
+ t.Run(crs, func(t *testing.T) {
+ got, err :=
geoArrowCRSToIcebergCRS(geoarrow.Metadata{
+ CRS: stringCRS(crs),
+ CRSType: geoarrow.CRSTypeSRID,
+ })
+ require.NoError(t, err)
+ assert.Equal(t, crs, got)
+ })
+ }
+ })
+
t.Run("rejects empty string CRS", func(t *testing.T) {
_, err := geoArrowCRSToIcebergCRS(geoarrow.Metadata{CRS:
stringCRS("")})
require.Error(t, err)
diff --git a/table/arrow_utils_test.go b/table/arrow_utils_test.go
index 4b616ee35..a09fcdc73 100644
--- a/table/arrow_utils_test.go
+++ b/table/arrow_utils_test.go
@@ -19,6 +19,7 @@ package table_test
import (
"bufio"
+ "bytes"
"context"
"encoding/base64"
"encoding/json"
@@ -32,6 +33,10 @@ import (
"github.com/apache/arrow-go/v18/arrow/decimal128"
"github.com/apache/arrow-go/v18/arrow/extensions"
"github.com/apache/arrow-go/v18/arrow/memory"
+ "github.com/apache/arrow-go/v18/parquet"
+ "github.com/apache/arrow-go/v18/parquet/file"
+ "github.com/apache/arrow-go/v18/parquet/pqarrow"
+ "github.com/apache/arrow-go/v18/parquet/schema"
"github.com/apache/arrow-go/v18/parquet/variant"
"github.com/apache/iceberg-go"
"github.com/apache/iceberg-go/table"
@@ -522,12 +527,6 @@ func TestIcebergGeoTypesToArrowSchema(t *testing.T) {
ice: iceberg.GeometryType{},
geoarrowMetaJSON: `{"crs":"OGC:CRS84"}`,
},
- // Geometry with srid:0 should result in an unset (omitted) CRS
- {
- name: "geometry_srid_0",
- ice: geomSRID0,
- geoarrowMetaJSON: `{}`,
- },
// Geometry with custom CRSes (authority:code and partial
projjson)
{
name: "geometry_epsg_4267",
@@ -571,12 +570,6 @@ func TestIcebergGeoTypesToArrowSchema(t *testing.T) {
ice: geogThomas,
geoarrowMetaJSON:
`{"crs":"OGC:CRS84","edges":"thomas"}`,
},
- // Geography with srid:0 should result in an unset (omitted)
CRS and spherical edges
- {
- name: "geography_srid_0",
- ice: geogSRID0,
- geoarrowMetaJSON: `{"edges":"spherical"}`,
- },
// Geography with custom CRSes (authority:code and partial
projjson)
{
name: "geography_epsg_4267",
@@ -589,6 +582,18 @@ func TestIcebergGeoTypesToArrowSchema(t *testing.T) {
ice: geomSRID,
geoarrowMetaJSON: `{"crs":"4326", "crs_type":"srid"}`,
},
+ // srid:0 means an unknown CRS, which must be written
explicitly: an omitted
+ // CRS means OGC:CRS84 per the Parquet geospatial spec.
+ {
+ name: "geometry_srid_0",
+ ice: geomSRID0,
+ geoarrowMetaJSON: `{"crs":"0","crs_type":"srid"}`,
+ },
+ {
+ name: "geography_srid_0",
+ ice: geogSRID0,
+ geoarrowMetaJSON:
`{"crs":"0","crs_type":"srid","edges":"spherical"}`,
+ },
}
// The following tests focus on edge cases and pinning specific
behavior for read case
@@ -598,9 +603,9 @@ func TestIcebergGeoTypesToArrowSchema(t *testing.T) {
geoarrowMetaJSON string
}{
{
- // Pin that a present crs_type without CRS still
follows the srid:0 default.
- name: "geometry_srid_0_with_crs_type",
- ice: geomSRID0,
+ // Pin that a present crs_type without CRS still
follows the default CRS.
+ name: "geometry_no_crs_with_crs_type",
+ ice: defaultGeometry,
geoarrowMetaJSON: `{"crs_type":"authority_code"}`,
},
{
@@ -631,10 +636,10 @@ func TestIcebergGeoTypesToArrowSchema(t *testing.T) {
},
// Translated from arrow-rs geo logical type read tests
(https://github.com/apache/arrow-rs/pull/10065)
- // Geometry with no CRS should be GEOMETRY(srid:0)
+ // An omitted CRS means the default CRS, so this is plain
GEOMETRY
{
name: "geometry_no_crs",
- ice: geomSRID0,
+ ice: defaultGeometry,
geoarrowMetaJSON: `{}`,
},
// Geometry with string CRS
@@ -676,12 +681,30 @@ func TestIcebergGeoTypesToArrowSchema(t *testing.T) {
ice: iceberg.GeometryType{},
geoarrowMetaJSON:
`{"crs":{"id":{"authority":"OGC","code":"CRS84"}}}`,
},
- // Geography with no CRS, spherical edges
+ // An explicit srid CRS is preserved; only an absent CRS means
the default
+ {
+ name: "geometry_explicit_srid_0",
+ ice: geomSRID0,
+ geoarrowMetaJSON: `{"crs":"0","crs_type":"srid"}`,
+ },
+ // A writer that stores the srid already prefixed keeps a
single prefix
+ {
+ name: "geometry_prefixed_srid_0",
+ ice: geomSRID0,
+ geoarrowMetaJSON: `{"crs":"srid:0","crs_type":"srid"}`,
+ },
+ // Geography with no CRS, spherical edges: both are the Iceberg
defaults
{
name: "geography_no_crs_spherical",
- ice: geogSRID0,
+ ice: iceberg.GeographyType{},
geoarrowMetaJSON: `{"edges":"spherical"}`,
},
+ // Geography with no CRS keeps its explicit edge algorithm
+ {
+ name: "geography_no_crs_karney",
+ ice: geogKarneyDefaultCRS,
+ geoarrowMetaJSON: `{"edges":"karney"}`,
+ },
// Geography with OGC:CRS84 and spherical edges
{
name:
"geography_ogc_crs84_spherical_canonical",
@@ -2622,19 +2645,25 @@ func singleColumnRecord(field arrow.Field, arr
arrow.Array) arrow.RecordBatch {
return array.NewRecordBatch(sc, []arrow.Array{arr}, int64(arr.Len()))
}
-func TestToRequestedSchemaGeoExtensionRewrap(t *testing.T) {
- // POINT(1 2) and POINT(3 4) little-endian WKB.
- point12 := []byte{
- 0x01, 0x01, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x3f,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40,
- }
- point34 := []byte{
- 0x01, 0x01, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x40,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x40,
+// wkbPointRows returns POINT(1 2), NULL and POINT(3 4) as little-endian WKB.
+func wkbPointRows() [][]byte {
+ return [][]byte{
+ {
+ 0x01, 0x01, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x3f,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40,
+ },
+ nil,
+ {
+ 0x01, 0x01, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x40,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x40,
+ },
}
- values := [][]byte{point12, nil, point34}
+}
+
+func TestToRequestedSchemaGeoExtensionRewrap(t *testing.T) {
+ values := wkbPointRows()
geography, err := iceberg.GeographyTypeOf("OGC:CRS84", "spherical")
require.NoError(t, err)
@@ -2756,3 +2785,398 @@ func
TestToRequestedSchemaMismatchedExtensionNameNotRewrapped(t *testing.T) {
// silently rewrapped.
require.Error(t, err)
}
+
+// absentCRSWKBType returns the geoarrow.wkb type a reader reconstructs for a
+// column whose stored GeoArrow metadata carries no CRS. Omitted edges mean
+// planar, so the geography cases must state their edge algorithm.
+func absentCRSWKBType(edges geoarrow.EdgeInterpolation) arrow.ExtensionType {
+ return geoarrow.NewWKBType(geoarrow.WKBWithBinaryStorage(),
+ geoarrow.WKBWithMetadata(geoarrow.Metadata{Edges: edges}))
+}
+
+func TestArrowGeoAbsentCRSMetadataToIcebergDefaults(t *testing.T) {
+ tests := []struct {
+ name string
+ edges geoarrow.EdgeInterpolation
+ want iceberg.Type
+ wantType string
+ }{
+ {
+ name: "geometry_crs_omitted",
+ want: iceberg.GeometryType{},
+ wantType: "geometry",
+ },
+ {
+ name: "geography_crs_omitted",
+ edges: geoarrow.EdgeSpherical,
+ want: iceberg.GeographyType{},
+ wantType: "geography",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ dt := absentCRSWKBType(tt.edges)
+
+ sc, err :=
table.ArrowSchemaToIceberg(arrow.NewSchema([]arrow.Field{{
+ Name: "geom", Type: dt, Nullable: true,
Metadata: fieldIDMeta("1"),
+ }}, nil), false, nil)
+ require.NoError(t, err)
+
+ field, ok := sc.FindFieldByID(1)
+ require.True(t, ok)
+ assert.Equal(t, tt.wantType, field.Type.String())
+ assert.True(t, tt.want.Equals(field.Type), "expected
%s, got %s", tt.want, field.Type)
+ })
+ }
+}
+
+func TestToRequestedSchemaGeoAbsentCRSProjection(t *testing.T) {
+ values := wkbPointRows()
+
+ tests := []struct {
+ name string
+ edges geoarrow.EdgeInterpolation
+ icebergType iceberg.Type
+ }{
+ {
+ name: "geometry_crs_omitted",
+ icebergType: iceberg.GeometryType{},
+ },
+ {
+ name: "geography_crs_omitted",
+ edges: geoarrow.EdgeSpherical,
+ icebergType: iceberg.GeographyType{},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ mem :=
memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ sourceType := absentCRSWKBType(tt.edges)
+ source := newExtensionArrayOverBinary(t, mem,
sourceType, values)
+ defer source.Release()
+
+ rec := singleColumnRecord(arrow.Field{
+ Name: "geom", Type: sourceType, Nullable: true,
Metadata: fieldIDMeta("1"),
+ }, source)
+ defer rec.Release()
+
+ fileSchema, err :=
table.ArrowSchemaToIceberg(rec.Schema(), false, nil)
+ require.NoError(t, err)
+
+ requested := iceberg.NewSchema(0, iceberg.NestedField{
+ ID: 1, Name: "geom", Type: tt.icebergType,
Required: false,
+ })
+
+ ctx := compute.WithAllocator(context.Background(), mem)
+ out, err := table.ToRequestedSchema(ctx, requested,
fileSchema, rec, table.SchemaOptions{IncludeFieldIDs: true})
+ require.NoError(t, err)
+ defer out.Release()
+
+ want, err := table.TypeToArrowType(tt.icebergType,
true, false)
+ require.NoError(t, err)
+
+ got, ok := out.Column(0).(array.ExtensionArray)
+ require.True(t, ok, "expected extension array, got %T",
out.Column(0))
+ assert.True(t, arrow.TypeEqual(want, got.DataType()),
"expected: %s\ngot: %s", want, got.DataType())
+
+ storage := got.Storage()
+ require.Equal(t, len(values), storage.Len())
+
+ for i, v := range values {
+ if v == nil {
+ assert.True(t, storage.IsNull(i), "row
%d should be null", i)
+
+ continue
+ }
+ require.False(t, storage.IsNull(i), "row %d
should not be null", i)
+ assert.Equal(t, v, binaryValueAt(t, storage, i))
+ }
+ })
+ }
+}
+
+// roundTripWKBType returns the Arrow type a reader reconstructs for a column
+// written from icebergType, i.e. what a table sees reading its own files.
+func roundTripWKBType(t *testing.T, icebergType iceberg.Type)
arrow.ExtensionType {
+ t.Helper()
+
+ written, err := table.TypeToArrowType(icebergType, true, false)
+ require.NoError(t, err)
+
+ wkb, ok := written.(*geoarrow.WKBType)
+ require.True(t, ok, "expected geoarrow.wkb, got %T", written)
+
+ dt, err := geoarrow.NewWKBType().Deserialize(wkb.StorageType(),
wkb.Serialize())
+ require.NoError(t, err)
+
+ return dt
+}
+
+func TestToRequestedSchemaGeoSRID0Projection(t *testing.T) {
+ values := wkbPointRows()
+
+ geomSRID0, err := iceberg.GeometryTypeOf("srid:0")
+ require.NoError(t, err)
+ geogSRID0, err := iceberg.GeographyTypeOf("srid:0", "spherical")
+ require.NoError(t, err)
+
+ tests := []struct {
+ name string
+ icebergType iceberg.Type
+ wantType string
+ }{
+ {
+ name: "geometry_srid_0",
+ icebergType: geomSRID0,
+ wantType: "geometry(srid:0)",
+ },
+ {
+ name: "geography_srid_0",
+ icebergType: geogSRID0,
+ wantType: "geography(srid:0)",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ mem :=
memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ sourceType := roundTripWKBType(t, tt.icebergType)
+ source := newExtensionArrayOverBinary(t, mem,
sourceType, values)
+ defer source.Release()
+
+ rec := singleColumnRecord(arrow.Field{
+ Name: "geom", Type: sourceType, Nullable: true,
Metadata: fieldIDMeta("1"),
+ }, source)
+ defer rec.Release()
+
+ fileSchema, err :=
table.ArrowSchemaToIceberg(rec.Schema(), false, nil)
+ require.NoError(t, err)
+
+ fileField, ok := fileSchema.FindFieldByID(1)
+ require.True(t, ok)
+ assert.Equal(t, tt.wantType, fileField.Type.String())
+
+ requested := iceberg.NewSchema(0, iceberg.NestedField{
+ ID: 1, Name: "geom", Type: tt.icebergType,
Required: false,
+ })
+
+ ctx := compute.WithAllocator(context.Background(), mem)
+ out, err := table.ToRequestedSchema(ctx, requested,
fileSchema, rec, table.SchemaOptions{IncludeFieldIDs: true})
+ require.NoError(t, err)
+ defer out.Release()
+
+ want, err := table.TypeToArrowType(tt.icebergType,
true, false)
+ require.NoError(t, err)
+
+ got, ok := out.Column(0).(array.ExtensionArray)
+ require.True(t, ok, "expected extension array, got %T",
out.Column(0))
+ assert.True(t, arrow.TypeEqual(want, got.DataType()),
"expected: %s\ngot: %s", want, got.DataType())
+
+ storage := got.Storage()
+ require.Equal(t, len(values), storage.Len())
+
+ for i, v := range values {
+ if v == nil {
+ assert.True(t, storage.IsNull(i), "row
%d should be null", i)
+
+ continue
+ }
+ require.False(t, storage.IsNull(i), "row %d
should not be null", i)
+ assert.Equal(t, v, binaryValueAt(t, storage, i))
+ }
+ })
+ }
+}
+
+func TestToRequestedSchemaGeoMismatchedCRSFails(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ sourceType := geoarrow.NewWKBType(geoarrow.WKBWithBinaryStorage(),
geoarrow.WKBWithMetadata(geoarrow.Metadata{
+ CRS: jsonCRS("EPSG:3857"),
+ CRSType: geoarrow.CRSTypeAuthorityCode,
+ }))
+ source := newExtensionArrayOverBinary(t, mem, sourceType,
wkbPointRows())
+ defer source.Release()
+
+ rec := singleColumnRecord(arrow.Field{
+ Name: "geom", Type: sourceType, Nullable: true, Metadata:
fieldIDMeta("1"),
+ }, source)
+ defer rec.Release()
+
+ fileSchema, err := table.ArrowSchemaToIceberg(rec.Schema(), false, nil)
+ require.NoError(t, err)
+
+ fileField, ok := fileSchema.FindFieldByID(1)
+ require.True(t, ok)
+ require.Equal(t, "geometry(EPSG:3857)", fileField.Type.String())
+
+ requested := iceberg.NewSchema(0, iceberg.NestedField{
+ ID: 1, Name: "geom", Type: iceberg.GeometryType{}, Required:
false,
+ })
+
+ ctx := compute.WithAllocator(context.Background(), mem)
+ _, err = table.ToRequestedSchema(ctx, requested, fileSchema, rec,
table.SchemaOptions{IncludeFieldIDs: true})
+ require.Error(t, err)
+ assert.ErrorContains(t, err, "cannot promote geometry(EPSG:3857) to
geometry")
+}
+
+// A file whose CRS is absent reads as the default CRS, so a table declaring
+// srid:0 rejects it: CRS is not a promotable difference under the Iceberg
spec.
+func TestToRequestedSchemaGeoAbsentCRSAgainstSRID0SchemaFails(t *testing.T) {
+ geomSRID0, err := iceberg.GeometryTypeOf("srid:0")
+ require.NoError(t, err)
+ geogSRID0, err := iceberg.GeographyTypeOf("srid:0", "spherical")
+ require.NoError(t, err)
+
+ tests := []struct {
+ name string
+ edges geoarrow.EdgeInterpolation
+ icebergType iceberg.Type
+ wantErr string
+ }{
+ {
+ name: "geometry_crs_omitted",
+ icebergType: geomSRID0,
+ wantErr: "cannot promote geometry to
geometry(srid:0)",
+ },
+ {
+ name: "geography_crs_omitted",
+ edges: geoarrow.EdgeSpherical,
+ icebergType: geogSRID0,
+ wantErr: "cannot promote geography to
geography(srid:0)",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ mem :=
memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ sourceType := absentCRSWKBType(tt.edges)
+ source := newExtensionArrayOverBinary(t, mem,
sourceType, wkbPointRows())
+ defer source.Release()
+
+ rec := singleColumnRecord(arrow.Field{
+ Name: "geom", Type: sourceType, Nullable: true,
Metadata: fieldIDMeta("1"),
+ }, source)
+ defer rec.Release()
+
+ fileSchema, err :=
table.ArrowSchemaToIceberg(rec.Schema(), false, nil)
+ require.NoError(t, err)
+
+ requested := iceberg.NewSchema(0, iceberg.NestedField{
+ ID: 1, Name: "geom", Type: tt.icebergType,
Required: false,
+ })
+
+ ctx := compute.WithAllocator(context.Background(), mem)
+ _, err = table.ToRequestedSchema(ctx, requested,
fileSchema, rec, table.SchemaOptions{IncludeFieldIDs: true})
+ require.Error(t, err)
+ assert.ErrorContains(t, err, tt.wantErr)
+ })
+ }
+}
+
+// TestGeoTypeParquetRoundTrip pins the CRS a geo column carries on the Parquet
+// wire. arrow-go writes no GEOMETRY or GEOGRAPHY logical type, so the CRS
+// travels in the GeoArrow metadata of the stored Arrow schema.
+func TestGeoTypeParquetRoundTrip(t *testing.T) {
+ geomSRID0, err := iceberg.GeometryTypeOf("srid:0")
+ require.NoError(t, err)
+ geogSRID0, err := iceberg.GeographyTypeOf("srid:0", "spherical")
+ require.NoError(t, err)
+ geomSRID, err := iceberg.GeometryTypeOf("srid:4326")
+ require.NoError(t, err)
+ geomEPSG3857, err := iceberg.GeometryTypeOf("EPSG:3857")
+ require.NoError(t, err)
+
+ tests := []struct {
+ name string
+ icebergType iceberg.Type
+ geoarrowMetaJSON string
+ }{
+ {
+ name: "geometry_default_crs",
+ icebergType: iceberg.GeometryType{},
+ geoarrowMetaJSON:
`{"crs":"OGC:CRS84","crs_type":"authority_code"}`,
+ },
+ {
+ name: "geometry_srid_0",
+ icebergType: geomSRID0,
+ geoarrowMetaJSON: `{"crs":"0","crs_type":"srid"}`,
+ },
+ {
+ name: "geography_srid_0",
+ icebergType: geogSRID0,
+ geoarrowMetaJSON:
`{"crs":"0","crs_type":"srid","edges":"spherical"}`,
+ },
+ {
+ name: "geometry_srid_4326",
+ icebergType: geomSRID,
+ geoarrowMetaJSON: `{"crs":"4326","crs_type":"srid"}`,
+ },
+ {
+ name: "geometry_authority_code",
+ icebergType: geomEPSG3857,
+ geoarrowMetaJSON:
`{"crs":"EPSG:3857","crs_type":"authority_code"}`,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ mem := memory.NewGoAllocator()
+
+ dt, err := table.TypeToArrowType(tt.icebergType, true,
false)
+ require.NoError(t, err)
+
+ wkb, ok := dt.(*geoarrow.WKBType)
+ require.True(t, ok, "expected geoarrow.wkb, got %T", dt)
+ assert.JSONEq(t, tt.geoarrowMetaJSON, wkb.Serialize())
+
+ source := newExtensionArrayOverBinary(t, mem, wkb,
wkbPointRows())
+ defer source.Release()
+
+ rec := singleColumnRecord(arrow.Field{
+ Name: "geom", Type: wkb, Nullable: true,
Metadata: fieldIDMeta("1"),
+ }, source)
+ defer rec.Release()
+
+ var buf bytes.Buffer
+ writer, err := pqarrow.NewFileWriter(rec.Schema(), &buf,
+
parquet.NewWriterProperties(parquet.WithAllocator(mem)),
+
pqarrow.NewArrowWriterProperties(pqarrow.WithAllocator(mem),
pqarrow.WithStoreSchema()))
+ require.NoError(t, err)
+ require.NoError(t, writer.Write(rec))
+ require.NoError(t, writer.Close())
+
+ rdr, err :=
file.NewParquetReader(bytes.NewReader(buf.Bytes()))
+ require.NoError(t, err)
+ defer rdr.Close()
+
+ // Revisit the CRS spelling once arrow-go emits the geo
logical types:
+ // the Parquet CRS field takes the prefixed srid:<id>
form.
+ logical := rdr.MetaData().Schema.Column(0).LogicalType()
+ assert.True(t, logical.Equals(schema.NoLogicalType{}),
"unexpected logical type %s", logical)
+
+ arrRdr, err := pqarrow.NewFileReader(rdr,
pqarrow.ArrowReadProperties{}, mem)
+ require.NoError(t, err)
+
+ readSchema, err := arrRdr.Schema()
+ require.NoError(t, err)
+
+ readIceberg, err :=
table.ArrowSchemaToIceberg(readSchema, false, nil)
+ require.NoError(t, err)
+
+ field, ok := readIceberg.FindFieldByID(1)
+ require.True(t, ok)
+ assert.Equal(t, tt.icebergType.String(),
field.Type.String())
+ assert.True(t, tt.icebergType.Equals(field.Type),
"expected %s, got %s", tt.icebergType, field.Type)
+ })
+ }
+}
diff --git a/types.go b/types.go
index f19bd1003..a58cde5cd 100644
--- a/types.go
+++ b/types.go
@@ -248,7 +248,7 @@ func (t *typeIFace) UnmarshalJSON(b []byte) error {
return fmt.Errorf("%w: %s",
ErrInvalidTypeString, typename)
}
- crs := defaultGeoCRS
+ crs := DefaultGeoCRS
if matches[1] != "" {
crs = strings.TrimSpace(matches[1])
}
@@ -925,7 +925,10 @@ func (VariantType) Equals(other Type) bool {
func (VariantType) Type() string { return "variant" }
func (VariantType) String() string { return "variant" }
-const defaultGeoCRS = "OGC:CRS84"
+// DefaultGeoCRS is the CRS of the geometry and geography types when no CRS is
+// given; a Parquet GEOMETRY or GEOGRAPHY logical type without a CRS means the
+// same value.
+const DefaultGeoCRS = "OGC:CRS84"
type GeometryType struct {
crs string
@@ -936,7 +939,7 @@ func GeometryTypeOf(crs string) (GeometryType, error) {
return GeometryType{}, fmt.Errorf("%w: invalid CRS: (empty
string)", ErrInvalidTypeString)
}
crs = strings.TrimSpace(crs)
- if crs == defaultGeoCRS {
+ if crs == DefaultGeoCRS {
return GeometryType{}, nil
}
@@ -945,7 +948,7 @@ func GeometryTypeOf(crs string) (GeometryType, error) {
func (g GeometryType) CRS() string {
if g.crs == "" {
- return defaultGeoCRS
+ return DefaultGeoCRS
}
return g.crs
@@ -1000,7 +1003,7 @@ func GeographyTypeOf(crs string, algorithm string)
(GeographyType, error) {
}
crs = strings.TrimSpace(crs)
normalizedCRS := crs
- if normalizedCRS == defaultGeoCRS {
+ if normalizedCRS == DefaultGeoCRS {
normalizedCRS = ""
}
@@ -1018,7 +1021,7 @@ func GeographyTypeOf(crs string, algorithm string)
(GeographyType, error) {
func (g GeographyType) CRS() string {
if g.crs == "" {
- return defaultGeoCRS
+ return DefaultGeoCRS
}
return g.crs
@@ -1053,7 +1056,7 @@ func (g GeographyType) Type() string {
return fmt.Sprintf("geography(%s)", g.crs)
}
if !hasCRS && hasAlgo {
- return fmt.Sprintf("geography(%s, %s)", defaultGeoCRS,
g.algorithm)
+ return fmt.Sprintf("geography(%s, %s)", DefaultGeoCRS,
g.algorithm)
}
return fmt.Sprintf("geography(%s, %s)", g.crs, g.algorithm)
