This is an automated email from the ASF dual-hosted git repository.
mchades pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new dc912f9754 [#11329] fix(catalog-glue): Support complex types for
Iceberg tables in Glue catalog (#11429)
dc912f9754 is described below
commit dc912f97543e64ecfd4e44910dd14b001a057865
Author: Yuhui <[email protected]>
AuthorDate: Mon Jun 8 10:57:03 2026 +0800
[#11329] fix(catalog-glue): Support complex types for Iceberg tables in
Glue catalog (#11429)
### What changes were proposed in this pull request?
Support `ListType`, `MapType`, and `StructType` columns when creating
Iceberg
tables via the Glue catalog.
### Why are the changes needed?
Creating an Iceberg table with complex column types via the Glue catalog
threw
`UnsupportedOperationException`, while the standard Iceberg catalog
worked fine.
Fix: #11329
### Does this PR introduce _any_ user-facing change?
Yes. Users can now create Iceberg tables with `ListType`, `MapType`, and
`StructType` columns via the Glue catalog.
### How was this patch tested?
- Unit tests in `TestGlueIcebergTableHelper`
- Integration test
`AwsGlueCatalogIT#testCreateIcebergTableWithComplexTypes`
against real AWS Glue + S3
---
.../catalog/glue/GlueIcebergTableHelper.java | 55 +++++++++++++++++--
.../catalog/glue/TestGlueIcebergTableHelper.java | 61 +++++++++++++++++++++
.../glue/integration/test/AwsGlueCatalogIT.java | 64 ++++++++++++++++++++++
3 files changed, 174 insertions(+), 6 deletions(-)
diff --git
a/catalogs/catalog-glue/src/main/java/org/apache/gravitino/catalog/glue/GlueIcebergTableHelper.java
b/catalogs/catalog-glue/src/main/java/org/apache/gravitino/catalog/glue/GlueIcebergTableHelper.java
index f0fe363875..8438e6754a 100644
---
a/catalogs/catalog-glue/src/main/java/org/apache/gravitino/catalog/glue/GlueIcebergTableHelper.java
+++
b/catalogs/catalog-glue/src/main/java/org/apache/gravitino/catalog/glue/GlueIcebergTableHelper.java
@@ -40,6 +40,7 @@ import static
org.apache.gravitino.rel.types.Types.TimestampType;
import static org.apache.gravitino.rel.types.Types.UUIDType;
import static org.apache.gravitino.rel.types.Types.VarCharType;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import java.util.ArrayList;
import java.util.HashMap;
@@ -442,9 +443,13 @@ final class GlueIcebergTableHelper {
TableChange.UpdateColumnType upd = (TableChange.UpdateColumnType)
change;
Preconditions.checkArgument(
upd.fieldName().length == 1, "Nested column type updates are not
supported");
+ org.apache.iceberg.types.Type newIcebergType =
toIcebergType(upd.getNewDataType());
+ Preconditions.checkArgument(
+ newIcebergType instanceof
org.apache.iceberg.types.Type.PrimitiveType,
+ "Iceberg only supports primitive type promotion via
updateColumn, got: %s",
+ upd.getNewDataType().simpleString());
update.updateColumn(
- upd.fieldName()[0],
- (org.apache.iceberg.types.Type.PrimitiveType)
toIcebergType(upd.getNewDataType()));
+ upd.fieldName()[0],
(org.apache.iceberg.types.Type.PrimitiveType) newIcebergType);
} else if (change instanceof TableChange.UpdateColumnComment) {
TableChange.UpdateColumnComment upd =
(TableChange.UpdateColumnComment) change;
Preconditions.checkArgument(
@@ -578,13 +583,15 @@ final class GlueIcebergTableHelper {
// Type conversion (Gravitino -> Iceberg)
//
---------------------------------------------------------------------------
- private static Schema toIcebergSchema(Column[] columns) {
+ @VisibleForTesting
+ static Schema toIcebergSchema(Column[] columns) {
List<Types.NestedField> fields = new ArrayList<>();
- // Field IDs are assigned sequentially starting from 0. This is the
Iceberg convention
- // for initial schema creation. Schema evolution reuses existing IDs from
the table.
+ // Top-level columns use ordinal IDs (0, 1, ...). Nested field IDs start
after that range
+ // so all IDs are unique within the schema, as required by the Schema
constructor.
+ int[] nextId = {columns.length};
for (int i = 0; i < columns.length; i++) {
Column col = columns[i];
- org.apache.iceberg.types.Type icebergType =
toIcebergType(col.dataType());
+ org.apache.iceberg.types.Type icebergType =
toIcebergType(col.dataType(), nextId);
if (col.nullable()) {
fields.add(Types.NestedField.optional(i, col.name(), icebergType,
col.comment()));
} else {
@@ -595,6 +602,42 @@ final class GlueIcebergTableHelper {
}
private static org.apache.iceberg.types.Type toIcebergType(Type type) {
+ int[] nextId = {0};
+ return toIcebergType(type, nextId);
+ }
+
+ private static org.apache.iceberg.types.Type toIcebergType(Type type, int[]
nextId) {
+ if (type instanceof ListType) {
+ ListType listType = (ListType) type;
+ org.apache.iceberg.types.Type elementType =
toIcebergType(listType.elementType(), nextId);
+ int elementId = nextId[0]++;
+ return listType.elementNullable()
+ ? Types.ListType.ofOptional(elementId, elementType)
+ : Types.ListType.ofRequired(elementId, elementType);
+ }
+ if (type instanceof MapType) {
+ MapType mapType = (MapType) type;
+ org.apache.iceberg.types.Type keyType = toIcebergType(mapType.keyType(),
nextId);
+ org.apache.iceberg.types.Type valueType =
toIcebergType(mapType.valueType(), nextId);
+ int keyId = nextId[0]++;
+ int valueId = nextId[0]++;
+ return mapType.valueNullable()
+ ? Types.MapType.ofOptional(keyId, valueId, keyType, valueType)
+ : Types.MapType.ofRequired(keyId, valueId, keyType, valueType);
+ }
+ if (type instanceof StructType) {
+ StructType structType = (StructType) type;
+ List<Types.NestedField> nestedFields = new ArrayList<>();
+ for (StructType.Field field : structType.fields()) {
+ org.apache.iceberg.types.Type fieldType = toIcebergType(field.type(),
nextId);
+ int fieldId = nextId[0]++;
+ nestedFields.add(
+ field.nullable()
+ ? Types.NestedField.optional(fieldId, field.name(), fieldType,
field.comment())
+ : Types.NestedField.required(fieldId, field.name(), fieldType,
field.comment()));
+ }
+ return Types.StructType.of(nestedFields);
+ }
if (type instanceof BooleanType) {
return Types.BooleanType.get();
}
diff --git
a/catalogs/catalog-glue/src/test/java/org/apache/gravitino/catalog/glue/TestGlueIcebergTableHelper.java
b/catalogs/catalog-glue/src/test/java/org/apache/gravitino/catalog/glue/TestGlueIcebergTableHelper.java
index 5b99b63e2f..e3f3f3bb8b 100644
---
a/catalogs/catalog-glue/src/test/java/org/apache/gravitino/catalog/glue/TestGlueIcebergTableHelper.java
+++
b/catalogs/catalog-glue/src/test/java/org/apache/gravitino/catalog/glue/TestGlueIcebergTableHelper.java
@@ -20,6 +20,9 @@ package org.apache.gravitino.catalog.glue;
import static
org.apache.gravitino.catalog.glue.GlueIcebergTableHelper.fromIcebergType;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -130,6 +133,64 @@ class TestGlueIcebergTableHelper {
assertEquals(false, structType.fields()[1].nullable());
}
+ // -------------------------------------------------------------------------
+ // toIcebergSchema — complex types
+ // -------------------------------------------------------------------------
+
+ @Test
+ void testToIcebergSchemaWithListType() {
+ Column col =
+ GlueColumn.builder()
+ .withName("col_list")
+ .withType(Types.ListType.nullable(Types.StringType.get()))
+ .build();
+ Schema schema = GlueIcebergTableHelper.toIcebergSchema(new Column[] {col});
+
+ NestedField field = schema.findField("col_list");
+ assertInstanceOf(ListType.class, field.type());
+ ListType listType = (ListType) field.type();
+ assertEquals(StringType.get(), listType.elementType());
+ assertTrue(listType.isElementOptional());
+ }
+
+ @Test
+ void testToIcebergSchemaWithMapType() {
+ Column col =
+ GlueColumn.builder()
+ .withName("col_map")
+ .withType(Types.MapType.of(Types.StringType.get(),
Types.LongType.get(), false))
+ .build();
+ Schema schema = GlueIcebergTableHelper.toIcebergSchema(new Column[] {col});
+
+ NestedField field = schema.findField("col_map");
+ assertInstanceOf(MapType.class, field.type());
+ MapType mapType = (MapType) field.type();
+ assertEquals(StringType.get(), mapType.keyType());
+ assertEquals(LongType.get(), mapType.valueType());
+ assertFalse(mapType.isValueOptional());
+ }
+
+ @Test
+ void testToIcebergSchemaWithStructType() {
+ Types.StructType gravitinoStruct =
+ Types.StructType.of(
+ Types.StructType.Field.of("name", Types.StringType.get(), true,
"the name"),
+ Types.StructType.Field.of("age", Types.IntegerType.get(), false,
null));
+ Column col =
GlueColumn.builder().withName("col_struct").withType(gravitinoStruct).build();
+ Schema schema = GlueIcebergTableHelper.toIcebergSchema(new Column[] {col});
+
+ NestedField field = schema.findField("col_struct");
+ assertInstanceOf(StructType.class, field.type());
+ StructType structType = (StructType) field.type();
+ assertEquals(2, structType.fields().size());
+ assertEquals("name", structType.fields().get(0).name());
+ assertEquals(StringType.get(), structType.fields().get(0).type());
+ assertTrue(structType.fields().get(0).isOptional());
+ assertEquals("age", structType.fields().get(1).name());
+ assertEquals(IntegerType.get(), structType.fields().get(1).type());
+ assertFalse(structType.fields().get(1).isOptional());
+ }
+
// -------------------------------------------------------------------------
// loadTable — column type overwrite
// -------------------------------------------------------------------------
diff --git
a/catalogs/catalog-glue/src/test/java/org/apache/gravitino/catalog/glue/integration/test/AwsGlueCatalogIT.java
b/catalogs/catalog-glue/src/test/java/org/apache/gravitino/catalog/glue/integration/test/AwsGlueCatalogIT.java
index 8ac09cf389..eb1437e3a4 100644
---
a/catalogs/catalog-glue/src/test/java/org/apache/gravitino/catalog/glue/integration/test/AwsGlueCatalogIT.java
+++
b/catalogs/catalog-glue/src/test/java/org/apache/gravitino/catalog/glue/integration/test/AwsGlueCatalogIT.java
@@ -19,6 +19,8 @@
package org.apache.gravitino.catalog.glue.integration.test;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import com.google.common.base.Preconditions;
import java.util.Collections;
@@ -89,6 +91,68 @@ class AwsGlueCatalogIT extends AbstractGlueCatalogIT {
return config;
}
+ @Test
+ void testCreateIcebergTableWithComplexTypes() {
+ String bucket = System.getenv("AWS_S3_TEST_BUCKET");
+ String schema = "glue_it_" + System.nanoTime();
+ ops.createSchema(NameIdentifier.of("ml", "cat", schema), null,
Collections.emptyMap());
+
+ Map<String, String> props = new HashMap<>();
+ props.put(GlueConstants.TABLE_FORMAT, "ICEBERG");
+ props.put(GlueConstants.LOCATION, "s3://" + bucket + "/" + schema + "/");
+
+ Column[] cols = {
+ Column.of("tags", Types.ListType.nullable(Types.StringType.get()), "list
col"),
+ Column.of(
+ "scores",
+ Types.MapType.of(Types.StringType.get(), Types.DoubleType.get(),
false),
+ "map col"),
+ Column.of(
+ "info",
+ Types.StructType.of(
+ Types.StructType.Field.of("name", Types.StringType.get(), true,
null),
+ Types.StructType.Field.of("age", Types.IntegerType.get(), false,
null)),
+ "struct col"),
+ };
+
+ ops.createTable(
+ NameIdentifier.of("ml", "cat", schema, "ice_complex"),
+ cols,
+ null,
+ props,
+ new Transform[0],
+ Distributions.NONE,
+ new SortOrder[0],
+ new Index[0]);
+
+ try {
+ Table loaded = ops.loadTable(NameIdentifier.of("ml", "cat", schema,
"ice_complex"));
+ assertEquals(3, loaded.columns().length);
+
+ assertEquals("list col", loaded.columns()[0].comment());
+ Types.ListType listType = (Types.ListType)
loaded.columns()[0].dataType();
+ assertEquals(Types.StringType.get(), listType.elementType());
+ assertTrue(listType.elementNullable());
+
+ assertEquals("map col", loaded.columns()[1].comment());
+ Types.MapType mapType = (Types.MapType) loaded.columns()[1].dataType();
+ assertEquals(Types.StringType.get(), mapType.keyType());
+ assertEquals(Types.DoubleType.get(), mapType.valueType());
+ assertFalse(mapType.valueNullable());
+
+ assertEquals("struct col", loaded.columns()[2].comment());
+ Types.StructType structType = (Types.StructType)
loaded.columns()[2].dataType();
+ assertEquals(2, structType.fields().length);
+ assertEquals("name", structType.fields()[0].name());
+ assertEquals(Types.StringType.get(), structType.fields()[0].type());
+ assertEquals("age", structType.fields()[1].name());
+ assertEquals(Types.IntegerType.get(), structType.fields()[1].type());
+ } finally {
+ ops.dropTable(NameIdentifier.of("ml", "cat", schema, "ice_complex"));
+ ops.dropSchema(NameIdentifier.of("ml", "cat", schema), false);
+ }
+ }
+
@Test
void testAlterIcebergMetadata() {
String bucket = System.getenv("AWS_S3_TEST_BUCKET");
