This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 41231c73585 [Feature](iceberg) Support schema change for complex types
in Iceberg external tables (#60169)
41231c73585 is described below
commit 41231c73585d250be843df31bc22520bb71f5ffc
Author: Chenjunwei <[email protected]>
AuthorDate: Tue Mar 3 18:21:33 2026 +0800
[Feature](iceberg) Support schema change for complex types in Iceberg
external tables (#60169)
Problem Summary:
Currently, Doris does not support `ALTER TABLE ... MODIFY COLUMN` for
complex type columns (STRUCT/ARRAY/MAP) in Iceberg external tables. This
PR adds support for safe schema evolution of complex types in Iceberg
tables.
**Supported operations:**
| Complex Type | Allowed Operations | Prohibited Operations |
|--------------|-------------------|----------------------|
| STRUCT | Append new fields, safe type promotion for existing fields |
Field rename/delete |
| ARRAY | Element type safe promotion | Incompatible element type change
|
| MAP | Value type safe promotion | Key type change |
**Safe type promotions supported in nested types:**
- `INT` → `BIGINT`, `LARGEINT`
- `TINYINT` → `SMALLINT`, `INT`, `BIGINT`, `LARGEINT`
- `SMALLINT` → `INT`, `BIGINT`, `LARGEINT`
- `BIGINT` → `LARGEINT`
- `FLOAT` → `DOUBLE`
- `VARCHAR(n)` → `VARCHAR(m)` where m > n
**Constraints:**
- All new nested fields must be nullable
- Cannot change optional to required
- Complex type default values only support NULL
---
.../java/org/apache/doris/catalog/ColumnType.java | 65 ++++--
.../datasource/iceberg/IcebergMetadataOps.java | 216 +++++++++++++++++--
.../java/org/apache/doris/catalog/ColumnTest.java | 9 +-
.../iceberg/IcebergMetadataOpsValidationTest.java | 180 ++++++++++++++++
.../iceberg/iceberg_schema_change_ddl.out | 28 +--
.../iceberg/iceberg_schema_change_ddl.groovy | 3 +-
...test_iceberg_schema_change_complex_types.groovy | 232 +++++++++++++++++++++
7 files changed, 689 insertions(+), 44 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java
index dd311d619c5..39e852a4b94 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java
@@ -190,21 +190,62 @@ public abstract class ColumnType {
}
}
- // This method defines the char type
- // to support the schema-change behavior of length growth.
- // return true if the checkType and other are both char-type otherwise
return false,
- // which used in checkSupportSchemaChangeForComplexType
- private static boolean checkSupportSchemaChangeForCharType(Type checkType,
Type other) throws DdlException {
+ // This method checks if a primitive type change is allowed in nested
complex types.
+ // Supports:
+ // 1. VARCHAR length increase
+ // 2. Safe numeric type promotions (INT -> BIGINT, FLOAT -> DOUBLE, etc.)
+ // 3. Exact type match
+ private static boolean checkSupportSchemaChangeForNestedPrimitive(Type
checkType, Type other) throws DdlException {
+ // 1. Check VARCHAR length increase
if (checkType.getPrimitiveType() == PrimitiveType.VARCHAR
&& other.getPrimitiveType() == PrimitiveType.VARCHAR) {
- // currently nested types only support light schema change for
internal fields, for string types,
- // only varchar can do light schema change
checkForTypeLengthChange(checkType, other);
return true;
- } else {
- // types equal can return true
- return checkType.equals(other);
}
+
+ // 2. Check exact type match (including STRING == STRING)
+ if (checkType.equals(other)) {
+ return true;
+ }
+
+ // 3. Check safe numeric type promotions for nested types
+ // These are safe promotions that don't lose precision:
+ // - INT -> BIGINT, LARGEINT
+ // - FLOAT -> DOUBLE
+ PrimitiveType srcType = checkType.getPrimitiveType();
+ PrimitiveType dstType = other.getPrimitiveType();
+
+ // INT -> BIGINT, LARGEINT
+ if (srcType == PrimitiveType.INT
+ && (dstType == PrimitiveType.BIGINT || dstType ==
PrimitiveType.LARGEINT)) {
+ return true;
+ }
+
+ // TINYINT -> SMALLINT, INT, BIGINT, LARGEINT
+ if (srcType == PrimitiveType.TINYINT
+ && (dstType == PrimitiveType.SMALLINT || dstType ==
PrimitiveType.INT
+ || dstType == PrimitiveType.BIGINT || dstType ==
PrimitiveType.LARGEINT)) {
+ return true;
+ }
+
+ // SMALLINT -> INT, BIGINT, LARGEINT
+ if (srcType == PrimitiveType.SMALLINT
+ && (dstType == PrimitiveType.INT || dstType ==
PrimitiveType.BIGINT
+ || dstType == PrimitiveType.LARGEINT)) {
+ return true;
+ }
+
+ // BIGINT -> LARGEINT
+ if (srcType == PrimitiveType.BIGINT && dstType ==
PrimitiveType.LARGEINT) {
+ return true;
+ }
+
+ // FLOAT -> DOUBLE
+ if (srcType == PrimitiveType.FLOAT && dstType == PrimitiveType.DOUBLE)
{
+ return true;
+ }
+
+ return false;
}
private static void validateStructFieldCompatibility(StructField
originalField, StructField newField)
@@ -272,9 +313,9 @@ public abstract class ColumnType {
checkSupportSchemaChangeForComplexType(((MapType)
checkType).getValueType(),
((MapType) other).getValueType(), true);
} else {
- // only support char-type schema change behavior for nested
complex type
+ // Support safe type promotions for nested primitive types
// if nested is false, we do not check return value.
- if (nested && !checkSupportSchemaChangeForCharType(checkType,
other)) {
+ if (nested &&
!checkSupportSchemaChangeForNestedPrimitive(checkType, other)) {
throw new DdlException(
"Cannot change " + checkType.toSql() + " to " +
other.toSql() + " in nested types");
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
index bcdb7e2f39d..751c98c37d8 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
@@ -18,8 +18,11 @@
package org.apache.doris.datasource.iceberg;
import org.apache.doris.analysis.ColumnPosition;
+import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.ColumnType;
import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.MapType;
import org.apache.doris.catalog.StructField;
import org.apache.doris.catalog.StructType;
import org.apache.doris.common.DdlException;
@@ -68,6 +71,7 @@ import org.apache.iceberg.expressions.Expressions;
import org.apache.iceberg.expressions.Literal;
import org.apache.iceberg.expressions.Term;
import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
import org.apache.iceberg.types.Types.NestedField;
import org.apache.iceberg.view.View;
import org.apache.logging.log4j.LogManager;
@@ -79,6 +83,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.stream.Collectors;
@@ -727,15 +732,36 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
public void modifyColumn(ExternalTable dorisTable, Column column,
ColumnPosition position, long updateTime)
throws UserException {
Table icebergTable = IcebergUtils.getIcebergTable(dorisTable);
- validateForModifyColumn(column, icebergTable);
- Type icebergType =
IcebergUtils.dorisTypeToIcebergType(column.getType());
+ NestedField currentCol =
icebergTable.schema().findField(column.getName());
+ if (currentCol == null) {
+ throw new UserException("Column " + column.getName() + " does not
exist");
+ }
+
+ validateCommonColumnInfo(column);
UpdateSchema updateSchema = icebergTable.updateSchema();
- updateSchema.updateColumn(column.getName(),
icebergType.asPrimitiveType(), column.getComment());
- if (column.isAllowNull()) {
- // we can change a required column to optional, but not the other
way around
- // because we don't know whether there is existing data with null
values.
- updateSchema.makeColumnOptional(column.getName());
+
+ if (column.getType().isComplexType()) {
+ // Complex type processing branch
+ validateForModifyComplexColumn(column, currentCol);
+ applyComplexTypeChange(updateSchema, column.getName(),
currentCol.type(), column.getType());
+ if (column.isAllowNull()) {
+ updateSchema.makeColumnOptional(column.getName());
+ }
+ if (!Objects.equals(currentCol.doc(), column.getComment())) {
+ updateSchema.updateColumnDoc(column.getName(),
column.getComment());
+ }
+ } else {
+ // Primitive type processing (existing logic)
+ validateForModifyColumn(column, currentCol);
+ Type icebergType =
IcebergUtils.dorisTypeToIcebergType(column.getType());
+ updateSchema.updateColumn(column.getName(),
icebergType.asPrimitiveType(), column.getComment());
+ if (column.isAllowNull()) {
+ // we can change a required column to optional, but not the
other way around
+ // because we don't know whether there is existing data with
null values.
+ updateSchema.makeColumnOptional(column.getName());
+ }
}
+
if (position != null) {
applyPosition(updateSchema, position, column.getName());
}
@@ -748,23 +774,184 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
refreshTable(dorisTable, updateTime);
}
- private void validateForModifyColumn(Column column, Table icebergTable)
throws UserException {
- validateCommonColumnInfo(column);
+ private void validateForModifyColumn(Column column, NestedField
currentCol) throws UserException {
// check complex type
if (column.getType().isComplexType()) {
throw new UserException("Modify column type to non-primitive type
is not supported: " + column.getType());
}
- // check exist
- NestedField currentCol =
icebergTable.schema().findField(column.getName());
- if (currentCol == null) {
- throw new UserException("Column " + column.getName() + " does not
exist");
- }
// check nullable
if (currentCol.isOptional() && !column.isAllowNull()) {
throw new UserException("Can not change nullable column " +
column.getName() + " to not null");
}
}
+ private void validateForModifyComplexColumn(Column column, NestedField
currentCol) throws UserException {
+ if (!column.getType().isComplexType()) {
+ throw new UserException("Modify column type to non-complex type is
not supported: " + column.getType());
+ }
+ Type oldIcebergType = currentCol.type();
+ if (oldIcebergType.isPrimitiveType()) {
+ throw new UserException("Modify column type from non-complex to
complex is not supported: "
+ + column.getName());
+ }
+
+ org.apache.doris.catalog.Type oldDorisType =
IcebergUtils.icebergTypeToDorisType(oldIcebergType, false, false);
+ org.apache.doris.catalog.Type newDorisType = column.getType();
+ if (!isSameComplexCategory(oldIcebergType, newDorisType)) {
+ throw new UserException("Cannot change complex column type
category from "
+ + oldDorisType.toSql() + " to " + newDorisType.toSql());
+ }
+ try {
+ ColumnType.checkSupportSchemaChangeForComplexType(oldDorisType,
newDorisType, false);
+ } catch (DdlException e) {
+ throw new UserException(e.getMessage(), e);
+ }
+ if (currentCol.isOptional() && !column.isAllowNull()) {
+ throw new UserException("Cannot change nullable column " +
column.getName() + " to not null");
+ }
+ if (column.getDefaultValue() != null ||
column.getDefaultValueExprDef() != null) {
+ throw new UserException("Complex type default value only supports
NULL");
+ }
+ }
+
+ private boolean isSameComplexCategory(Type oldIcebergType,
org.apache.doris.catalog.Type newDorisType) {
+ switch (oldIcebergType.typeId()) {
+ case STRUCT:
+ return newDorisType.isStructType();
+ case LIST:
+ return newDorisType.isArrayType();
+ case MAP:
+ return newDorisType.isMapType();
+ default:
+ return false;
+ }
+ }
+
+ private void applyComplexTypeChange(UpdateSchema updateSchema, String path,
+ org.apache.iceberg.types.Type oldIcebergType,
+ org.apache.doris.catalog.Type newDorisType) throws UserException {
+ switch (oldIcebergType.typeId()) {
+ case STRUCT:
+ applyStructChange(updateSchema, path,
oldIcebergType.asStructType(), (StructType) newDorisType);
+ break;
+ case LIST:
+ applyListChange(updateSchema, path, (Types.ListType)
oldIcebergType, (ArrayType) newDorisType);
+ break;
+ case MAP:
+ applyMapChange(updateSchema, path, (Types.MapType)
oldIcebergType, (MapType) newDorisType);
+ break;
+ default:
+ throw new UserException("Unsupported complex type for modify:
" + oldIcebergType);
+ }
+ }
+
+ private void applyStructChange(UpdateSchema updateSchema, String path,
+ Types.StructType oldStructType, StructType newStructType) throws
UserException {
+ List<NestedField> oldFields = oldStructType.fields();
+ List<StructField> newFields = newStructType.getFields();
+
+ for (int i = 0; i < oldFields.size(); i++) {
+ NestedField oldField = oldFields.get(i);
+ StructField newField = newFields.get(i);
+ String fieldPath = path + "." + oldField.name();
+
+ if (oldField.isOptional() && !newField.getContainsNull()) {
+ throw new UserException("Cannot change nullable column " +
fieldPath + " to not null");
+ }
+
+ org.apache.iceberg.types.Type oldFieldType = oldField.type();
+ org.apache.doris.catalog.Type newFieldType = newField.getType();
+ if (oldFieldType.isPrimitiveType()) {
+ org.apache.doris.catalog.Type oldDorisFieldType =
+ IcebergUtils.icebergTypeToDorisType(oldFieldType,
false, false);
+ boolean typeChanged = !oldDorisFieldType.equals(newFieldType);
+ boolean commentChanged = !Objects.equals(oldField.doc(),
newField.getComment());
+ if (typeChanged || commentChanged) {
+ org.apache.iceberg.types.Type newIcebergFieldType =
+ IcebergUtils.dorisTypeToIcebergType(newFieldType);
+ updateSchema.updateColumn(fieldPath,
newIcebergFieldType.asPrimitiveType(),
+ newField.getComment());
+ }
+ } else {
+ applyComplexTypeChange(updateSchema, fieldPath, oldFieldType,
newFieldType);
+ if (!Objects.equals(oldField.doc(), newField.getComment())) {
+ updateSchema.updateColumnDoc(fieldPath,
newField.getComment());
+ }
+ }
+
+ if (!oldField.isOptional() && newField.getContainsNull()) {
+ updateSchema.makeColumnOptional(fieldPath);
+ }
+ }
+
+ for (int i = oldFields.size(); i < newFields.size(); i++) {
+ StructField newField = newFields.get(i);
+ if (!newField.getContainsNull()) {
+ throw new UserException("New struct field '" +
newField.getName() + "' must be nullable");
+ }
+ org.apache.iceberg.types.Type newFieldIcebergType =
+ IcebergUtils.dorisTypeToIcebergType(newField.getType());
+ updateSchema.addColumn(path, newField.getName(),
newFieldIcebergType, newField.getComment());
+ }
+ }
+
+ private void applyListChange(UpdateSchema updateSchema, String path,
+ Types.ListType oldListType, ArrayType newArrayType) throws
UserException {
+ String elementPath = path + "." +
oldListType.field(oldListType.elementId()).name();
+ if (oldListType.isElementOptional() &&
!newArrayType.getContainsNull()) {
+ throw new UserException("Cannot change nullable column " +
elementPath + " to not null");
+ }
+ org.apache.iceberg.types.Type oldElementType =
oldListType.elementType();
+ org.apache.doris.catalog.Type newElementType =
newArrayType.getItemType();
+ if (oldElementType.isPrimitiveType()) {
+ org.apache.doris.catalog.Type oldDorisElementType =
+ IcebergUtils.icebergTypeToDorisType(oldElementType, false,
false);
+ if (!oldDorisElementType.equals(newElementType)) {
+ org.apache.iceberg.types.Type newIcebergElementType =
+ IcebergUtils.dorisTypeToIcebergType(newElementType);
+ updateSchema.updateColumn(elementPath,
newIcebergElementType.asPrimitiveType(), null);
+ }
+ } else {
+ applyComplexTypeChange(updateSchema, elementPath, oldElementType,
newElementType);
+ }
+ if (!oldListType.isElementOptional() &&
newArrayType.getContainsNull()) {
+ updateSchema.makeColumnOptional(elementPath);
+ }
+ }
+
+ private void applyMapChange(UpdateSchema updateSchema, String path,
+ Types.MapType oldMapType, MapType newMapType) throws UserException
{
+ org.apache.iceberg.types.Type oldKeyType = oldMapType.keyType();
+ org.apache.doris.catalog.Type newKeyType = newMapType.getKeyType();
+ org.apache.doris.catalog.Type oldDorisKeyType =
+ IcebergUtils.icebergTypeToDorisType(oldKeyType, false, false);
+ if (!oldDorisKeyType.equals(newKeyType)) {
+ throw new UserException("Cannot change MAP key type from "
+ + oldDorisKeyType.toSql() + " to " + newKeyType.toSql());
+ }
+
+ String valuePath = path + "." +
oldMapType.field(oldMapType.valueId()).name();
+ if (oldMapType.isValueOptional() &&
!newMapType.getIsValueContainsNull()) {
+ throw new UserException("Cannot change nullable column " +
valuePath + " to not null");
+ }
+ org.apache.iceberg.types.Type oldValueType = oldMapType.valueType();
+ org.apache.doris.catalog.Type newValueType = newMapType.getValueType();
+ if (oldValueType.isPrimitiveType()) {
+ org.apache.doris.catalog.Type oldDorisValueType =
+ IcebergUtils.icebergTypeToDorisType(oldValueType, false,
false);
+ if (!oldDorisValueType.equals(newValueType)) {
+ org.apache.iceberg.types.Type newIcebergValueType =
+ IcebergUtils.dorisTypeToIcebergType(newValueType);
+ updateSchema.updateColumn(valuePath,
newIcebergValueType.asPrimitiveType(), null);
+ }
+ } else {
+ applyComplexTypeChange(updateSchema, valuePath, oldValueType,
newValueType);
+ }
+ if (!oldMapType.isValueOptional() &&
newMapType.getIsValueContainsNull()) {
+ updateSchema.makeColumnOptional(valuePath);
+ }
+ }
+
private void validateCommonColumnInfo(Column column) throws UserException {
// check aggregation method
if (column.isAggregated()) {
@@ -1046,4 +1233,3 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
return builder.build();
}
}
-
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnTest.java
index cde493d4adb..c3c82c5fc3e 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnTest.java
@@ -151,11 +151,18 @@ public class ColumnTest {
Assert.fail("No exception throws.");
}
- @Test(expected = DdlException.class)
+ @Test
public void testSchemaChangeArrayToArray() throws DdlException {
Column oldColumn = new Column("a", ArrayType.create(Type.TINYINT,
true), false, null, true, "0", "");
Column newColumn = new Column("a", ArrayType.create(Type.INT, true),
false, null, true, "0", "");
oldColumn.checkSchemaChangeAllowed(newColumn);
+ }
+
+ @Test(expected = DdlException.class)
+ public void testSchemaChangeArrayToArrayDowngrade() throws DdlException {
+ Column oldColumn = new Column("a", ArrayType.create(Type.INT, true),
false, null, true, "0", "");
+ Column newColumn = new Column("a", ArrayType.create(Type.TINYINT,
true), false, null, true, "0", "");
+ oldColumn.checkSchemaChangeAllowed(newColumn);
Assert.fail("No exception throws.");
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java
new file mode 100644
index 00000000000..74baff368d8
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java
@@ -0,0 +1,180 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.iceberg;
+
+import org.apache.doris.catalog.ArrayType;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.MapType;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.common.UserException;
+import org.apache.doris.common.security.authentication.ExecutionAuthenticator;
+import org.apache.doris.datasource.ExternalCatalog;
+
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.types.Types.NestedField;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.Collections;
+
+public class IcebergMetadataOpsValidationTest {
+
+ private IcebergMetadataOps ops;
+ private Method validateForModifyColumnMethod;
+ private Method validateForModifyComplexColumnMethod;
+
+ @Before
+ public void setUp() throws Exception {
+ ExternalCatalog dorisCatalog = Mockito.mock(ExternalCatalog.class);
+ Catalog icebergCatalog = Mockito.mock(Catalog.class,
+
Mockito.withSettings().extraInterfaces(SupportsNamespaces.class));
+ Mockito.when(dorisCatalog.getExecutionAuthenticator()).thenReturn(new
ExecutionAuthenticator() {
+ });
+
Mockito.when(dorisCatalog.getProperties()).thenReturn(Collections.emptyMap());
+ ops = new IcebergMetadataOps(dorisCatalog, icebergCatalog);
+
+ validateForModifyColumnMethod =
IcebergMetadataOps.class.getDeclaredMethod(
+ "validateForModifyColumn", Column.class, NestedField.class);
+ validateForModifyColumnMethod.setAccessible(true);
+ validateForModifyComplexColumnMethod =
IcebergMetadataOps.class.getDeclaredMethod(
+ "validateForModifyComplexColumn", Column.class,
NestedField.class);
+ validateForModifyComplexColumnMethod.setAccessible(true);
+ }
+
+ @Test
+ public void testValidateForModifyColumnRejectsComplexType() {
+ Column column = new Column("arr_i", ArrayType.create(Type.INT, true),
true);
+ NestedField currentCol = Types.NestedField.required(1, "arr_i",
Types.IntegerType.get());
+ assertUserException(() -> invokeValidateForModifyColumn(column,
currentCol),
+ "Modify column type to non-primitive type is not supported");
+ }
+
+ @Test
+ public void testValidateForModifyColumnRejectsNullableToNotNull() {
+ Column column = new Column("int_col", Type.INT, false);
+ NestedField currentCol = Types.NestedField.optional(1, "int_col",
Types.IntegerType.get());
+ assertUserException(() -> invokeValidateForModifyColumn(column,
currentCol),
+ "Can not change nullable column int_col to not null");
+ }
+
+ @Test
+ public void testValidateForModifyColumnSuccess() throws Throwable {
+ Column column = new Column("int_col", Type.INT, true);
+ NestedField currentCol = Types.NestedField.required(1, "int_col",
Types.IntegerType.get());
+ invokeValidateForModifyColumn(column, currentCol);
+ }
+
+ @Test
+ public void testValidateForModifyComplexColumnRejectsPrimitiveType() {
+ Column column = new Column("arr_i", Type.INT, true);
+ NestedField currentCol = Types.NestedField.required(1, "arr_i",
+ Types.ListType.ofOptional(2, Types.IntegerType.get()));
+ assertUserException(() -> invokeValidateForModifyComplexColumn(column,
currentCol),
+ "Modify column type to non-complex type is not supported");
+ }
+
+ @Test
+ public void
testValidateForModifyComplexColumnRejectsIncompatibleNestedType() {
+ Column column = new Column("arr_i", ArrayType.create(Type.SMALLINT,
true), true);
+ NestedField currentCol = Types.NestedField.required(1, "arr_i",
+ Types.ListType.ofOptional(2, Types.IntegerType.get()));
+ assertUserException(() -> invokeValidateForModifyComplexColumn(column,
currentCol),
+ "Cannot change int to smallint in nested types");
+ }
+
+ @Test
+ public void testValidateForModifyComplexColumnRejectsPrimitiveToComplex() {
+ Column column = new Column("arr_i", ArrayType.create(Type.INT, true),
true);
+ NestedField currentCol = Types.NestedField.required(1, "arr_i",
Types.IntegerType.get());
+ assertUserException(() -> invokeValidateForModifyComplexColumn(column,
currentCol),
+ "Modify column type from non-complex to complex is not
supported");
+ }
+
+ @Test
+ public void
testValidateForModifyComplexColumnRejectsDifferentComplexCategory() {
+ Column column = new Column("arr_i", new MapType(Type.INT, Type.INT),
true);
+ NestedField currentCol = Types.NestedField.required(1, "arr_i",
+ Types.ListType.ofOptional(2, Types.IntegerType.get()));
+ assertUserException(() -> invokeValidateForModifyComplexColumn(column,
currentCol),
+ "Cannot change complex column type category");
+ }
+
+ @Test
+ public void testValidateForModifyComplexColumnRejectsNullableToNotNull() {
+ Column column = new Column("arr_i", ArrayType.create(Type.INT, true),
false);
+ NestedField currentCol = Types.NestedField.optional(1, "arr_i",
+ Types.ListType.ofOptional(2, Types.IntegerType.get()));
+ assertUserException(() -> invokeValidateForModifyComplexColumn(column,
currentCol),
+ "Cannot change nullable column arr_i to not null");
+ }
+
+ @Test
+ public void testValidateForModifyComplexColumnRejectsDefaultValue() {
+ Column column = new Column("arr_i", ArrayType.create(Type.INT, true),
+ false, null, true, "1", "");
+ NestedField currentCol = Types.NestedField.required(1, "arr_i",
+ Types.ListType.ofOptional(2, Types.IntegerType.get()));
+ assertUserException(() -> invokeValidateForModifyComplexColumn(column,
currentCol),
+ "Complex type default value only supports NULL");
+ }
+
+ @Test
+ public void testValidateForModifyComplexColumnSuccess() throws Throwable {
+ Column column = new Column("arr_i", ArrayType.create(Type.BIGINT,
true), true);
+ NestedField currentCol = Types.NestedField.required(1, "arr_i",
+ Types.ListType.ofOptional(2, Types.IntegerType.get()));
+ invokeValidateForModifyComplexColumn(column, currentCol);
+ }
+
+ private void invokeValidateForModifyColumn(Column column, NestedField
currentCol) throws Throwable {
+ invokeValidationMethod(validateForModifyColumnMethod, column,
currentCol);
+ }
+
+ private void invokeValidateForModifyComplexColumn(Column column,
NestedField currentCol) throws Throwable {
+ invokeValidationMethod(validateForModifyComplexColumnMethod, column,
currentCol);
+ }
+
+ private void invokeValidationMethod(Method method, Column column,
NestedField currentCol) throws Throwable {
+ try {
+ method.invoke(ops, column, currentCol);
+ } catch (InvocationTargetException e) {
+ throw e.getCause();
+ }
+ }
+
+ private void assertUserException(ThrowingRunnable runnable, String
expectedMessage) {
+ try {
+ runnable.run();
+ Assert.fail("expected UserException");
+ } catch (Throwable t) {
+ Assert.assertTrue(t instanceof UserException);
+ Assert.assertTrue(t.getMessage().contains(expectedMessage));
+ }
+ }
+
+ @FunctionalInterface
+ private interface ThrowingRunnable {
+ void run() throws Throwable;
+ }
+}
diff --git
a/regression-test/data/external_table_p0/iceberg/iceberg_schema_change_ddl.out
b/regression-test/data/external_table_p0/iceberg/iceberg_schema_change_ddl.out
index 2f68c298f37..dab31ada31b 100644
---
a/regression-test/data/external_table_p0/iceberg/iceberg_schema_change_ddl.out
+++
b/regression-test/data/external_table_p0/iceberg/iceberg_schema_change_ddl.out
@@ -22,7 +22,7 @@ email text Yes true \N
1 Alice 25 \N 95.5 \N
2 Bob 30 \N 87.2 \N
3 Charlie 22 \N 92.8 \N
-4 David 28 123-456-7890 89.1 [email protected]
+4 David 28 123-456-7890 89.09999999999999
[email protected]
-- !add_3 --
4 [email protected]
@@ -45,7 +45,7 @@ address struct<city:text,country:text> Yes true
\N
1 Alice 25 \N 95.5 \N \N
2 Bob 30 \N 87.2 \N \N
3 Charlie 22 \N 92.8 \N \N
-4 David 28 123-456-7890 89.1 [email protected] \N
+4 David 28 123-456-7890 89.09999999999999
[email protected] \N
5 Eve 26 223-345-132 91.3 [email protected]
{"city":"New York", "country":"USA"}
-- !add_multi_3 --
@@ -64,7 +64,7 @@ address struct<city:text,country:text> Yes true
\N
1 95.5
2 87.2
3 92.8
-4 89.1
+4 89.09999999999999
5 91.3
-- !rename_3 --
@@ -84,7 +84,7 @@ address struct<city:text,country:text> Yes true
\N
1 25 \N 95.5 \N \N
2 30 \N 87.2 \N \N
3 22 \N 92.8 \N \N
-4 28 123-456-7890 89.1 [email protected] \N
+4 28 123-456-7890 89.09999999999999 [email protected]
\N
5 26 223-345-132 91.3 [email protected] {"city":"New
York", "country":"USA"}
-- !drop_3 --
@@ -129,7 +129,7 @@ col2 text Yes true \N User
defined column2
-- !modify_3 --
30 2 \N 87.2 \N \N \N \N
-28 4 123-456-7890 89.1 [email protected] \N \N
\N
+28 4 123-456-7890 89.09999999999999 [email protected]
\N \N \N
26 5 223-345-132 91.3 [email protected] {"city":"New
York", "country":"USA"} \N \N
-- !before_no_comment --
@@ -193,10 +193,10 @@ test_decimal decimal(10,2) Yes true \N
1 25 \N \N 95.5 \N \N \N \N \N
2 30 \N \N 87.2 \N \N \N \N \N
3 22 \N \N 92.8 \N \N \N \N \N
-4 28 \N \N 89.1 123-456-7890 [email protected]
\N \N \N
+4 28 \N \N 89.09999999999999 123-456-7890
[email protected] \N \N \N
5 26 \N \N 91.3 223-345-132 [email protected]
{"city":"New York", "country":"USA"} \N \N
-6 \N \N \N 100.0 123-456-7890 [email protected]
{"city":"Los Angeles", "country":"USA"} \N \N
-7 \N \N \N 0.0 \N \N \N
3.140000104904175 123.45
+6 \N \N \N 100 123-456-7890 [email protected]
{"city":"Los Angeles", "country":"USA"} \N \N
+7 \N \N \N 0 \N \N \N
3.140000104904175 123.45
-- !before_rename_show_tables_old --
iceberg_ddl_test
@@ -224,19 +224,19 @@ test_decimal decimal(10,2) Yes true \N
1 25 \N \N 95.5 \N \N \N \N \N
2 30 \N \N 87.2 \N \N \N \N \N
3 22 \N \N 92.8 \N \N \N \N \N
-4 28 \N \N 89.1 123-456-7890 [email protected]
\N \N \N
+4 28 \N \N 89.09999999999999 123-456-7890
[email protected] \N \N \N
5 26 \N \N 91.3 223-345-132 [email protected]
{"city":"New York", "country":"USA"} \N \N
-6 \N \N \N 100.0 123-456-7890 [email protected]
{"city":"Los Angeles", "country":"USA"} \N \N
-7 \N \N \N 0.0 \N \N \N
3.140000104904175 123.45
+6 \N \N \N 100 123-456-7890 [email protected]
{"city":"Los Angeles", "country":"USA"} \N \N
+7 \N \N \N 0 \N \N \N
3.140000104904175 123.45
-- !rename_table_back_1 --
1 25 \N \N 95.5 \N \N \N \N \N
2 30 \N \N 87.2 \N \N \N \N \N
3 22 \N \N 92.8 \N \N \N \N \N
-4 28 \N \N 89.1 123-456-7890 [email protected]
\N \N \N
+4 28 \N \N 89.09999999999999 123-456-7890
[email protected] \N \N \N
5 26 \N \N 91.3 223-345-132 [email protected]
{"city":"New York", "country":"USA"} \N \N
-6 \N \N \N 100.0 123-456-7890 [email protected]
{"city":"Los Angeles", "country":"USA"} \N \N
-7 \N \N \N 0.0 \N \N \N
3.140000104904175 123.45
+6 \N \N \N 100 123-456-7890 [email protected]
{"city":"Los Angeles", "country":"USA"} \N \N
+7 \N \N \N 0 \N \N \N
3.140000104904175 123.45
-- !partition_init_1 --
id int Yes true \N
diff --git
a/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl.groovy
b/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl.groovy
index 33ce0648110..1762a088953 100644
---
a/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl.groovy
+++
b/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl.groovy
@@ -310,7 +310,7 @@ suite("iceberg_schema_change_ddl",
"p0,external,doris,external_docker,external_d
test {
sql """ ALTER TABLE ${table_name} MODIFY COLUMN email STRUCT<name:
STRING> """
- exception "Modify column type to non-primitive type is not supported"
+ exception "Modify column type from non-complex to complex is not
supported"
}
// Test 8: reorder columns
@@ -445,4 +445,3 @@ suite("iceberg_schema_change_ddl",
"p0,external,doris,external_docker,external_d
// Clean up
sql """ drop table if exists ${int_partition_table_name} """
}
-
diff --git
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_change_complex_types.groovy
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_change_complex_types.groovy
new file mode 100644
index 00000000000..da36702e770
--- /dev/null
+++
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_change_complex_types.groovy
@@ -0,0 +1,232 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_iceberg_schema_change_complex_types",
"p0,external,doris,external_docker,external_docker_doris") {
+
+ String enabled = context.config.otherConfigs.get("enableIcebergTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disable iceberg test.")
+ return
+ }
+
+ String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port")
+ String minio_port = context.config.otherConfigs.get("iceberg_minio_port")
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String catalog_name = "test_iceberg_schema_change_complex_types"
+ String db_name = "iceberg_schema_change_complex_types_db"
+ String table_name = "iceberg_complex_modify"
+
+ sql """drop catalog if exists ${catalog_name}"""
+ sql """
+ CREATE CATALOG ${catalog_name} PROPERTIES (
+ 'type'='iceberg',
+ 'iceberg.catalog.type'='rest',
+ 'uri' = 'http://${externalEnvIp}:${rest_port}',
+ "s3.access_key" = "admin",
+ "s3.secret_key" = "password",
+ "s3.endpoint" = "http://${externalEnvIp}:${minio_port}";,
+ "s3.region" = "us-east-1"
+ );"""
+
+ sql """switch ${catalog_name};"""
+ sql """drop database if exists ${db_name} force"""
+ sql """create database ${db_name}"""
+ sql """use ${db_name};"""
+
+ sql """set enable_fallback_to_original_planner=false;"""
+ sql """set show_column_comment_in_describe=true;"""
+
+ sql """drop table if exists ${table_name}"""
+ sql """
+ CREATE TABLE ${table_name} (
+ id INT NOT NULL,
+ arr_i ARRAY<INT>,
+ arr_f ARRAY<FLOAT>,
+ mp_i MAP<INT, INT>,
+ mp_f MAP<INT, FLOAT>,
+ st STRUCT<
+ si: INT,
+ sf: FLOAT,
+ sv: STRING COMMENT 'v1',
+ sm: MAP<INT, INT> COMMENT 'm1',
+ sa: ARRAY<INT> COMMENT 'a1'
+ > NOT NULL COMMENT 'old struct'
+ );"""
+
+ sql """
+ INSERT INTO ${table_name} VALUES (
+ 1,
+ ARRAY(1, 2),
+ ARRAY(CAST(1.1 AS FLOAT)),
+ MAP(1, 10),
+ MAP(1, CAST(1.1 AS FLOAT)),
+ STRUCT(1, CAST(1.1 AS FLOAT), 'abc', MAP(1, 10), ARRAY(1, 2))
+ )"""
+
+ // Complex type default value only supports NULL
+ test {
+ sql """
+ ALTER TABLE ${table_name} MODIFY COLUMN st STRUCT<
+ si: INT,
+ sf: FLOAT,
+ sv: STRING COMMENT 'v1',
+ sm: MAP<INT, INT> COMMENT 'm1',
+ sa: ARRAY<INT> COMMENT 'a1'
+ > DEFAULT 'x'"""
+ exception "just support null"
+ }
+
+ // Cannot change nullable complex column to not null
+ test {
+ sql """ALTER TABLE ${table_name} MODIFY COLUMN arr_i ARRAY<INT> NOT
NULL"""
+ exception "Cannot change nullable column arr_i to not null"
+ }
+
+ // Map key type change is not allowed
+ test {
+ sql """ALTER TABLE ${table_name} MODIFY COLUMN mp_i MAP<BIGINT, INT>"""
+ exception "Cannot change MAP key type"
+ }
+
+ // Complex type category change is not allowed
+ test {
+ sql """ALTER TABLE ${table_name} MODIFY COLUMN arr_i MAP<INT, INT>"""
+ exception "Cannot change complex column type category"
+ }
+
+ // Struct field rename is not allowed
+ test {
+ sql """
+ ALTER TABLE ${table_name} MODIFY COLUMN st STRUCT<
+ si_rename: INT,
+ sf: FLOAT,
+ sv: STRING COMMENT 'v1',
+ sm: MAP<INT, INT> COMMENT 'm1',
+ sa: ARRAY<INT> COMMENT 'a1'
+ > NOT NULL COMMENT 'old struct'"""
+ exception "Cannot rename struct field"
+ }
+
+ // Struct field drop is not allowed
+ test {
+ sql """
+ ALTER TABLE ${table_name} MODIFY COLUMN st STRUCT<
+ si: INT,
+ sf: FLOAT,
+ sv: STRING COMMENT 'v1',
+ sm: MAP<INT, INT> COMMENT 'm1'
+ > NOT NULL COMMENT 'old struct'"""
+ exception "Cannot reduce struct fields"
+ }
+
+ // Nested type downgrade is not allowed
+ test {
+ sql """ALTER TABLE ${table_name} MODIFY COLUMN arr_i ARRAY<SMALLINT>"""
+ exception "Cannot change int to smallint in nested types"
+ }
+
+ // Array element type promotions
+ sql """ALTER TABLE ${table_name} MODIFY COLUMN arr_i ARRAY<BIGINT>"""
+ sql """ALTER TABLE ${table_name} MODIFY COLUMN arr_f ARRAY<DOUBLE>"""
+
+ // Map value type promotions
+ sql """ALTER TABLE ${table_name} MODIFY COLUMN mp_i MAP<INT, BIGINT>"""
+ sql """ALTER TABLE ${table_name} MODIFY COLUMN mp_f MAP<INT, DOUBLE>"""
+
+ // Struct field promotions, nested complex updates, add field, and comment
updates
+ sql """
+ ALTER TABLE ${table_name} MODIFY COLUMN st STRUCT<
+ si: BIGINT,
+ sf: DOUBLE,
+ sv: STRING COMMENT 'v2',
+ sm: MAP<INT, BIGINT> COMMENT 'm2',
+ sa: ARRAY<BIGINT> COMMENT 'a2',
+ sn: STRING COMMENT 'new field',
+ sn_map: MAP<INT, BIGINT> COMMENT 'new map',
+ sn_arr: ARRAY<BIGINT> COMMENT 'new array'
+ > NULL COMMENT 'new struct'"""
+
+ // Insert new data after schema change and keep old data readable
+ sql """
+ INSERT INTO ${table_name} VALUES (
+ 2,
+ ARRAY(10, 20, 30),
+ ARRAY(CAST(2.2 AS DOUBLE)),
+ MAP(2, 200),
+ MAP(2, CAST(2.2 AS DOUBLE)),
+ STRUCT(
+ 2,
+ CAST(2.2 AS DOUBLE),
+ 'xyz',
+ MAP(2, 200),
+ ARRAY(10, 20),
+ 'new field value',
+ MAP(3, 300),
+ ARRAY(30, 40)
+ )
+ )"""
+
+ def descRows = sql """DESC ${table_name}"""
+ def normalizeType = { String s -> s.toLowerCase().replaceAll("\\s+", "") }
+ def typeOf = { String name ->
+ def row = descRows.find { it[0].toString().equalsIgnoreCase(name) }
+ assertTrue(row != null, "column not found: ${name}")
+ return normalizeType(row[1].toString())
+ }
+
+ assertTrue(typeOf("arr_i").contains("array<bigint>"), descRows.toString())
+ assertTrue(typeOf("arr_f").contains("array<double>"), descRows.toString())
+ assertTrue(typeOf("mp_i").contains("map<int,bigint>"), descRows.toString())
+ assertTrue(typeOf("mp_f").contains("map<int,double>"), descRows.toString())
+
+ def stType = typeOf("st")
+ assertTrue(stType.contains("si:bigint"), stType)
+ assertTrue(stType.contains("sf:double"), stType)
+ assertTrue(stType.contains("sv:text"), stType)
+ assertTrue(stType.contains("sm:map<int,bigint>"), stType)
+ assertTrue(stType.contains("sa:array<bigint>"), stType)
+ assertTrue(stType.contains("sn:text"), stType)
+ assertTrue(stType.contains("sn_map:map<int,bigint>"), stType)
+ assertTrue(stType.contains("sn_arr:array<bigint>"), stType)
+
+ def stRow = descRows.find { it[0].toString().equalsIgnoreCase("st") }
+ assertTrue(stRow.toString().toLowerCase().contains("new struct"),
stRow.toString())
+ assertTrue(stRow[2].toString().equalsIgnoreCase("YES"), stRow.toString())
+
+ // Reorder columns should still work after complex type changes
+ sql """ALTER TABLE ${table_name} ORDER BY (id, arr_i, mp_i, st, arr_f,
mp_f)"""
+
+ def queryRes = sql """SELECT id,
+ STRUCT_ELEMENT(st, 'si') AS si,
+ STRUCT_ELEMENT(st, 'sn') AS sn,
+ ARRAY_SIZE(arr_i) AS arr_sz
+ FROM ${table_name} ORDER BY id"""
+ assertTrue(queryRes.size() == 2, queryRes.toString())
+ assertTrue(queryRes[0][0].toString() == "1", queryRes.toString())
+ assertTrue(queryRes[0][1].toString() == "1", queryRes.toString())
+ assertTrue(queryRes[0][2] == null, queryRes.toString())
+ assertTrue(queryRes[0][3].toString() == "2", queryRes.toString())
+
+ assertTrue(queryRes[1][0].toString() == "2", queryRes.toString())
+ assertTrue(queryRes[1][1].toString() == "2", queryRes.toString())
+ assertTrue(queryRes[1][2].toString() == "new field value",
queryRes.toString())
+ assertTrue(queryRes[1][3].toString() == "3", queryRes.toString())
+
+ sql """drop table if exists ${table_name}"""
+ sql """drop database if exists ${db_name} force"""
+ sql """drop catalog if exists ${catalog_name}"""
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
