This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 5899bf81e2 [test] Add more tests for schema update
5899bf81e2 is described below
commit 5899bf81e2b2a122daad126419d4f380e79263e4
Author: JingsongLi <[email protected]>
AuthorDate: Tue May 26 18:23:49 2026 +0800
[test] Add more tests for schema update
---
.../paimon/schema/NestedSchemaUtilsTest.java | 240 +++++++++++++++++++++
.../apache/paimon/table/SchemaEvolutionTest.java | 172 +++++++++++++++
2 files changed, 412 insertions(+)
diff --git
a/paimon-core/src/test/java/org/apache/paimon/schema/NestedSchemaUtilsTest.java
b/paimon-core/src/test/java/org/apache/paimon/schema/NestedSchemaUtilsTest.java
index ae5f7ada39..c5beadc75a 100644
---
a/paimon-core/src/test/java/org/apache/paimon/schema/NestedSchemaUtilsTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/schema/NestedSchemaUtilsTest.java
@@ -85,6 +85,35 @@ public class NestedSchemaUtilsTest {
assertThat(nullabilityChange.newNullability()).isFalse();
}
+ @Test
+ public void testNullabilityChangeNotNullToNullable() {
+ List<String> fieldNames = Arrays.asList("column1");
+ List<SchemaChange> schemaChanges = new ArrayList<>();
+
+ // Test non-nullable to nullable
+ NestedSchemaUtils.generateNestedColumnUpdates(
+ fieldNames, DataTypes.INT().notNull(),
DataTypes.INT().nullable(), schemaChanges);
+
+ assertThat(schemaChanges).hasSize(1);
+
assertThat(schemaChanges.get(0)).isInstanceOf(SchemaChange.UpdateColumnNullability.class);
+ SchemaChange.UpdateColumnNullability nullabilityChange =
+ (SchemaChange.UpdateColumnNullability) schemaChanges.get(0);
+ assertThat(nullabilityChange.fieldNames()).containsExactly("column1");
+ assertThat(nullabilityChange.newNullability()).isTrue();
+ }
+
+ @Test
+ public void testSameNullabilityNoChange() {
+ List<String> fieldNames = Arrays.asList("column1");
+ List<SchemaChange> schemaChanges = new ArrayList<>();
+
+ // Same nullability should not generate nullability change
+ NestedSchemaUtils.generateNestedColumnUpdates(
+ fieldNames, DataTypes.INT().nullable(),
DataTypes.INT().nullable(), schemaChanges);
+
+ assertThat(schemaChanges).isEmpty();
+ }
+
@Test
public void testTypeAndNullabilityChange() {
List<String> fieldNames = Arrays.asList("column1");
@@ -265,6 +294,217 @@ public class NestedSchemaUtilsTest {
.hasMessageContaining("can only be updated to multiset type");
}
+ @Test
+ public void testMultisetTypeUpdateNullability() {
+ List<String> fieldNames = Arrays.asList("multiset_column");
+ List<SchemaChange> schemaChanges = new ArrayList<>();
+
+ MultisetType oldType = new MultisetType(true, DataTypes.INT());
+ MultisetType newType = new MultisetType(false, DataTypes.INT());
+
+ NestedSchemaUtils.generateNestedColumnUpdates(fieldNames, oldType,
newType, schemaChanges);
+
+ assertThat(schemaChanges).hasSize(1);
+
assertThat(schemaChanges.get(0)).isInstanceOf(SchemaChange.UpdateColumnNullability.class);
+ SchemaChange.UpdateColumnNullability nullabilityChange =
+ (SchemaChange.UpdateColumnNullability) schemaChanges.get(0);
+
assertThat(nullabilityChange.fieldNames()).containsExactly("multiset_column");
+ assertThat(nullabilityChange.newNullability()).isFalse();
+ }
+
+ @Test
+ public void testArrayElementNullabilityChange() {
+ List<String> fieldNames = Arrays.asList("arr_column");
+ List<SchemaChange> schemaChanges = new ArrayList<>();
+
+ // Array element changes from nullable INT to non-nullable INT
+ ArrayType oldType = new ArrayType(true, DataTypes.INT().nullable());
+ ArrayType newType = new ArrayType(true, DataTypes.INT().notNull());
+
+ NestedSchemaUtils.generateNestedColumnUpdates(fieldNames, oldType,
newType, schemaChanges);
+
+ assertThat(schemaChanges).hasSize(1);
+
assertThat(schemaChanges.get(0)).isInstanceOf(SchemaChange.UpdateColumnNullability.class);
+ SchemaChange.UpdateColumnNullability nullabilityChange =
+ (SchemaChange.UpdateColumnNullability) schemaChanges.get(0);
+
assertThat(nullabilityChange.fieldNames()).containsExactly("arr_column",
"element");
+ assertThat(nullabilityChange.newNullability()).isFalse();
+ }
+
+ @Test
+ public void testArrayElementTypeAndNullabilityChange() {
+ List<String> fieldNames = Arrays.asList("arr_column");
+ List<SchemaChange> schemaChanges = new ArrayList<>();
+
+ // Array element changes from nullable INT to non-nullable BIGINT
+ ArrayType oldType = new ArrayType(true, DataTypes.INT().nullable());
+ ArrayType newType = new ArrayType(true, DataTypes.BIGINT().notNull());
+
+ NestedSchemaUtils.generateNestedColumnUpdates(fieldNames, oldType,
newType, schemaChanges);
+
+ assertThat(schemaChanges).hasSize(2);
+ assertThat(schemaChanges)
+ .anyMatch(change -> change instanceof
SchemaChange.UpdateColumnType);
+ assertThat(schemaChanges)
+ .anyMatch(change -> change instanceof
SchemaChange.UpdateColumnNullability);
+
+ SchemaChange.UpdateColumnType typeChange =
+ schemaChanges.stream()
+ .filter(c -> c instanceof
SchemaChange.UpdateColumnType)
+ .map(c -> (SchemaChange.UpdateColumnType) c)
+ .findFirst()
+ .orElse(null);
+ assertThat(typeChange).isNotNull();
+ assertThat(typeChange.fieldNames()).containsExactly("arr_column",
"element");
+
+ SchemaChange.UpdateColumnNullability nullabilityChange =
+ schemaChanges.stream()
+ .filter(c -> c instanceof
SchemaChange.UpdateColumnNullability)
+ .map(c -> (SchemaChange.UpdateColumnNullability) c)
+ .findFirst()
+ .orElse(null);
+ assertThat(nullabilityChange).isNotNull();
+
assertThat(nullabilityChange.fieldNames()).containsExactly("arr_column",
"element");
+ assertThat(nullabilityChange.newNullability()).isFalse();
+ }
+
+ @Test
+ public void testArrayNullabilityAndElementNullabilityChange() {
+ List<String> fieldNames = Arrays.asList("arr_column");
+ List<SchemaChange> schemaChanges = new ArrayList<>();
+
+ // Both array-level and element-level nullability change
+ ArrayType oldType = new ArrayType(true, DataTypes.INT().nullable());
+ ArrayType newType = new ArrayType(false, DataTypes.INT().notNull());
+
+ NestedSchemaUtils.generateNestedColumnUpdates(fieldNames, oldType,
newType, schemaChanges);
+
+ assertThat(schemaChanges).hasSize(2);
+
+ // element-level nullability change
+ SchemaChange.UpdateColumnNullability elementNullability =
+ schemaChanges.stream()
+ .filter(c -> c instanceof
SchemaChange.UpdateColumnNullability)
+ .map(c -> (SchemaChange.UpdateColumnNullability) c)
+ .filter(c -> c.fieldNames().length == 2)
+ .findFirst()
+ .orElse(null);
+ assertThat(elementNullability).isNotNull();
+
assertThat(elementNullability.fieldNames()).containsExactly("arr_column",
"element");
+ assertThat(elementNullability.newNullability()).isFalse();
+
+ // array-level nullability change
+ SchemaChange.UpdateColumnNullability arrayNullability =
+ schemaChanges.stream()
+ .filter(c -> c instanceof
SchemaChange.UpdateColumnNullability)
+ .map(c -> (SchemaChange.UpdateColumnNullability) c)
+ .filter(c -> c.fieldNames().length == 1)
+ .findFirst()
+ .orElse(null);
+ assertThat(arrayNullability).isNotNull();
+
assertThat(arrayNullability.fieldNames()).containsExactly("arr_column");
+ assertThat(arrayNullability.newNullability()).isFalse();
+ }
+
+ @Test
+ public void testMapValueNullabilityChange() {
+ List<String> fieldNames = Arrays.asList("map_column");
+ List<SchemaChange> schemaChanges = new ArrayList<>();
+
+ // Map value changes from nullable to non-nullable
+ MapType oldType = new MapType(true, DataTypes.STRING(),
DataTypes.INT().nullable());
+ MapType newType = new MapType(true, DataTypes.STRING(),
DataTypes.INT().notNull());
+
+ NestedSchemaUtils.generateNestedColumnUpdates(fieldNames, oldType,
newType, schemaChanges);
+
+ assertThat(schemaChanges).hasSize(1);
+
assertThat(schemaChanges.get(0)).isInstanceOf(SchemaChange.UpdateColumnNullability.class);
+ SchemaChange.UpdateColumnNullability nullabilityChange =
+ (SchemaChange.UpdateColumnNullability) schemaChanges.get(0);
+
assertThat(nullabilityChange.fieldNames()).containsExactly("map_column",
"value");
+ assertThat(nullabilityChange.newNullability()).isFalse();
+ }
+
+ @Test
+ public void testMapValueTypeAndNullabilityChange() {
+ List<String> fieldNames = Arrays.asList("map_column");
+ List<SchemaChange> schemaChanges = new ArrayList<>();
+
+ // Map value changes both type and nullability
+ MapType oldType = new MapType(true, DataTypes.STRING(),
DataTypes.INT().nullable());
+ MapType newType = new MapType(true, DataTypes.STRING(),
DataTypes.BIGINT().notNull());
+
+ NestedSchemaUtils.generateNestedColumnUpdates(fieldNames, oldType,
newType, schemaChanges);
+
+ assertThat(schemaChanges).hasSize(2);
+ assertThat(schemaChanges)
+ .anyMatch(change -> change instanceof
SchemaChange.UpdateColumnType);
+ assertThat(schemaChanges)
+ .anyMatch(change -> change instanceof
SchemaChange.UpdateColumnNullability);
+
+ SchemaChange.UpdateColumnType typeChange =
+ schemaChanges.stream()
+ .filter(c -> c instanceof
SchemaChange.UpdateColumnType)
+ .map(c -> (SchemaChange.UpdateColumnType) c)
+ .findFirst()
+ .orElse(null);
+ assertThat(typeChange).isNotNull();
+ assertThat(typeChange.fieldNames()).containsExactly("map_column",
"value");
+
+ SchemaChange.UpdateColumnNullability nullabilityChange =
+ schemaChanges.stream()
+ .filter(c -> c instanceof
SchemaChange.UpdateColumnNullability)
+ .map(c -> (SchemaChange.UpdateColumnNullability) c)
+ .findFirst()
+ .orElse(null);
+ assertThat(nullabilityChange).isNotNull();
+
assertThat(nullabilityChange.fieldNames()).containsExactly("map_column",
"value");
+ assertThat(nullabilityChange.newNullability()).isFalse();
+ }
+
+ @Test
+ public void testMultisetElementNullabilityChange() {
+ List<String> fieldNames = Arrays.asList("multiset_column");
+ List<SchemaChange> schemaChanges = new ArrayList<>();
+
+ // Multiset element changes from nullable to non-nullable
+ MultisetType oldType = new MultisetType(true,
DataTypes.INT().nullable());
+ MultisetType newType = new MultisetType(true,
DataTypes.INT().notNull());
+
+ NestedSchemaUtils.generateNestedColumnUpdates(fieldNames, oldType,
newType, schemaChanges);
+
+ assertThat(schemaChanges).hasSize(1);
+
assertThat(schemaChanges.get(0)).isInstanceOf(SchemaChange.UpdateColumnNullability.class);
+ SchemaChange.UpdateColumnNullability nullabilityChange =
+ (SchemaChange.UpdateColumnNullability) schemaChanges.get(0);
+
assertThat(nullabilityChange.fieldNames()).containsExactly("multiset_column",
"element");
+ assertThat(nullabilityChange.newNullability()).isFalse();
+ }
+
+ @Test
+ public void testRowFieldNullabilityChange() {
+ List<String> fieldNames = Arrays.asList("row_column");
+ List<SchemaChange> schemaChanges = new ArrayList<>();
+
+ RowType oldType =
+ RowType.of(
+ new DataField(0, "f1", DataTypes.INT().nullable()),
+ new DataField(1, "f2", DataTypes.STRING()));
+ RowType newType =
+ RowType.of(
+ new DataField(0, "f1", DataTypes.INT().notNull()),
+ new DataField(1, "f2", DataTypes.STRING()));
+
+ NestedSchemaUtils.generateNestedColumnUpdates(fieldNames, oldType,
newType, schemaChanges);
+
+ assertThat(schemaChanges).hasSize(1);
+
assertThat(schemaChanges.get(0)).isInstanceOf(SchemaChange.UpdateColumnNullability.class);
+ SchemaChange.UpdateColumnNullability nullabilityChange =
+ (SchemaChange.UpdateColumnNullability) schemaChanges.get(0);
+
assertThat(nullabilityChange.fieldNames()).containsExactly("row_column", "f1");
+ assertThat(nullabilityChange.newNullability()).isFalse();
+ }
+
@Test
public void testRowTypeAddField() {
List<String> fieldNames = Arrays.asList("row_column");
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/SchemaEvolutionTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/SchemaEvolutionTest.java
index 8b118e0562..58d82b0bc9 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/SchemaEvolutionTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/SchemaEvolutionTest.java
@@ -243,6 +243,178 @@ public class SchemaEvolutionTest {
.hasMessageContaining("Cannot update primary key");
}
+ @Test
+ public void testUpdateColumnNullability() throws Exception {
+ Schema schema =
+ Schema.newBuilder()
+ .column("f0", DataTypes.INT())
+ .column("f1", DataTypes.BIGINT())
+ .build();
+ schemaManager.createTable(schema);
+
+ // nullable -> NOT NULL (disabled by default)
+ assertThatThrownBy(
+ () ->
+ schemaManager.commitChanges(
+ Collections.singletonList(
+
SchemaChange.updateColumnNullability("f0", false))))
+ .isInstanceOf(UnsupportedOperationException.class)
+ .hasMessageContaining("nullable to non nullable");
+
+ // enable null-to-not-null
+ schemaManager.commitChanges(
+ Collections.singletonList(
+ SchemaChange.setOption(
+
CoreOptions.DISABLE_ALTER_COLUMN_NULL_TO_NOT_NULL.key(), "false")));
+
+ // nullable -> NOT NULL
+ TableSchema tableSchema =
+ schemaManager.commitChanges(
+ Collections.singletonList(
+ SchemaChange.updateColumnNullability("f0",
false)));
+ assertThat(tableSchema.fields().get(0).type().isNullable()).isFalse();
+
+ // NOT NULL -> nullable
+ tableSchema =
+ schemaManager.commitChanges(
+ Collections.singletonList(
+ SchemaChange.updateColumnNullability("f0",
true)));
+ assertThat(tableSchema.fields().get(0).type().isNullable()).isTrue();
+ }
+
+ @Test
+ public void testUpdatePrimaryKeyNullability() throws Exception {
+ Schema schema =
+ Schema.newBuilder()
+ .column("k", DataTypes.INT())
+ .column("v", DataTypes.BIGINT())
+ .primaryKey("k")
+ .build();
+ schemaManager.createTable(schema);
+
+ List<SchemaChange> changes =
+
Collections.singletonList(SchemaChange.updateColumnNullability("k", true));
+ assertThatThrownBy(() -> schemaManager.commitChanges(changes))
+ .hasMessageContaining("Cannot change nullability of primary
key");
+ }
+
+ @Test
+ public void testUpdateFieldTypeWithNullabilityChange() throws Exception {
+ Schema schema =
+ Schema.newBuilder()
+ .column("f0", DataTypes.INT())
+ .column("f1", DataTypes.BIGINT())
+ .build();
+ schemaManager.createTable(schema);
+
+ // enable null-to-not-null
+ schemaManager.commitChanges(
+ Collections.singletonList(
+ SchemaChange.setOption(
+
CoreOptions.DISABLE_ALTER_COLUMN_NULL_TO_NOT_NULL.key(), "false")));
+
+ // updateColumnType with keepNullability=true should preserve
nullability
+ TableSchema tableSchema =
+ schemaManager.commitChanges(
+ Collections.singletonList(
+ SchemaChange.updateColumnType(
+ "f0", DataTypes.BIGINT().notNull(),
true)));
+
assertThat(tableSchema.fields().get(0).type()).isEqualTo(DataTypes.BIGINT());
+ assertThat(tableSchema.fields().get(0).type().isNullable()).isTrue();
+
+ // updateColumnType with keepNullability=false should update
nullability
+ tableSchema =
+ schemaManager.commitChanges(
+ Collections.singletonList(
+ SchemaChange.updateColumnType(
+ "f0", DataTypes.DOUBLE().notNull(),
false)));
+
assertThat(tableSchema.fields().get(0).type()).isEqualTo(DataTypes.DOUBLE().notNull());
+ assertThat(tableSchema.fields().get(0).type().isNullable()).isFalse();
+ }
+
+ @Test
+ public void testUpdateNestedColumnNullability() throws Exception {
+ RowType innerType =
+ RowType.of(
+ new DataField(2, "f1", DataTypes.INT()),
+ new DataField(3, "f2", DataTypes.BIGINT()));
+ RowType outerType =
+ RowType.of(
+ new DataField(0, "k", DataTypes.INT()), new
DataField(1, "v", innerType));
+
+ Schema schema =
+ new Schema(
+ outerType.getFields(),
+ Collections.singletonList("k"),
+ Collections.emptyList(),
+ new HashMap<>(),
+ "");
+ schemaManager.createTable(schema);
+
+ // enable null-to-not-null
+ schemaManager.commitChanges(
+ Collections.singletonList(
+ SchemaChange.setOption(
+
CoreOptions.DISABLE_ALTER_COLUMN_NULL_TO_NOT_NULL.key(), "false")));
+
+ // update nested field nullability
+ TableSchema tableSchema =
+ schemaManager.commitChanges(
+ Collections.singletonList(
+ SchemaChange.updateColumnNullability(
+ new String[] {"v", "f1"}, false)));
+ RowType resultType = tableSchema.logicalRowType();
+ RowType nestedType = (RowType) resultType.getFields().get(1).type();
+
assertThat(nestedType.getFields().get(0).type().isNullable()).isFalse();
+ assertThat(nestedType.getFields().get(1).type().isNullable()).isTrue();
+ }
+
+ @Test
+ public void testUpdateColumnNullabilityWithData() throws Exception {
+ Schema schema =
+ new Schema(
+ RowType.of(DataTypes.INT(),
DataTypes.BIGINT()).getFields(),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ new HashMap<>(),
+ "");
+ schemaManager.createTable(schema);
+
+ FileStoreTable table =
FileStoreTableFactory.create(LocalFileIO.create(), tablePath);
+
+ StreamTableWrite write = table.newWrite(commitUser);
+ write.write(GenericRow.of(1, 1L));
+ write.write(GenericRow.of(2, null));
+ TableCommitImpl commit = table.newCommit(commitUser);
+ commit.commit(0, write.prepareCommit(true, 0));
+ write.close();
+ commit.close();
+
+ // enable null-to-not-null
+ schemaManager.commitChanges(
+ SchemaChange.setOption(
+
CoreOptions.DISABLE_ALTER_COLUMN_NULL_TO_NOT_NULL.key(), "false"));
+ // change f0 to NOT NULL
+ schemaManager.commitChanges(SchemaChange.updateColumnNullability("f0",
false));
+
+ table = FileStoreTableFactory.create(LocalFileIO.create(), tablePath);
+
+ // data written before nullability change should still be readable
+ List<String> rows = readRecords(table, null);
+ assertThat(rows).containsExactlyInAnyOrder("1, 1", "2, NULL");
+
+ // write new data after nullability change
+ write = table.newWrite(commitUser);
+ write.write(GenericRow.of(3, 3L));
+ commit = table.newCommit(commitUser);
+ commit.commit(1, write.prepareCommit(true, 1));
+ write.close();
+ commit.close();
+
+ rows = readRecords(table, null);
+ assertThat(rows).containsExactlyInAnyOrder("1, 1", "2, NULL", "3, 3");
+ }
+
@Test
public void testRenameField() throws Exception {
Schema schema =
