This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new f41f35695a [iceberg] Avoid dummy column name clash when creating
partitioned REST tables (#10561)
f41f35695a is described below
commit f41f35695a8dd4d70de6c05a6d8706176e6f208a
Author: Eunbin Son <[email protected]>
AuthorDate: Fri Oct 16 09:51:42 2026 +0900
[iceberg] Avoid dummy column name clash when creating partitioned REST
tables (#10561)
---
.../iceberg/IcebergRestMetadataCommitter.java | 14 ++++-
.../iceberg/IcebergRestMetadataCommitterTest.java | 67 ++++++++++++++++++++++
2 files changed, 78 insertions(+), 3 deletions(-)
diff --git
a/paimon-iceberg/src/main/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitter.java
b/paimon-iceberg/src/main/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitter.java
index e5bf3b67aa..3ac1cbee77 100644
---
a/paimon-iceberg/src/main/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitter.java
+++
b/paimon-iceberg/src/main/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitter.java
@@ -603,12 +603,20 @@ public class IcebergRestMetadataCommitter implements
IcebergMetadataCommitter {
int size =
spec.fields().stream().mapToInt(PartitionField::sourceId).max().orElseThrow();
- // prefill the schema with dummy fields
+ Set<String> partitionNames = new HashSet<>();
+ for (PartitionField f : spec.fields()) {
+
partitionNames.add(newMetadata.schema().findColumnName(f.sourceId()));
+ }
+ // prefill the schema with dummy fields, whose names must not
clash with the
+ // partition fields kept in this schema
NestedField[] columns = new NestedField[size];
for (int idx = 0; idx < size; idx++) {
int fieldId = idx + 1;
- columns[idx] =
- NestedField.optional(fieldId, "f" + fieldId,
Types.BooleanType.get());
+ String name = "f" + fieldId;
+ while (partitionNames.contains(name)) {
+ name = "_" + name;
+ }
+ columns[idx] = NestedField.optional(fieldId, name,
Types.BooleanType.get());
}
// find and set partition fields with offset -1, so they align
correctly after table
// creation
diff --git
a/paimon-iceberg/src/test/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitterTest.java
b/paimon-iceberg/src/test/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitterTest.java
index bb24cad57d..1d7742e28f 100644
---
a/paimon-iceberg/src/test/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitterTest.java
+++
b/paimon-iceberg/src/test/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitterTest.java
@@ -40,6 +40,7 @@ import org.apache.paimon.schema.SchemaManager;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.sink.TableCommitImpl;
import org.apache.paimon.table.sink.TableWriteImpl;
+import org.apache.paimon.types.DataField;
import org.apache.paimon.types.DataType;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.RowType;
@@ -312,6 +313,72 @@ public class IcebergRestMetadataCommitterTest {
runPartitionSpecCompatibilityTest(expectedPartitionSpec);
}
+ @Test
+ public void testPartitionNameClashingWithDummyFieldName() throws Exception
{
+ // the dummy schema names its placeholder for field id 1 "f1"
+ runDummyFieldNameClashTest(new String[] {"k", "v", "f1"},
Collections.singletonList("f1"));
+ }
+
+ @Test
+ public void testPartitionNamesClashingWithPrefixedDummyFieldName() throws
Exception {
+ runDummyFieldNameClashTest(
+ new String[] {"k", "v", "f1", "_f1"}, Arrays.asList("f1",
"_f1"));
+ }
+
+ private void runDummyFieldNameClashTest(String[] fieldNames, List<String>
partitionKeys)
+ throws Exception {
+ DataType[] fieldTypes = new DataType[fieldNames.length];
+ Arrays.fill(fieldTypes, DataTypes.STRING());
+ FileStoreTable table =
+ createPaimonTable(
+ RowType.of(fieldTypes, fieldNames),
+ partitionKeys,
+ Collections.emptyList(),
+ -1,
+ randomFormat(),
+ Collections.emptyMap());
+
+ String commitUser = UUID.randomUUID().toString();
+ TableWriteImpl<?> write = table.newWrite(commitUser);
+ TableCommitImpl commit = table.newCommit(commitUser);
+
+ List<String> expected = new ArrayList<>();
+ for (int r = 0; r < 2; r++) {
+ for (int i = 0; i < 2; i++) {
+ Object[] values = new Object[fieldNames.length];
+ List<String> strings = new ArrayList<>();
+ for (int c = 0; c < fieldNames.length; c++) {
+ String value = fieldNames[c] + "-" + r + "-" + i;
+ values[c] = BinaryString.fromString(value);
+ strings.add(value);
+ }
+ write.write(GenericRow.of(values));
+ expected.add("Record(" + String.join(", ", strings) + ")");
+ }
+ commit.commit(r, write.prepareCommit(true, r));
+ assertThat(getIcebergResult()).hasSameElementsAs(expected);
+ }
+ write.close();
+ commit.close();
+
+ // partition fields keep their Paimon field ids and no partition
evolution happens
+ NestedField[] partitionFields = new NestedField[partitionKeys.size()];
+ for (int i = 0; i < partitionKeys.size(); i++) {
+ DataField field = table.rowType().getField(partitionKeys.get(i));
+ partitionFields[i] =
+ NestedField.of(
+ field.id(),
+ !field.type().isNullable(),
+ field.name(),
+ Types.StringType.get());
+ }
+ PartitionSpec.Builder specBuilder = PartitionSpec.builderFor(new
Schema(partitionFields));
+ partitionKeys.forEach(specBuilder::identity);
+ PartitionSpec expectedSpec = specBuilder.build();
+ assertThat(expectedSpec.specId()).isEqualTo(0);
+ runPartitionSpecCompatibilityTest(expectedSpec);
+ }
+
private void runCompatibilityTest(
RowType rowType,
List<String> partitionKeys,
