This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new eb44f08c5 [common] Fix can not create table by java api with ROW data
type (#1547)
eb44f08c5 is described below
commit eb44f08c5d695ff7ce8dc8e07dfa6647fb19a2a3
Author: Kerwin <[email protected]>
AuthorDate: Wed Jul 19 13:40:09 2023 +0800
[common] Fix can not create table by java api with ROW data type (#1547)
---
.../main/java/org/apache/paimon/schema/Schema.java | 11 +++-
.../org/apache/paimon/catalog/CatalogTestBase.java | 19 ++++++-
.../apache/paimon/schema/SchemaBuilderTest.java | 21 ++++++-
.../org/apache/paimon/schema/TableSchemaTest.java | 66 +++++++++++++++++-----
4 files changed, 97 insertions(+), 20 deletions(-)
diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/Schema.java
b/paimon-core/src/main/java/org/apache/paimon/schema/Schema.java
index cecec0f58..85fee3d56 100644
--- a/paimon-core/src/main/java/org/apache/paimon/schema/Schema.java
+++ b/paimon-core/src/main/java/org/apache/paimon/schema/Schema.java
@@ -21,6 +21,7 @@ package org.apache.paimon.schema;
import org.apache.paimon.annotation.Public;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.ReassignFieldId;
import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.Preconditions;
@@ -35,6 +36,7 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
/**
@@ -216,10 +218,10 @@ public class Schema {
@Nullable private String comment;
- private int highestFieldId = -1;
+ private final AtomicInteger highestFieldId = new AtomicInteger(-1);
public int getHighestFieldId() {
- return highestFieldId;
+ return highestFieldId.get();
}
/**
@@ -242,7 +244,10 @@ public class Schema {
public Builder column(String columnName, DataType dataType, @Nullable
String description) {
Preconditions.checkNotNull(columnName, "Column name must not be
null.");
Preconditions.checkNotNull(dataType, "Data type must not be
null.");
- columns.add(new DataField(++highestFieldId, columnName, dataType,
description));
+
+ int id = highestFieldId.incrementAndGet();
+ DataType reassignDataType = ReassignFieldId.reassign(dataType,
highestFieldId);
+ columns.add(new DataField(id, columnName, reassignDataType,
description));
return this;
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java
b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java
index 9c71fb57d..12a3e6b27 100644
--- a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java
+++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java
@@ -208,7 +208,24 @@ public abstract class CatalogTestBase {
catalog.createDatabase("test_db", false);
// Create table creates a new table when it does not exist
Identifier identifier = Identifier.create("test_db", "new_table");
- catalog.createTable(identifier, DEFAULT_TABLE_SCHEMA, false);
+ Schema schema =
+ Schema.newBuilder()
+ .column("pk1", DataTypes.INT())
+ .column("pk2", DataTypes.STRING())
+ .column("pk3", DataTypes.STRING())
+ .column(
+ "col1",
+ DataTypes.ROW(
+ DataTypes.STRING(),
+ DataTypes.BIGINT(),
+ DataTypes.TIMESTAMP(),
+ DataTypes.ARRAY(DataTypes.STRING())))
+ .column("col2", DataTypes.MAP(DataTypes.STRING(),
DataTypes.BIGINT()))
+ .column("col3",
DataTypes.ARRAY(DataTypes.ROW(DataTypes.STRING())))
+ .partitionKeys("pk1", "pk2")
+ .primaryKey("pk1", "pk2", "pk3")
+ .build();
+ catalog.createTable(identifier, schema, false);
boolean exists = catalog.tableExists(identifier);
assertThat(exists).isTrue();
diff --git
a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaBuilderTest.java
b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaBuilderTest.java
index 864b77b67..881cffb87 100644
--- a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaBuilderTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaBuilderTest.java
@@ -23,7 +23,8 @@ import org.apache.paimon.types.DataTypes;
import org.junit.jupiter.api.Test;
-import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
/** Test for {@link Schema.Builder}. */
public class SchemaBuilderTest {
@@ -64,4 +65,22 @@ public class SchemaBuilderTest {
"Partition key constraint [id, id] must not
contain duplicate columns. Found: [id]"
+ ""));
}
+
+ @Test
+ public void testHighestFieldId() {
+ Schema.Builder builder =
+ Schema.newBuilder()
+ .column("id", DataTypes.INT())
+ .column(
+ "col1",
+ DataTypes.ROW(
+ DataTypes.STRING(),
+ DataTypes.INT(),
+ DataTypes.ARRAY(DataTypes.STRING())))
+ .column("col2", DataTypes.MAP(DataTypes.STRING(),
DataTypes.STRING()))
+ .column("col3",
DataTypes.ARRAY(DataTypes.ROW(DataTypes.STRING())))
+ .partitionKeys("id")
+ .partitionKeys("id");
+ assertThat(builder.getHighestFieldId()).isEqualTo(7);
+ }
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/schema/TableSchemaTest.java
b/paimon-core/src/test/java/org/apache/paimon/schema/TableSchemaTest.java
index b85faabc6..2d39a4c57 100644
--- a/paimon-core/src/test/java/org/apache/paimon/schema/TableSchemaTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/schema/TableSchemaTest.java
@@ -19,10 +19,9 @@
package org.apache.paimon.schema;
import org.apache.paimon.types.DataField;
-import org.apache.paimon.types.IntType;
+import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.RowType;
-import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.util.Arrays;
@@ -32,6 +31,7 @@ import java.util.List;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
/** Test for {@link TableSchema}. */
public class TableSchemaTest {
@@ -40,40 +40,76 @@ public class TableSchemaTest {
public void testInvalidPrimaryKeys() {
List<DataField> fields =
Arrays.asList(
- new DataField(0, "f0", new IntType()),
- new DataField(1, "f1", new IntType()),
- new DataField(2, "f2", new IntType()));
+ new DataField(0, "f0", DataTypes.INT()),
+ new DataField(1, "f1", DataTypes.INT()),
+ new DataField(2, "f2", DataTypes.INT()));
List<String> partitionKeys = Collections.singletonList("f0");
List<String> primaryKeys = Collections.singletonList("f1");
Map<String, String> options = new HashMap<>();
- Assertions.assertThrows(
- IllegalStateException.class,
- () -> new TableSchema(1, fields, 10, partitionKeys,
primaryKeys, options, ""));
+ assertThatThrownBy(
+ () ->
+ new TableSchema(
+ 1, fields, 10, partitionKeys,
primaryKeys, options, ""))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessage("Primary key constraint [f1] should include all
partition fields [f0]");
}
@Test
public void testInvalidFieldIds() {
List<DataField> fields =
Arrays.asList(
- new DataField(0, "f0", new IntType()),
- new DataField(0, "f1", new IntType()));
- Assertions.assertThrows(
- RuntimeException.class, () ->
RowType.currentHighestFieldId(fields));
+ new DataField(0, "f0", DataTypes.INT()),
+ new DataField(0, "f1", DataTypes.INT()));
+
+ assertThatThrownBy(() -> RowType.currentHighestFieldId(fields))
+ .isInstanceOf(RuntimeException.class)
+ .hasMessage("Broken schema, field id 0 is duplicated.");
}
@Test
public void testHighestFieldId() {
List<DataField> fields =
Arrays.asList(
- new DataField(0, "f0", new IntType()),
- new DataField(20, "f1", new IntType()));
+ new DataField(0, "f0", DataTypes.INT()),
+ new DataField(20, "f1", DataTypes.INT()));
assertThat(RowType.currentHighestFieldId(fields)).isEqualTo(20);
+
+ List<DataField> fields1 =
+ Arrays.asList(
+ new DataField(0, "f0", DataTypes.INT()),
+ new DataField(
+ 1,
+ "f1",
+ DataTypes.ROW(
+ new DataField(2, "f0",
DataTypes.STRING()),
+ new DataField(3, "f1",
DataTypes.ARRAY(DataTypes.INT())))),
+ new DataField(4, "f2", DataTypes.STRING()),
+ new DataField(
+ 5,
+ "f3",
+ DataTypes.ARRAY(
+ DataTypes.ROW(
+ new DataField(6, "f0",
DataTypes.BIGINT())))));
+ assertThat(RowType.currentHighestFieldId(fields1)).isEqualTo(6);
+
+ List<DataField> fields2 =
+ Arrays.asList(
+ new DataField(0, "f0", DataTypes.INT()),
+ new DataField(
+ 1,
+ "f1",
+ DataTypes.ROW(
+ DataTypes.STRING(),
DataTypes.ARRAY(DataTypes.INT()))),
+ new DataField(2, "f2", DataTypes.STRING()));
+ assertThatThrownBy(() -> RowType.currentHighestFieldId(fields2))
+ .isInstanceOf(RuntimeException.class)
+ .hasMessage("Broken schema, field id 0 is duplicated.");
}
static RowType newRowType(boolean isNullable, int fieldId) {
return new RowType(
isNullable,
- Collections.singletonList(new DataField(fieldId,
"nestedField", new IntType())));
+ Collections.singletonList(new DataField(fieldId,
"nestedField", DataTypes.INT())));
}
}
