This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new a288809ca9 [core] Support parsing row type json without field id
(#4876)
a288809ca9 is described below
commit a288809ca990a75574fb2b82ba37f62ad2b88341
Author: Zouxxyy <[email protected]>
AuthorDate: Fri Jan 10 14:15:55 2025 +0800
[core] Support parsing row type json without field id (#4876)
---
.../apache/paimon/types/DataTypeJsonParser.java | 32 +++++++++++----
.../main/java/org/apache/paimon/types/RowType.java | 6 ++-
.../paimon/schema/DataTypeJsonParserTest.java | 45 ++++++++++++++++++++--
3 files changed, 72 insertions(+), 11 deletions(-)
diff --git
a/paimon-common/src/main/java/org/apache/paimon/types/DataTypeJsonParser.java
b/paimon-common/src/main/java/org/apache/paimon/types/DataTypeJsonParser.java
index 19f2dbfe7b..40790f06fb 100644
---
a/paimon-common/src/main/java/org/apache/paimon/types/DataTypeJsonParser.java
+++
b/paimon-common/src/main/java/org/apache/paimon/types/DataTypeJsonParser.java
@@ -26,9 +26,12 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import static org.apache.paimon.utils.Preconditions.checkState;
+
/**
* Parser for creating instances of {@link org.apache.paimon.types.DataType}
from a serialized
* string created with {@link org.apache.paimon.types.DataType#serializeJson}.
@@ -36,9 +39,20 @@ import java.util.stream.Stream;
public final class DataTypeJsonParser {
public static DataField parseDataField(JsonNode json) {
- int id = json.get("id").asInt();
+ return parseDataField(json, null);
+ }
+
+ private static DataField parseDataField(JsonNode json, AtomicInteger
fieldId) {
+ int id;
+ JsonNode idNode = json.get("id");
+ if (idNode != null) {
+ checkState(fieldId == null || fieldId.get() == -1, "Partial field
id is not allowed.");
+ id = idNode.asInt();
+ } else {
+ id = fieldId.incrementAndGet();
+ }
String name = json.get("name").asText();
- DataType type = parseDataType(json.get("type"));
+ DataType type = parseDataType(json.get("type"), fieldId);
JsonNode descriptionNode = json.get("description");
String description = null;
if (descriptionNode != null) {
@@ -48,26 +62,30 @@ public final class DataTypeJsonParser {
}
public static DataType parseDataType(JsonNode json) {
+ return parseDataType(json, new AtomicInteger(-1));
+ }
+
+ public static DataType parseDataType(JsonNode json, AtomicInteger fieldId)
{
if (json.isTextual()) {
return parseAtomicTypeSQLString(json.asText());
} else if (json.isObject()) {
String typeString = json.get("type").asText();
if (typeString.startsWith("ARRAY")) {
- DataType element = parseDataType(json.get("element"));
+ DataType element = parseDataType(json.get("element"), fieldId);
return new ArrayType(!typeString.contains("NOT NULL"),
element);
} else if (typeString.startsWith("MULTISET")) {
- DataType element = parseDataType(json.get("element"));
+ DataType element = parseDataType(json.get("element"), fieldId);
return new MultisetType(!typeString.contains("NOT NULL"),
element);
} else if (typeString.startsWith("MAP")) {
- DataType key = parseDataType(json.get("key"));
- DataType value = parseDataType(json.get("value"));
+ DataType key = parseDataType(json.get("key"), fieldId);
+ DataType value = parseDataType(json.get("value"), fieldId);
return new MapType(!typeString.contains("NOT NULL"), key,
value);
} else if (typeString.startsWith("ROW")) {
JsonNode fieldArray = json.get("fields");
Iterator<JsonNode> iterator = fieldArray.elements();
List<DataField> fields = new ArrayList<>(fieldArray.size());
while (iterator.hasNext()) {
- fields.add(parseDataField(iterator.next()));
+ fields.add(parseDataField(iterator.next(), fieldId));
}
return new RowType(!typeString.contains("NOT NULL"), fields);
}
diff --git a/paimon-common/src/main/java/org/apache/paimon/types/RowType.java
b/paimon-common/src/main/java/org/apache/paimon/types/RowType.java
index fecb5bed9e..681a07af58 100644
--- a/paimon-common/src/main/java/org/apache/paimon/types/RowType.java
+++ b/paimon-common/src/main/java/org/apache/paimon/types/RowType.java
@@ -356,7 +356,11 @@ public final class RowType extends DataType {
}
public static Builder builder() {
- return builder(true, new AtomicInteger(-1));
+ return builder(new AtomicInteger(-1));
+ }
+
+ public static Builder builder(AtomicInteger fieldId) {
+ return builder(true, fieldId);
}
public static Builder builder(boolean isNullable, AtomicInteger fieldId) {
diff --git
a/paimon-core/src/test/java/org/apache/paimon/schema/DataTypeJsonParserTest.java
b/paimon-core/src/test/java/org/apache/paimon/schema/DataTypeJsonParserTest.java
index 52ecff282c..2397af83aa 100644
---
a/paimon-core/src/test/java/org/apache/paimon/schema/DataTypeJsonParserTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/schema/DataTypeJsonParserTest.java
@@ -26,6 +26,7 @@ import org.apache.paimon.types.CharType;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.DataType;
import org.apache.paimon.types.DataTypeJsonParser;
+import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.DateType;
import org.apache.paimon.types.DecimalType;
import org.apache.paimon.types.DoubleType;
@@ -43,6 +44,7 @@ import org.apache.paimon.types.VarBinaryType;
import org.apache.paimon.types.VarCharType;
import org.apache.paimon.utils.JsonSerdeUtil;
+import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
@@ -50,6 +52,7 @@ import javax.annotation.Nullable;
import java.util.Arrays;
import java.util.Collections;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Stream;
import static org.assertj.core.api.Assertions.assertThat;
@@ -199,6 +202,44 @@ public class DataTypeJsonParserTest {
}
}
+ @Test
+ void testParingRowTypeWithoutFieldId() {
+ String jsonString1 =
+
"{\"type\":\"ROW\",\"fields\":[{\"name\":\"field1\",\"type\":\"INT\"},{\"name\":\"field2\",\"type\":\"STRING\"}]}";
+ RowType rowType1 =
+ RowType.builder()
+ .fields(
+ new DataType[] {DataTypes.INT(),
DataTypes.STRING()},
+ new String[] {"field1", "field2"})
+ .build();
+ assertThat(parse(jsonString1)).isEqualTo(rowType1);
+
+ String jsonString2 =
+
"{\"type\":\"ROW\",\"fields\":[{\"name\":\"field1\",\"type\":\"INT\"},{\"name\":\"field2\",\"type\":{\"type\":\"ROW\",\"fields\":[{\"name\":\"s1\",\"type\":\"INT\"},{\"name\":\"s2\",\"type\":\"STRING\"}]}}]}";
+ RowType rowType2 =
+ RowType.builder()
+ .fields(
+ new DataType[] {
+ DataTypes.INT(),
+ RowType.builder(new AtomicInteger(1))
+ .fields(
+ new DataType[] {
+ DataTypes.INT(),
DataTypes.STRING()
+ },
+ new String[] {"s1", "s2"})
+ .build()
+ },
+ new String[] {"field1", "field2"})
+ .build();
+ assertThat(parse(jsonString2)).isEqualTo(rowType2);
+
+ String jsonString3 =
+
"{\"type\":\"ROW\",\"fields\":[{\"name\":\"field1\",\"type\":\"INT\"},{\"id\":1,
\"name\":\"field2\",\"type\":\"STRING\"}]}";
+ assertThatThrownBy(() -> parse(jsonString3))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("Partial field id is not allowed.");
+ }
+
private static String toJson(DataType type) {
return JsonSerdeUtil.toFlatJson(type);
}
@@ -207,9 +248,7 @@ public class DataTypeJsonParserTest {
if (!json.startsWith("\"") && !json.startsWith("{")) {
json = "\"" + json + "\"";
}
- String dataFieldJson =
- String.format("{\"id\": 0, \"name\": \"dummy\", \"type\":
%s}", json);
- return JsonSerdeUtil.fromJson(dataFieldJson, DataField.class).type();
+ return JsonSerdeUtil.fromJson(json, DataType.class);
}
//
--------------------------------------------------------------------------------------------
