This is an automated email from the ASF dual-hosted git repository.
yuqi4733 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new a30b149f8e [#8940] improvement(core): Support serialize/deserialize
for distributionImpl and sortOrderImpl (#9198)
a30b149f8e is described below
commit a30b149f8eec2ac40e70dad1701963d4a62581de
Author: Junda Yang <[email protected]>
AuthorDate: Thu Nov 20 23:14:36 2025 -0800
[#8940] improvement(core): Support serialize/deserialize for
distributionImpl and sortOrderImpl (#9198)
### What changes were proposed in this pull request?
- Added JSON serializers/deserializers for Partitioning objects in
JsonUtils
- Enhanced POConverters to properly serialize/deserialize Distribution,
SortOrder[], Index[], and Transform[] (partitions) by converting to DTOs
before JSON serialization
- Simplified Indexes.IndexImpl by removing custom Jackson serializers
(105 lines deleted) and adding equals()/hashCode() methods
- Added comprehensive serialization tests in new TestSerializer class
### Why are the changes needed?
Enable proper persistence of Lance table metadata (distribution, sort
orders, indexes, partitions) in the relational backend. Previously these
fields were not being serialized/deserialized, preventing complete
metadata management for generic lakehouse tables.
Fix: #8940
### Does this PR introduce _any_ user-facing change?
No. Internal serialization improvement - table metadata is now properly
persisted and retrieved.
### How was this patch tested?
- New TestSerializer unit tests for Distribution, SortOrder, Index, and
Partitioning serialization
- Enhanced TestPOConverters to validate table entity conversion with all
metadata fields
- Updated CatalogGenericLakehouseLanceIT integration test to create and
load tables with full metadata
Co-authored-by: Mini Yu <[email protected]>
---
.../org/apache/gravitino/rel/indexes/Indexes.java | 105 +++---------
.../java/org/apache/gravitino/rel/TestIndex.java | 57 -------
.../test/CatalogGenericLakehouseLanceIT.java | 52 +++++-
.../java/org/apache/gravitino/json/JsonUtils.java | 6 +
.../org/apache/gravitino/json/TestSerializer.java | 186 +++++++++++++++++++++
.../storage/relational/utils/POConverters.java | 68 +++++++-
.../storage/relational/utils/TestPOConverters.java | 18 ++
7 files changed, 340 insertions(+), 152 deletions(-)
diff --git a/api/src/main/java/org/apache/gravitino/rel/indexes/Indexes.java
b/api/src/main/java/org/apache/gravitino/rel/indexes/Indexes.java
index d1b1a1f523..c6303e7938 100644
--- a/api/src/main/java/org/apache/gravitino/rel/indexes/Indexes.java
+++ b/api/src/main/java/org/apache/gravitino/rel/indexes/Indexes.java
@@ -18,21 +18,8 @@
*/
package org.apache.gravitino.rel.indexes;
-import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.JsonDeserializer;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.JsonSerializer;
-import com.fasterxml.jackson.databind.SerializerProvider;
-import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
-import com.fasterxml.jackson.databind.annotation.JsonSerialize;
-import com.fasterxml.jackson.databind.node.ArrayNode;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import java.io.IOException;
-import java.util.List;
-import java.util.Locale;
+import com.google.common.base.Objects;
+import java.util.Arrays;
/** Helper methods to create index to pass into Apache Gravitino. */
public class Indexes {
@@ -89,76 +76,7 @@ public class Indexes {
.build();
}
- /** Custom JSON serializer for Index objects. */
- public static class IndexSerializer extends JsonSerializer<Index> {
- @Override
- public void serialize(Index value, JsonGenerator gen, SerializerProvider
serializers)
- throws IOException {
- gen.writeStartObject();
- gen.writeStringField("indexType",
value.type().name().toUpperCase(Locale.ROOT));
- if (null != value.name()) {
- gen.writeStringField("name", value.name());
- }
- gen.writeFieldName("fieldNames");
- gen.writeObject(value.fieldNames());
- gen.writeEndObject();
- }
- }
-
- /** Custom JSON deserializer for Index objects. */
- public static class IndexDeserializer extends JsonDeserializer<Index> {
-
- @Override
- public Index deserialize(JsonParser p, DeserializationContext ctxt) throws
IOException {
- JsonNode node = p.getCodec().readTree(p);
- Preconditions.checkArgument(
- node != null && !node.isNull() && node.isObject(),
- "Index must be a valid JSON object, but found: %s",
- node);
-
- IndexImpl.Builder builder = IndexImpl.builder();
- Preconditions.checkArgument(
- node.has("indexType"), "Cannot parse index from missing type: %s",
node);
- String indexType = getString("indexType", node);
-
builder.withIndexType(Index.IndexType.valueOf(indexType.toUpperCase(Locale.ROOT)));
- if (node.has("name")) {
- builder.withName(getString("name", node));
- }
- Preconditions.checkArgument(
- node.has("fieldNames"), "Cannot parse index from missing field
names: %s", node);
- List<String[]> fieldNames = Lists.newArrayList();
- node.get("fieldNames").forEach(field ->
fieldNames.add(getStringArray((ArrayNode) field)));
- builder.withFieldNames(fieldNames.toArray(new String[0][0]));
- return builder.build();
- }
-
- private static String[] getStringArray(ArrayNode node) {
- String[] array = new String[node.size()];
- for (int i = 0; i < node.size(); i++) {
- array[i] = node.get(i).asText();
- }
- return array;
- }
-
- private static String getString(String property, JsonNode node) {
- Preconditions.checkArgument(node.has(property), "Cannot parse missing
string: %s", property);
- JsonNode pNode = node.get(property);
- return convertToString(property, pNode);
- }
-
- private static String convertToString(String property, JsonNode pNode) {
- Preconditions.checkArgument(
- pNode != null && !pNode.isNull() && pNode.isTextual(),
- "Cannot parse to a string value %s: %s",
- property,
- pNode);
- return pNode.asText();
- }
- }
-
/** The user side implementation of the index. */
- @JsonSerialize(using = IndexSerializer.class)
- @JsonDeserialize(using = IndexDeserializer.class)
public static final class IndexImpl implements Index {
private final IndexType indexType;
@@ -203,6 +121,25 @@ public class Indexes {
return fieldNames;
}
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ IndexImpl index = (IndexImpl) o;
+ return indexType == index.indexType
+ && Objects.equal(name, index.name)
+ && Arrays.deepEquals(fieldNames, index.fieldNames);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(indexType, name, Arrays.hashCode(fieldNames));
+ }
+
/**
* @return the builder for creating a new instance of IndexImpl.
*/
diff --git a/api/src/test/java/org/apache/gravitino/rel/TestIndex.java
b/api/src/test/java/org/apache/gravitino/rel/TestIndex.java
deleted file mode 100644
index 4a807fbb7b..0000000000
--- a/api/src/test/java/org/apache/gravitino/rel/TestIndex.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.gravitino.rel;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.MapperFeature;
-import com.fasterxml.jackson.databind.SerializationFeature;
-import com.fasterxml.jackson.databind.cfg.EnumFeature;
-import com.fasterxml.jackson.databind.json.JsonMapper;
-import org.apache.gravitino.rel.indexes.Index;
-import org.apache.gravitino.rel.indexes.Indexes;
-import org.apache.gravitino.rel.indexes.Indexes.IndexImpl;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-
-public class TestIndex {
-
- @Test
- void testIndexSerialization() throws JsonProcessingException {
- String[][] fields = {{"column1"}, {"column2", "subcolumn"}};
- Index index = Indexes.unique("test_index", fields);
-
- JsonMapper jsonMapper =
- JsonMapper.builder()
- .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false)
- .configure(EnumFeature.WRITE_ENUMS_TO_LOWERCASE, true)
- .enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS)
- .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
-
.configure(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL, true)
- .build();
-
- String json = jsonMapper.writeValueAsString(index);
-
- Index deserializedIndex = jsonMapper.readValue(json, IndexImpl.class);
- Assertions.assertEquals(index.type(), deserializedIndex.type());
- Assertions.assertEquals(index.name(), deserializedIndex.name());
- Assertions.assertArrayEquals(index.fieldNames(),
deserializedIndex.fieldNames());
- }
-}
diff --git
a/catalogs/catalog-generic-lakehouse/src/test/java/org/apache/gravitino/catalog/lakehouse/integration/test/CatalogGenericLakehouseLanceIT.java
b/catalogs/catalog-generic-lakehouse/src/test/java/org/apache/gravitino/catalog/lakehouse/integration/test/CatalogGenericLakehouseLanceIT.java
index c4790c4d85..1b4fd8b172 100644
---
a/catalogs/catalog-generic-lakehouse/src/test/java/org/apache/gravitino/catalog/lakehouse/integration/test/CatalogGenericLakehouseLanceIT.java
+++
b/catalogs/catalog-generic-lakehouse/src/test/java/org/apache/gravitino/catalog/lakehouse/integration/test/CatalogGenericLakehouseLanceIT.java
@@ -58,7 +58,22 @@ import org.apache.gravitino.integration.test.util.BaseIT;
import org.apache.gravitino.integration.test.util.GravitinoITUtils;
import org.apache.gravitino.rel.Column;
import org.apache.gravitino.rel.Table;
+import org.apache.gravitino.rel.expressions.NamedReference;
+import org.apache.gravitino.rel.expressions.distributions.Distribution;
+import org.apache.gravitino.rel.expressions.distributions.Distributions;
+import org.apache.gravitino.rel.expressions.distributions.Strategy;
+import org.apache.gravitino.rel.expressions.literals.Literals;
+import org.apache.gravitino.rel.expressions.sorts.NullOrdering;
+import org.apache.gravitino.rel.expressions.sorts.SortDirection;
+import org.apache.gravitino.rel.expressions.sorts.SortOrder;
+import org.apache.gravitino.rel.expressions.sorts.SortOrders;
+import org.apache.gravitino.rel.expressions.transforms.Transform;
import org.apache.gravitino.rel.expressions.transforms.Transforms;
+import org.apache.gravitino.rel.indexes.Index;
+import org.apache.gravitino.rel.indexes.Index.IndexType;
+import org.apache.gravitino.rel.indexes.Indexes;
+import org.apache.gravitino.rel.partitions.Partitions;
+import org.apache.gravitino.rel.partitions.RangePartition;
import org.apache.gravitino.rel.types.Types;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
@@ -176,6 +191,30 @@ public class CatalogGenericLakehouseLanceIT extends BaseIT
{
properties = createProperties();
properties.put("format", "lance");
+ Distribution distribution =
+ Distributions.of(Strategy.EVEN, 10,
NamedReference.field(LANCE_COL_NAME1));
+ SortOrder[] sortOrders =
+ new SortOrder[] {
+ SortOrders.of(
+ NamedReference.field(LANCE_COL_NAME2),
+ SortDirection.ASCENDING,
+ NullOrdering.NULLS_FIRST)
+ };
+ Index[] indexes =
+ new Index[] {
+ Indexes.of(IndexType.UNIQUE_KEY, "unique_index", new String[][]
{{LANCE_COL_NAME3}})
+ };
+
+ RangePartition p1 =
+ Partitions.range(
+ "p1", Literals.stringLiteral("20220101"), Literals.NULL,
Collections.emptyMap());
+ RangePartition p2 =
+ Partitions.range(
+ "p2", Literals.stringLiteral("20220301"), Literals.NULL,
Collections.emptyMap());
+ Transform[] partitioning = {
+ Transforms.range(new String[] {LANCE_COL_NAME3}, new RangePartition[]
{p1, p2})
+ };
+
createdTable =
catalog
.asTableCatalog()
@@ -184,9 +223,10 @@ public class CatalogGenericLakehouseLanceIT extends BaseIT
{
columns,
TABLE_COMMENT,
properties,
- Transforms.EMPTY_TRANSFORM,
- null,
- null);
+ partitioning,
+ distribution,
+ sortOrders,
+ indexes);
Assertions.assertEquals(createdTable.name(), tableName);
createdTableProperties = createdTable.properties();
Assertions.assertEquals("lance", createdTableProperties.get("format"));
@@ -198,6 +238,12 @@ public class CatalogGenericLakehouseLanceIT extends BaseIT
{
Assertions.assertEquals(expectedTableLocation,
createdTableProperties.get("location"));
Assertions.assertTrue(new File(expectedTableLocation).exists());
+ Table loadTable = catalog.asTableCatalog().loadTable(nameIdentifier);
+ Assertions.assertEquals(distribution, loadTable.distribution());
+ Assertions.assertArrayEquals(sortOrders, loadTable.sortOrder());
+ Assertions.assertArrayEquals(indexes, loadTable.index());
+ Assertions.assertArrayEquals(partitioning, loadTable.partitioning());
+
// Now try to load table
Table loadedTable = catalog.asTableCatalog().loadTable(nameIdentifier);
Assertions.assertEquals(createdTable.name(), loadedTable.name());
diff --git a/common/src/main/java/org/apache/gravitino/json/JsonUtils.java
b/common/src/main/java/org/apache/gravitino/json/JsonUtils.java
index fb9a33f268..b35243b17b 100644
--- a/common/src/main/java/org/apache/gravitino/json/JsonUtils.java
+++ b/common/src/main/java/org/apache/gravitino/json/JsonUtils.java
@@ -18,6 +18,10 @@
*/
package org.apache.gravitino.json;
+import static
org.apache.gravitino.dto.rel.expressions.FunctionArg.ArgType.FIELD;
+import static
org.apache.gravitino.dto.rel.expressions.FunctionArg.ArgType.FUNCTION;
+import static
org.apache.gravitino.dto.rel.expressions.FunctionArg.ArgType.LITERAL;
+
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.PropertyAccessor;
import com.fasterxml.jackson.core.JacksonException;
@@ -304,6 +308,8 @@ public class JsonUtils {
new SimpleModule()
.addDeserializer(Type.class, new TypeDeserializer())
.addSerializer(Type.class, new TypeSerializer())
+ .addDeserializer(Partitioning.class, new
PartitioningDeserializer())
+ .addSerializer(Partitioning.class, new
PartitioningSerializer())
.addDeserializer(Expression.class, new
ColumnDefaultValueDeserializer())
.addSerializer(Expression.class, new
ColumnDefaultValueSerializer())
.addDeserializer(StatisticValue.class, new
StatisticValueDeserializer())
diff --git a/common/src/test/java/org/apache/gravitino/json/TestSerializer.java
b/common/src/test/java/org/apache/gravitino/json/TestSerializer.java
new file mode 100644
index 0000000000..a8fff05622
--- /dev/null
+++ b/common/src/test/java/org/apache/gravitino/json/TestSerializer.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.json;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import org.apache.gravitino.dto.rel.DistributionDTO;
+import org.apache.gravitino.dto.rel.SortOrderDTO;
+import org.apache.gravitino.dto.rel.expressions.LiteralDTO;
+import org.apache.gravitino.dto.rel.indexes.IndexDTO;
+import org.apache.gravitino.dto.rel.partitioning.DayPartitioningDTO;
+import org.apache.gravitino.dto.rel.partitioning.Partitioning;
+import org.apache.gravitino.dto.rel.partitioning.RangePartitioningDTO;
+import org.apache.gravitino.dto.rel.partitions.RangePartitionDTO;
+import org.apache.gravitino.dto.util.DTOConverters;
+import org.apache.gravitino.rel.expressions.FunctionExpression;
+import org.apache.gravitino.rel.expressions.NamedReference;
+import org.apache.gravitino.rel.expressions.distributions.Distributions;
+import
org.apache.gravitino.rel.expressions.distributions.Distributions.DistributionImpl;
+import org.apache.gravitino.rel.expressions.distributions.Strategy;
+import org.apache.gravitino.rel.expressions.literals.Literals;
+import org.apache.gravitino.rel.expressions.sorts.NullOrdering;
+import org.apache.gravitino.rel.expressions.sorts.SortDirection;
+import org.apache.gravitino.rel.expressions.sorts.SortOrders;
+import org.apache.gravitino.rel.expressions.sorts.SortOrders.SortImpl;
+import org.apache.gravitino.rel.expressions.transforms.Transform;
+import org.apache.gravitino.rel.indexes.Index.IndexType;
+import org.apache.gravitino.rel.indexes.Indexes;
+import org.apache.gravitino.rel.indexes.Indexes.IndexImpl;
+import org.apache.gravitino.rel.types.Types;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+public class TestSerializer {
+
+ @Test
+ void testDistributionImplSerializer() throws JsonProcessingException {
+ DistributionImpl distribution =
+ (DistributionImpl) Distributions.of(Strategy.EVEN, 10,
NamedReference.field("col1"));
+ String actualJson =
+
JsonUtils.anyFieldMapper().writeValueAsString(DTOConverters.toDTO(distribution));
+ String expectedJson =
+ """
+
{"strategy":"even","number":10,"funcArgs":[{"type":"field","fieldName":["col1"]}]}""";
+ Assertions.assertEquals(expectedJson, actualJson);
+ DistributionDTO deserialized =
+ JsonUtils.anyFieldMapper().readValue(actualJson,
DistributionDTO.class);
+ Assertions.assertEquals(distribution, DTOConverters.fromDTO(deserialized));
+
+ distribution =
+ (DistributionImpl)
+ Distributions.of(
+ Strategy.EVEN,
+ 10,
+ FunctionExpression.of(
+ "bucket", Literals.integerLiteral(10),
NamedReference.field("col_1")));
+ actualJson =
JsonUtils.anyFieldMapper().writeValueAsString(DTOConverters.toDTO(distribution));
+ expectedJson =
+ """
+
{"strategy":"even","number":10,"funcArgs":[{"type":"function","funcName":"bucket",\
+
"funcArgs":[{"type":"literal","dataType":"integer","value":"10"},{"type":"field",\
+ "fieldName":["col_1"]}]}]}""";
+ Assertions.assertEquals(expectedJson, actualJson);
+ deserialized = JsonUtils.anyFieldMapper().readValue(actualJson,
DistributionDTO.class);
+ Assertions.assertEquals(distribution, DTOConverters.fromDTO(deserialized));
+ }
+
+ @Test
+ void testSortOrderSerializer() throws JsonProcessingException {
+ SortImpl sortOrder =
+ SortOrders.of(
+ NamedReference.field("col1"), SortDirection.ASCENDING,
NullOrdering.NULLS_LAST);
+ String actualJson =
+
JsonUtils.anyFieldMapper().writeValueAsString(DTOConverters.toDTO(sortOrder));
+ String expectedJson =
+ """
+
{"sortTerm":{"type":"field","fieldName":["col1"]},"direction":"asc",\
+ "nullOrdering":"nulls_last"}""";
+ Assertions.assertEquals(expectedJson, actualJson);
+
+ SortOrderDTO deserialized =
+ JsonUtils.anyFieldMapper().readValue(actualJson, SortOrderDTO.class);
+ Assertions.assertEquals(sortOrder, DTOConverters.fromDTO(deserialized));
+
+ sortOrder =
+ SortOrders.of(
+ FunctionExpression.of("lower", NamedReference.field("col_1")),
+ SortDirection.DESCENDING);
+ actualJson =
JsonUtils.anyFieldMapper().writeValueAsString(DTOConverters.toDTO(sortOrder));
+ expectedJson =
+ """
+
{"sortTerm":{"type":"function","funcName":"lower","funcArgs":[{"type":"field",\
+
"fieldName":["col_1"]}]},"direction":"desc","nullOrdering":"nulls_last"}""";
+ Assertions.assertEquals(expectedJson, actualJson);
+ deserialized = JsonUtils.anyFieldMapper().readValue(actualJson,
SortOrderDTO.class);
+ Assertions.assertEquals(sortOrder, DTOConverters.fromDTO(deserialized));
+ }
+
+ @Test
+ void testIndexImplSerializer() throws JsonProcessingException {
+ IndexImpl index =
+ (IndexImpl)
+ Indexes.of(IndexType.PRIMARY_KEY, "index_1", new String[][] {new
String[] {"col1"}});
+
+ String actualJson =
JsonUtils.anyFieldMapper().writeValueAsString((DTOConverters.toDTO(index)));
+ String expectedJson =
+ """
+
{"indexType":"PRIMARY_KEY","name":"index_1","fieldNames":[["col1"]]}""";
+ Assertions.assertEquals(expectedJson, actualJson);
+ IndexDTO deserialized = JsonUtils.anyFieldMapper().readValue(actualJson,
IndexDTO.class);
+ Assertions.assertEquals(index, DTOConverters.fromDTO(deserialized));
+
+ index =
+ (IndexImpl)
+ Indexes.of(
+ IndexType.UNIQUE_KEY,
+ "index_2",
+ new String[][] {new String[] {"col1"}, new String[] {"col2"}});
+ actualJson = JsonUtils.anyFieldMapper().writeValueAsString(index);
+ expectedJson =
+ """
+
{"indexType":"unique_key","name":"index_2","fieldNames":[["col1"],["col2"]]}""";
+ Assertions.assertEquals(expectedJson, actualJson);
+ }
+
+ @Test
+ @Disabled("Disable until Partitioning serializer is implemented")
+ void testPartitioningSerializer() throws JsonProcessingException {
+ Transform transform = DayPartitioningDTO.of("dt");
+ String actualJson =
+
JsonUtils.anyFieldMapper().writeValueAsString(DTOConverters.toDTO(transform));
+ String expectedJson = """
+ {"strategy":"day","fieldName":["dt"]}""";
+ Assertions.assertEquals(expectedJson, actualJson);
+
+ Partitioning deserialized =
+ JsonUtils.anyFieldMapper().readValue(actualJson, Partitioning.class);
+ Assertions.assertEquals(transform, DTOConverters.fromDTO(deserialized));
+
+ transform =
+ RangePartitioningDTO.of(
+ new String[] {"dt"},
+ new RangePartitionDTO[] {
+ RangePartitionDTO.builder()
+ .withName("p1")
+ .withLower(
+ LiteralDTO.builder()
+ .withValue("2023-01-01")
+ .withDataType(Types.StringType.get())
+ .build())
+ .withUpper(
+ LiteralDTO.builder()
+ .withValue("2024-01-01")
+ .withDataType(Types.StringType.get())
+ .build())
+ .build(),
+ });
+
+ actualJson = JsonUtils.anyFieldMapper().writeValueAsString(transform);
+ expectedJson =
+ """
+
{"type":"range_partitioning","fieldName":["dt"],"assignments":[{"name":"p1",
+
"properties":null,"upper":{"type":"literal","dataType":"string","value":"2024-01-01"},
+
"lower":{"type":"literal","dataType":"string","value":"2023-01-01"}}]}""";
+ Assertions.assertEquals(expectedJson, actualJson);
+ deserialized = JsonUtils.anyFieldMapper().readValue(actualJson,
Partitioning.class);
+ Assertions.assertEquals(transform, DTOConverters.fromDTO(deserialized));
+ }
+}
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/utils/POConverters.java
b/core/src/main/java/org/apache/gravitino/storage/relational/utils/POConverters.java
index 48cf8cc00e..63f46f4e11 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/utils/POConverters.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/utils/POConverters.java
@@ -36,7 +36,11 @@ import org.apache.gravitino.authorization.Privilege;
import org.apache.gravitino.authorization.Privileges;
import org.apache.gravitino.authorization.SecurableObject;
import org.apache.gravitino.authorization.SecurableObjects;
+import org.apache.gravitino.dto.rel.DistributionDTO;
+import org.apache.gravitino.dto.rel.SortOrderDTO;
import org.apache.gravitino.dto.rel.expressions.FunctionArg;
+import org.apache.gravitino.dto.rel.indexes.IndexDTO;
+import org.apache.gravitino.dto.rel.partitioning.Partitioning;
import org.apache.gravitino.dto.util.DTOConverters;
import org.apache.gravitino.file.Fileset;
import org.apache.gravitino.json.JsonUtils;
@@ -60,7 +64,6 @@ import org.apache.gravitino.policy.Policy;
import org.apache.gravitino.policy.PolicyContent;
import org.apache.gravitino.rel.Column;
import org.apache.gravitino.rel.expressions.Expression;
-import org.apache.gravitino.rel.indexes.Indexes.IndexImpl;
import org.apache.gravitino.rel.types.Type;
import org.apache.gravitino.storage.relational.po.CatalogPO;
import org.apache.gravitino.storage.relational.po.ColumnPO;
@@ -407,9 +410,24 @@ public class POConverters {
.withIndexes(
tableEntity.getIndexes() == null
? null
- :
JsonUtils.anyFieldMapper().writeValueAsString(tableEntity.getIndexes()));
-
- // TODO, handle these fields(distribution, sort order, partition) later
+ : JsonUtils.anyFieldMapper()
+
.writeValueAsString(DTOConverters.toDTOs(tableEntity.getIndexes())))
+ .withDistribution(
+ tableEntity.getDistribution() == null
+ ? null
+ : JsonUtils.anyFieldMapper()
+
.writeValueAsString(DTOConverters.toDTO(tableEntity.getDistribution())))
+ .withSortOrders(
+ tableEntity.getSortOrder() == null
+ ? null
+ : JsonUtils.anyFieldMapper()
+
.writeValueAsString(DTOConverters.toDTOs(tableEntity.getSortOrder())))
+ .withPartitions(
+ tableEntity.getPartitions() == null
+ ? null
+ : JsonUtils.anyFieldMapper()
+
.writeValueAsString(DTOConverters.toDTOs(tableEntity.getPartitions())));
+ // TODO support partitions later
return builder.build();
} catch (JsonProcessingException e) {
throw new RuntimeException("Failed to serialize json object:", e);
@@ -451,9 +469,25 @@ public class POConverters {
.withIndexes(
newTable.getIndexes() == null
? null
- :
JsonUtils.anyFieldMapper().writeValueAsString(newTable.getIndexes()))
+ : JsonUtils.anyFieldMapper()
+
.writeValueAsString(DTOConverters.toDTOs(newTable.getIndexes())))
+ .withDistribution(
+ newTable.getDistribution() == null
+ ? null
+ : JsonUtils.anyFieldMapper()
+
.writeValueAsString(DTOConverters.toDTO(newTable.getDistribution())))
+ .withSortOrders(
+ newTable.getSortOrder() == null
+ ? null
+ : JsonUtils.anyFieldMapper()
+
.writeValueAsString(DTOConverters.toDTOs(newTable.getSortOrder())))
+ .withPartitions(
+ newTable.getPartitions() == null
+ ? null
+ : JsonUtils.anyFieldMapper()
+
.writeValueAsString(DTOConverters.toDTOs(newTable.getPartitions())))
+ // TODO support partitions later
.withFormat(newTable.getFormat());
- // TODO other fields(partitioning, distribution, sortorder) in the
refactor PRs.
return builder.build();
} catch (JsonProcessingException e) {
@@ -482,11 +516,29 @@ public class POConverters {
.withColumns(fromColumnPOs(columnPOs))
.withAuditInfo(
JsonUtils.anyFieldMapper().readValue(tablePO.getAuditInfo(),
AuditInfo.class))
- // TODO add field partition, distribution and sort order;
+ .withDistribution(
+ StringUtils.isBlank(tablePO.getDistribution())
+ ? null
+ : DTOConverters.fromDTO(
+ JsonUtils.anyFieldMapper()
+ .readValue(tablePO.getDistribution(),
DistributionDTO.class)))
+ .withSortOrder(
+ StringUtils.isBlank(tablePO.getSortOrders())
+ ? null
+ : DTOConverters.fromDTOs(
+ JsonUtils.anyFieldMapper()
+ .readValue(tablePO.getSortOrders(),
SortOrderDTO[].class)))
.withIndexes(
StringUtils.isBlank(tablePO.getIndexes())
? null
- : JsonUtils.anyFieldMapper().readValue(tablePO.getIndexes(),
IndexImpl[].class))
+ : DTOConverters.fromDTOs(
+
JsonUtils.anyFieldMapper().readValue(tablePO.getIndexes(), IndexDTO[].class)))
+ // TODO add field partition, distribution and sort order;
+ .withPartitions(
+ StringUtils.isBlank(tablePO.getPartitions())
+ ? null
+ : JsonUtils.anyFieldMapper()
+ .readValue(tablePO.getPartitions(),
Partitioning[].class))
.withFormat(tablePO.getFormat())
.withComment(tablePO.getComment())
.withProperties(
diff --git
a/core/src/test/java/org/apache/gravitino/storage/relational/utils/TestPOConverters.java
b/core/src/test/java/org/apache/gravitino/storage/relational/utils/TestPOConverters.java
index db9d0cce86..eea8837baf 100644
---
a/core/src/test/java/org/apache/gravitino/storage/relational/utils/TestPOConverters.java
+++
b/core/src/test/java/org/apache/gravitino/storage/relational/utils/TestPOConverters.java
@@ -65,7 +65,13 @@ import org.apache.gravitino.policy.Policy;
import org.apache.gravitino.policy.PolicyContent;
import org.apache.gravitino.policy.PolicyContents;
import org.apache.gravitino.rel.expressions.Expression;
+import org.apache.gravitino.rel.expressions.NamedReference;
+import org.apache.gravitino.rel.expressions.distributions.Distributions;
+import org.apache.gravitino.rel.expressions.distributions.Strategy;
import org.apache.gravitino.rel.expressions.literals.Literals;
+import org.apache.gravitino.rel.expressions.sorts.SortDirection;
+import org.apache.gravitino.rel.expressions.sorts.SortOrder;
+import org.apache.gravitino.rel.expressions.sorts.SortOrders;
import org.apache.gravitino.rel.types.Type;
import org.apache.gravitino.rel.types.Types;
import org.apache.gravitino.stats.StatisticValues;
@@ -616,6 +622,15 @@ public class TestPOConverters {
assertEquals(1, initPO.getCurrentVersion());
assertEquals(1, initPO.getLastVersion());
assertEquals(0, initPO.getDeletedAt());
+
+ TableEntity entity =
+ POConverters.fromTableAndColumnPOs(
+ initPO,
+ Lists.newArrayList(),
+ NamespaceUtil.ofTable("test_metalake", "test_catalog",
"test_schema"));
+
+ Assertions.assertEquals(tableEntity.getDistribution(),
entity.getDistribution());
+ Assertions.assertArrayEquals(tableEntity.getPartitions(),
entity.getPartitions());
}
@Test
@@ -1382,6 +1397,9 @@ public class TestPOConverters {
.withName(name)
.withNamespace(namespace)
.withColumns(columns)
+ .withDistribution(Distributions.of(Strategy.EVEN, 10,
NamedReference.field("key")))
+ .withSortOrder(
+ new SortOrder[] {SortOrders.of(NamedReference.field("col1"),
SortDirection.ASCENDING)})
.withAuditInfo(auditInfo)
.build();
}
