This is an automated email from the ASF dual-hosted git repository.
mchades pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 4acdc498c [#3368] feat(jdbc-doris): support pre-assign partition for
Doris table (#3887)
4acdc498c is described below
commit 4acdc498cfca005b6b4b83890565c16f57d0fe03
Author: XiaoZ <[email protected]>
AuthorDate: Tue Jul 30 11:13:31 2024 +0800
[#3368] feat(jdbc-doris): support pre-assign partition for Doris table
(#3887)
### What changes were proposed in this pull request?
Support pre-assign partition for Doris table.
### Why are the changes needed?
Fix: #3368
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
UT & IT
---------
Co-authored-by: zhanghan18 <[email protected]>
---
.../doris/operation/DorisTableOperations.java | 100 ++++++++---
.../operation/DorisTablePartitionOperations.java | 70 ++------
.../gravitino/catalog/doris/utils/DorisUtils.java | 71 ++++++++
.../doris/integration/test/CatalogDorisIT.java | 200 ++++++++++++++-------
.../doris/operation/TestDorisTableOperations.java | 87 ++++++++-
.../catalog/doris/utils/TestDorisUtils.java | 93 ++++++++--
.../gravitino/integration/test/util/ITUtils.java | 36 ++++
7 files changed, 493 insertions(+), 164 deletions(-)
diff --git
a/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTableOperations.java
b/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTableOperations.java
index 2ac519cb5..21795d803 100644
---
a/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTableOperations.java
+++
b/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTableOperations.java
@@ -18,6 +18,7 @@
*/
package org.apache.gravitino.catalog.doris.operation;
+import static
org.apache.gravitino.catalog.doris.utils.DorisUtils.generatePartitionSqlFragment;
import static org.apache.gravitino.rel.Column.DEFAULT_VALUE_NOT_SET;
import com.google.common.base.Preconditions;
@@ -54,16 +55,18 @@ import org.apache.gravitino.rel.Column;
import org.apache.gravitino.rel.TableChange;
import org.apache.gravitino.rel.expressions.distributions.Distribution;
import org.apache.gravitino.rel.expressions.distributions.Strategy;
+import org.apache.gravitino.rel.expressions.literals.Literal;
import org.apache.gravitino.rel.expressions.transforms.Transform;
import org.apache.gravitino.rel.expressions.transforms.Transforms;
import org.apache.gravitino.rel.indexes.Index;
import org.apache.gravitino.rel.indexes.Indexes;
+import org.apache.gravitino.rel.partitions.ListPartition;
+import org.apache.gravitino.rel.partitions.RangePartition;
/** Table operations for Apache Doris. */
public class DorisTableOperations extends JdbcTableOperations {
private static final String BACK_QUOTE = "`";
private static final String DORIS_AUTO_INCREMENT = "AUTO_INCREMENT";
-
private static final String NEW_LINE = "\n";
@Override
@@ -194,7 +197,6 @@ public class DorisTableOperations extends
JdbcTableOperations {
}
private static void appendIndexesSql(Index[] indexes, StringBuilder
sqlBuilder) {
-
if (indexes.length == 0) {
return;
}
@@ -224,47 +226,91 @@ public class DorisTableOperations extends
JdbcTableOperations {
Preconditions.checkArgument(
partitioning.length == 1, "Composite partition type is not supported");
- StringBuilder partitionSqlBuilder = new StringBuilder();
+ StringBuilder partitionSqlBuilder;
Set<String> columnNames =
Arrays.stream(columns).map(JdbcColumn::name).collect(Collectors.toSet());
if (partitioning[0] instanceof Transforms.RangeTransform) {
- partitionSqlBuilder.append(NEW_LINE).append(" PARTITION BY RANGE(");
- // TODO support multi-column range partitioning in doris
+ // We do not support multi-column range partitioning in doris for now
Transforms.RangeTransform rangePartition = (Transforms.RangeTransform)
partitioning[0];
+ partitionSqlBuilder = generateRangePartitionSql(rangePartition,
columnNames);
+ } else if (partitioning[0] instanceof Transforms.ListTransform) {
+ Transforms.ListTransform listPartition = (Transforms.ListTransform)
partitioning[0];
+ partitionSqlBuilder = generateListPartitionSql(listPartition,
columnNames);
+ } else {
+ throw new IllegalArgumentException("Unsupported partition type of
Doris");
+ }
+ sqlBuilder.append(partitionSqlBuilder);
+ }
+
+ private static StringBuilder generateRangePartitionSql(
+ Transforms.RangeTransform rangePartition, Set<String> columnNames) {
+ Preconditions.checkArgument(
+ rangePartition.fieldName().length == 1, "Doris partition does not
support nested field");
+ Preconditions.checkArgument(
+ columnNames.contains(rangePartition.fieldName()[0]),
+ "The partition field must be one of the columns");
+
+ StringBuilder partitionSqlBuilder = new StringBuilder(NEW_LINE);
+ String partitionDefinition =
+ String.format(" PARTITION BY RANGE(`%s`)",
rangePartition.fieldName()[0]);
+
partitionSqlBuilder.append(partitionDefinition).append(NEW_LINE).append("(");
+
+ // Assign range partitions
+ RangePartition[] assignments = rangePartition.assignments();
+ if (!ArrayUtils.isEmpty(assignments)) {
+ String partitionSqlFragments =
+ Arrays.stream(assignments)
+ .map(DorisUtils::generatePartitionSqlFragment)
+ .collect(Collectors.joining("," + NEW_LINE));
+ partitionSqlBuilder.append(NEW_LINE).append(partitionSqlFragments);
+ }
+
+ partitionSqlBuilder.append(NEW_LINE).append(")");
+ return partitionSqlBuilder;
+ }
+
+ private static StringBuilder generateListPartitionSql(
+ Transforms.ListTransform listPartition, Set<String> columnNames) {
+ ImmutableList.Builder<String> partitionColumnsBuilder =
ImmutableList.builder();
+ String[][] filedNames = listPartition.fieldNames();
+ for (String[] filedName : filedNames) {
Preconditions.checkArgument(
- rangePartition.fieldName().length == 1, "Doris partition does not
support nested field");
+ filedName.length == 1, "Doris partition does not support nested
field");
Preconditions.checkArgument(
- columnNames.contains(rangePartition.fieldName()[0]),
- "The partition field must be one of the columns");
+ columnNames.contains(filedName[0]), "The partition field must be one
of the columns");
- String partitionColumn = BACK_QUOTE + rangePartition.fieldName()[0] +
BACK_QUOTE;
- // TODO we currently do not support pre-assign partition when creating
range partitioning
- partitionSqlBuilder.append(partitionColumn).append(") () ");
- } else if (partitioning[0] instanceof Transforms.ListTransform) {
- Transforms.ListTransform listPartition = (Transforms.ListTransform)
partitioning[0];
- partitionSqlBuilder.append(" PARTITION BY LIST(");
+ partitionColumnsBuilder.add(BACK_QUOTE + filedName[0] + BACK_QUOTE);
+ }
+ String partitionColumns =
+
partitionColumnsBuilder.build().stream().collect(Collectors.joining(","));
+
+ StringBuilder partitionSqlBuilder = new StringBuilder(NEW_LINE);
+ String partitionDefinition = String.format(" PARTITION BY LIST(%s)",
partitionColumns);
+
partitionSqlBuilder.append(partitionDefinition).append(NEW_LINE).append("(");
- ImmutableList.Builder<String> partitionColumnsBuilder =
ImmutableList.builder();
- String[][] filedNames = listPartition.fieldNames();
- for (String[] filedName : filedNames) {
+ // Assign list partitions
+ ListPartition[] assignments = listPartition.assignments();
+ if (!ArrayUtils.isEmpty(assignments)) {
+ ImmutableList.Builder<String> partitions = ImmutableList.builder();
+ for (ListPartition part : assignments) {
+ Literal<?>[][] lists = part.lists();
Preconditions.checkArgument(
- filedName.length == 1, "Doris partition does not support nested
field");
+ lists.length > 0, "The number of values in list partition must be
greater than 0");
Preconditions.checkArgument(
- columnNames.contains(filedName[0]), "The partition field must be
one of the columns");
+ Arrays.stream(lists).allMatch(p -> p.length == filedNames.length),
+ "The number of partitioning columns must be consistent");
- partitionColumnsBuilder.add(BACK_QUOTE + filedName[0] + BACK_QUOTE);
+ partitions.add(generatePartitionSqlFragment(part));
}
- String partitionColumns =
-
partitionColumnsBuilder.build().stream().collect(Collectors.joining(","));
- // TODO we currently do not support pre-assign partition when creating
list partitioning table
- partitionSqlBuilder.append(partitionColumns).append(") () ");
- } else {
- throw new IllegalArgumentException("Unsupported partition type of
Doris");
+ partitionSqlBuilder
+ .append(NEW_LINE)
+ .append(partitions.build().stream().collect(Collectors.joining("," +
NEW_LINE)));
}
- sqlBuilder.append(partitionSqlBuilder);
+ partitionSqlBuilder.append(NEW_LINE).append(")");
+ return partitionSqlBuilder;
}
@Override
diff --git
a/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTablePartitionOperations.java
b/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTablePartitionOperations.java
index c3e5a2bba..e075717f4 100644
---
a/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTablePartitionOperations.java
+++
b/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTablePartitionOperations.java
@@ -28,6 +28,7 @@ import static
org.apache.gravitino.catalog.doris.DorisTablePartitionPropertiesMe
import static
org.apache.gravitino.catalog.doris.DorisTablePartitionPropertiesMetadata.VALUES_RANGE;
import static
org.apache.gravitino.catalog.doris.DorisTablePartitionPropertiesMetadata.VISIBLE_VERSION;
import static
org.apache.gravitino.catalog.doris.DorisTablePartitionPropertiesMetadata.VISIBLE_VERSION_TIME;
+import static
org.apache.gravitino.catalog.doris.utils.DorisUtils.generatePartitionSqlFragment;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
@@ -43,7 +44,6 @@ import java.util.Map;
import java.util.Objects;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import java.util.stream.Collectors;
import javax.sql.DataSource;
import org.apache.gravitino.catalog.jdbc.JdbcTable;
import org.apache.gravitino.catalog.jdbc.converter.JdbcExceptionConverter;
@@ -145,8 +145,8 @@ public final class DorisTablePartitionOperations extends
JdbcTablePartitionOpera
try (Connection connection = getConnection(loadedTable.databaseName())) {
Transform partitionInfo = loadedTable.partitioning()[0];
- String addPartitionSqlFormat = "ALTER TABLE `%s` ADD PARTITION `%s`
VALUES %s";
- String partitionValues;
+ String addPartitionSqlFormat = "ALTER TABLE `%s` ADD %s";
+ String partitionSqlFragment;
Partition added;
if (partition instanceof RangePartition) {
@@ -156,7 +156,7 @@ public final class DorisTablePartitionOperations extends
JdbcTablePartitionOpera
loadedTable.name());
RangePartition rangePartition = (RangePartition) partition;
- partitionValues = buildRangePartitionValues(rangePartition);
+ partitionSqlFragment = generatePartitionSqlFragment(rangePartition);
// The partition properties actually cannot be passed into Doris, we
just return an empty
// map instead.
@@ -173,9 +173,18 @@ public final class DorisTablePartitionOperations extends
JdbcTablePartitionOpera
loadedTable.name());
ListPartition listPartition = (ListPartition) partition;
- partitionValues =
- buildListPartitionValues(
- listPartition, ((Transforms.ListTransform)
partitionInfo).fieldNames().length);
+ Literal<?>[][] lists = listPartition.lists();
+ Preconditions.checkArgument(
+ lists.length > 0, "The number of values in list partition must be
greater than 0");
+ Preconditions.checkArgument(
+ Arrays.stream(lists)
+ .allMatch(
+ part ->
+ part.length
+ == ((Transforms.ListTransform)
partitionInfo).fieldNames().length),
+ "The number of partitioning columns must be consistent");
+
+ partitionSqlFragment = generatePartitionSqlFragment(listPartition);
added =
Partitions.list(listPartition.name(), listPartition.lists(),
Collections.emptyMap());
@@ -185,8 +194,7 @@ public final class DorisTablePartitionOperations extends
JdbcTablePartitionOpera
try (Statement statement = connection.createStatement()) {
statement.executeUpdate(
- String.format(
- addPartitionSqlFormat, loadedTable.name(), partition.name(),
partitionValues));
+ String.format(addPartitionSqlFormat, loadedTable.name(),
partitionSqlFragment));
return added;
}
} catch (SQLException e) {
@@ -287,48 +295,4 @@ public final class DorisTablePartitionOperations extends
JdbcTablePartitionOpera
return columnTypes.build();
}
}
-
- private String buildRangePartitionValues(RangePartition rangePartition) {
- Literal<?> upper = rangePartition.upper();
- Literal<?> lower = rangePartition.lower();
- String partitionValues;
- if (Literals.NULL.equals(upper) && Literals.NULL.equals(lower)) {
- partitionValues = "LESS THAN MAXVALUE";
- } else if (Literals.NULL.equals(lower)) {
- partitionValues = "LESS THAN (\"" + upper.value() + "\")";
- } else if (Literals.NULL.equals(upper)) {
- partitionValues = "[(\"" + lower.value() + "\"), (MAXVALUE))";
- } else {
- partitionValues = "[(\"" + lower.value() + "\"), (\"" + upper.value() +
"\"))";
- }
- return partitionValues;
- }
-
- private String buildListPartitionValues(ListPartition listPartition, int
partitionedFieldNums) {
- Literal<?>[][] lists = listPartition.lists();
- Preconditions.checkArgument(
- lists.length > 0, "The number of values in list partition must be
greater than 0");
-
- ImmutableList.Builder<String> listValues = ImmutableList.builder();
- for (Literal<?>[] part : lists) {
- Preconditions.checkArgument(
- part.length == partitionedFieldNums,
- "The number of partitioning columns must be consistent");
-
- StringBuilder values = new StringBuilder();
- if (part.length > 1) {
- values
- .append("(")
- .append(
- Arrays.stream(part)
- .map(p -> "\"" + p.value() + "\"")
- .collect(Collectors.joining(",")))
- .append(")");
- } else {
- values.append("\"").append(part[0].value()).append("\"");
- }
- listValues.add(values.toString());
- }
- return String.format("IN (%s)",
listValues.build().stream().collect(Collectors.joining(",")));
- }
}
diff --git
a/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/utils/DorisUtils.java
b/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/utils/DorisUtils.java
index 69eada5ee..a3d2ccc91 100644
---
a/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/utils/DorisUtils.java
+++
b/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/utils/DorisUtils.java
@@ -18,6 +18,7 @@
*/
package org.apache.gravitino.catalog.doris.utils;
+import com.google.common.collect.ImmutableList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
@@ -25,8 +26,13 @@ import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
+import org.apache.gravitino.rel.expressions.literals.Literal;
+import org.apache.gravitino.rel.expressions.literals.Literals;
import org.apache.gravitino.rel.expressions.transforms.Transform;
import org.apache.gravitino.rel.expressions.transforms.Transforms;
+import org.apache.gravitino.rel.partitions.ListPartition;
+import org.apache.gravitino.rel.partitions.Partition;
+import org.apache.gravitino.rel.partitions.RangePartition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -105,4 +111,69 @@ public final class DorisUtils {
return Optional.empty();
}
}
+
+ /**
+ * Generate sql fragment that create partition in Apache Doris.
+ *
+ * <p>The sql fragment looks like "PARTITION {partitionName} VALUES
{values}", for example:
+ *
+ * <pre>PARTITION `p20240724` VALUES LESS THAN ("2024-07-24")</pre>
+ *
+ * <pre>PARTITION `p20240724_v1` VALUES IN ("2024-07-24", "v1")</pre>
+ *
+ * @param partition The partition to be created.
+ * @return The partition sql fragment.
+ */
+ public static String generatePartitionSqlFragment(Partition partition) {
+ String partitionSqlFragment = "PARTITION `%s` VALUES %s";
+ if (partition instanceof RangePartition) {
+ return String.format(
+ partitionSqlFragment,
+ partition.name(),
+ generateRangePartitionValues((RangePartition) partition));
+ } else if (partition instanceof ListPartition) {
+ return String.format(
+ partitionSqlFragment,
+ partition.name(),
+ generateListPartitionSqlValues((ListPartition) partition));
+ } else {
+ throw new IllegalArgumentException("Unsupported partition type of
Doris");
+ }
+ }
+
+ private static String generateRangePartitionValues(RangePartition
rangePartition) {
+ Literal<?> upper = rangePartition.upper();
+ Literal<?> lower = rangePartition.lower();
+ String partitionValues;
+ if (Literals.NULL.equals(upper) && Literals.NULL.equals(lower)) {
+ partitionValues = "LESS THAN MAXVALUE";
+ } else if (Literals.NULL.equals(lower)) {
+ partitionValues = String.format("LESS THAN (\"%s\")", upper.value());
+ } else if (Literals.NULL.equals(upper)) {
+ partitionValues = String.format("[(\"%s\"), (MAXVALUE))", lower.value());
+ } else {
+ partitionValues = String.format("[(\"%s\"), (\"%s\"))", lower.value(),
upper.value());
+ }
+ return partitionValues;
+ }
+
+ private static String generateListPartitionSqlValues(ListPartition
listPartition) {
+ Literal<?>[][] lists = listPartition.lists();
+ ImmutableList.Builder<String> listValues = ImmutableList.builder();
+ for (Literal<?>[] part : lists) {
+ String values;
+ if (part.length > 1) {
+ values =
+ String.format(
+ "(%s)",
+ Arrays.stream(part)
+ .map(p -> "\"" + p.value() + "\"")
+ .collect(Collectors.joining(",")));
+ } else {
+ values = String.format("\"%s\"", part[0].value());
+ }
+ listValues.add(values);
+ }
+ return String.format("IN (%s)",
listValues.build().stream().collect(Collectors.joining(",")));
+ }
}
diff --git
a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java
b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java
index 2f32dc155..7bc073a0b 100644
---
a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java
+++
b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java
@@ -18,6 +18,7 @@
*/
package org.apache.gravitino.catalog.doris.integration.test;
+import static
org.apache.gravitino.integration.test.util.ITUtils.assertPartition;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -67,6 +68,7 @@ import org.apache.gravitino.rel.indexes.Indexes;
import org.apache.gravitino.rel.partitions.ListPartition;
import org.apache.gravitino.rel.partitions.Partition;
import org.apache.gravitino.rel.partitions.Partitions;
+import org.apache.gravitino.rel.partitions.RangePartition;
import org.apache.gravitino.rel.types.Types;
import org.apache.gravitino.utils.RandomNameUtils;
import org.junit.jupiter.api.AfterAll;
@@ -98,6 +100,7 @@ public class CatalogDorisIT extends AbstractIT {
public String DORIS_COL_NAME1 = "doris_col_name1";
public String DORIS_COL_NAME2 = "doris_col_name2";
public String DORIS_COL_NAME3 = "doris_col_name3";
+ public String DORIS_COL_NAME4 = "doris_col_name4";
// Because the creation of Schema Change is an asynchronous process, we need
to wait for a while
// For more information, you can refer to the comment in
@@ -196,8 +199,9 @@ public class CatalogDorisIT extends AbstractIT {
Column.of(DORIS_COL_NAME1, Types.IntegerType.get(), "col_1_comment",
false, false, null);
Column col2 = Column.of(DORIS_COL_NAME2, Types.VarCharType.of(10),
"col_2_comment");
Column col3 = Column.of(DORIS_COL_NAME3, Types.VarCharType.of(10),
"col_3_comment");
-
- return new Column[] {col1, col2, col3};
+ Column col4 =
+ Column.of(DORIS_COL_NAME4, Types.DateType.get(), "col_4_comment",
false, false, null);
+ return new Column[] {col1, col2, col3, col4};
}
private Map<String, String> createTableProperties() {
@@ -540,29 +544,29 @@ public class CatalogDorisIT extends AbstractIT {
tableCatalog.alterTable(
tableIdentifier,
TableChange.addColumn(
- new String[] {"col_4"}, Types.VarCharType.of(255),
"col_4_comment", true));
+ new String[] {"col_5"}, Types.VarCharType.of(255),
"col_5_comment", true));
Awaitility.await()
.atMost(MAX_WAIT_IN_SECONDS, TimeUnit.SECONDS)
.pollInterval(WAIT_INTERVAL_IN_SECONDS, TimeUnit.SECONDS)
.untilAsserted(
- () -> assertEquals(4,
tableCatalog.loadTable(tableIdentifier).columns().length));
+ () -> assertEquals(5,
tableCatalog.loadTable(tableIdentifier).columns().length));
ITUtils.assertColumn(
- Column.of("col_4", Types.VarCharType.of(255), "col_4_comment"),
- tableCatalog.loadTable(tableIdentifier).columns()[3]);
+ Column.of("col_5", Types.VarCharType.of(255), "col_5_comment"),
+ tableCatalog.loadTable(tableIdentifier).columns()[4]);
// change column position
// TODO: change column position is unstable, add it later
// drop column
tableCatalog.alterTable(
- tableIdentifier, TableChange.deleteColumn(new String[] {"col_4"},
true));
+ tableIdentifier, TableChange.deleteColumn(new String[] {"col_5"},
true));
Awaitility.await()
.atMost(MAX_WAIT_IN_SECONDS, TimeUnit.SECONDS)
.pollInterval(WAIT_INTERVAL_IN_SECONDS, TimeUnit.SECONDS)
.untilAsserted(
- () -> assertEquals(3,
tableCatalog.loadTable(tableIdentifier).columns().length));
+ () -> assertEquals(4,
tableCatalog.loadTable(tableIdentifier).columns().length));
}
@Test
@@ -703,23 +707,11 @@ public class CatalogDorisIT extends AbstractIT {
ListPartition p2 = Partitions.list("p2", p2Values, Collections.emptyMap());
ListPartition p3 = Partitions.list("p3", p3Values, Collections.emptyMap());
ListPartition p1Added = (ListPartition)
tablePartitionOperations.addPartition(p1);
- assertEquals("p1", p1Added.name());
- assertEquals(1, p1Added.lists().length);
- assertEquals(1, p1Added.lists()[0].length);
- assertEquals("1", p1Added.lists()[0][0].value());
- assertEquals(Types.IntegerType.get(), p1Added.lists()[0][0].dataType());
+ assertPartition(p1, p1Added);
ListPartition p2Added = (ListPartition)
tablePartitionOperations.addPartition(p2);
- assertEquals("p2", p2Added.name());
- assertEquals(1, p2Added.lists().length);
- assertEquals(1, p2Added.lists()[0].length);
- assertEquals("2", p2Added.lists()[0][0].value());
- assertEquals(Types.IntegerType.get(), p2Added.lists()[0][0].dataType());
+ assertPartition(p2, p2Added);
ListPartition p3Added = (ListPartition)
tablePartitionOperations.addPartition(p3);
- assertEquals("p3", p3Added.name());
- assertEquals(1, p3Added.lists().length);
- assertEquals(1, p3Added.lists()[0].length);
- assertEquals("3", p3Added.lists()[0][0].value());
- assertEquals(Types.IntegerType.get(), p3Added.lists()[0][0].dataType());
+ assertPartition(p3, p3Added);
// check partitions
Set<String> partitionNames =
@@ -733,43 +725,13 @@ public class CatalogDorisIT extends AbstractIT {
Arrays.stream(tablePartitionOperations.listPartitions())
.collect(Collectors.toMap(p -> p.name(), p -> (ListPartition) p));
assertEquals(3, partitions.size());
- ListPartition actualP1 = partitions.get("p1");
- assertEquals("p1", actualP1.name());
- assertEquals(1, actualP1.lists().length);
- assertEquals(1, actualP1.lists()[0].length);
- assertEquals("1", actualP1.lists()[0][0].value());
- assertEquals(Types.IntegerType.get(), actualP1.lists()[0][0].dataType());
- ListPartition actualP2 = partitions.get("p2");
- assertEquals("p2", actualP2.name());
- assertEquals(1, actualP2.lists().length);
- assertEquals(1, actualP2.lists()[0].length);
- assertEquals("2", actualP2.lists()[0][0].value());
- assertEquals(Types.IntegerType.get(), actualP2.lists()[0][0].dataType());
- ListPartition actualP3 = partitions.get("p3");
- assertEquals("p3", actualP3.name());
- assertEquals(1, actualP3.lists().length);
- assertEquals(1, actualP3.lists()[0].length);
- assertEquals("3", actualP3.lists()[0][0].value());
- assertEquals(Types.IntegerType.get(), actualP3.lists()[0][0].dataType());
-
- actualP1 = (ListPartition) tablePartitionOperations.getPartition("p1");
- assertEquals("p1", actualP1.name());
- assertEquals(1, actualP1.lists().length);
- assertEquals(1, actualP1.lists()[0].length);
- assertEquals("1", actualP1.lists()[0][0].value());
- assertEquals(Types.IntegerType.get(), actualP1.lists()[0][0].dataType());
- actualP2 = (ListPartition) tablePartitionOperations.getPartition("p2");
- assertEquals("p2", actualP2.name());
- assertEquals(1, actualP2.lists().length);
- assertEquals(1, actualP2.lists()[0].length);
- assertEquals("2", actualP2.lists()[0][0].value());
- assertEquals(Types.IntegerType.get(), actualP2.lists()[0][0].dataType());
- actualP3 = (ListPartition) tablePartitionOperations.getPartition("p3");
- assertEquals("p3", actualP3.name());
- assertEquals(1, actualP3.lists().length);
- assertEquals(1, actualP3.lists()[0].length);
- assertEquals("3", actualP3.lists()[0][0].value());
- assertEquals(Types.IntegerType.get(), actualP3.lists()[0][0].dataType());
+ assertPartition(p1, partitions.get("p1"));
+ assertPartition(p2, partitions.get("p2"));
+ assertPartition(p3, partitions.get("p3"));
+
+ assertPartition(p1, tablePartitionOperations.getPartition("p1"));
+ assertPartition(p2, tablePartitionOperations.getPartition("p2"));
+ assertPartition(p3, tablePartitionOperations.getPartition("p3"));
// drop partition
assertTrue(tablePartitionOperations.dropPartition("p3"));
@@ -783,6 +745,124 @@ public class CatalogDorisIT extends AbstractIT {
assertFalse(tablePartitionOperations.dropPartition("p3"));
}
+ @Test
+ void testCreatePartitionedTable() {
+ // create a range-partitioned table with assignments
+ String tableName =
GravitinoITUtils.genRandomName("test_create_range_partitioned_table");
+ NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, tableName);
+ Column[] columns = createColumns();
+ Distribution distribution = createDistribution();
+ Index[] indexes =
+ new Index[] {
+ Indexes.of(Index.IndexType.PRIMARY_KEY, "k1_index", new String[][]
{{DORIS_COL_NAME1}})
+ };
+ Map<String, String> properties = createTableProperties();
+ Literal todayLiteral = Literals.of("2024-07-24", Types.DateType.get());
+ Literal tomorrowLiteral = Literals.of("2024-07-25", Types.DateType.get());
+ RangePartition p1 = Partitions.range("p1", todayLiteral, Literals.NULL,
Collections.emptyMap());
+ RangePartition p2 =
+ Partitions.range("p2", tomorrowLiteral, todayLiteral,
Collections.emptyMap());
+ RangePartition p3 =
+ Partitions.range("p3", Literals.NULL, tomorrowLiteral,
Collections.emptyMap());
+ Transform[] partitioning = {
+ Transforms.range(new String[] {DORIS_COL_NAME4}, new RangePartition[]
{p1, p2, p3})
+ };
+ TableCatalog tableCatalog = catalog.asTableCatalog();
+ tableCatalog.createTable(
+ tableIdentifier,
+ columns,
+ table_comment,
+ properties,
+ partitioning,
+ distribution,
+ null,
+ indexes);
+ Table loadedTable = tableCatalog.loadTable(tableIdentifier);
+ ITUtils.assertionsTableInfo(
+ tableName,
+ table_comment,
+ Arrays.asList(columns),
+ properties,
+ indexes,
+ new Transform[] {Transforms.range(new String[] {DORIS_COL_NAME4})},
+ loadedTable);
+
+ // assert partition info
+ SupportsPartitions tablePartitionOperations =
loadedTable.supportPartitions();
+ Map<String, RangePartition> loadedRangePartitions =
+ Arrays.stream(tablePartitionOperations.listPartitions())
+ .collect(Collectors.toMap(Partition::name, p -> (RangePartition)
p));
+ assertTrue(loadedRangePartitions.size() == 3);
+ assertTrue(loadedRangePartitions.containsKey("p1"));
+ assertPartition(
+ Partitions.range(
+ "p1",
+ todayLiteral,
+ Literals.of("0000-01-01", Types.DateType.get()),
+ Collections.emptyMap()),
+ loadedRangePartitions.get("p1"));
+ assertTrue(loadedRangePartitions.containsKey("p2"));
+ assertPartition(p2, loadedRangePartitions.get("p2"));
+ assertTrue(loadedRangePartitions.containsKey("p3"));
+ assertPartition(
+ Partitions.range(
+ "p3",
+ Literals.of("MAXVALUE", Types.DateType.get()),
+ tomorrowLiteral,
+ Collections.emptyMap()),
+ loadedRangePartitions.get("p3"));
+
+ // create a list-partitioned table with assignments
+ tableName =
GravitinoITUtils.genRandomName("test_create_list_partitioned_table");
+ tableIdentifier = NameIdentifier.of(schemaName, tableName);
+ Literal<Integer> integerLiteral1 = Literals.integerLiteral(1);
+ Literal<Integer> integerLiteral2 = Literals.integerLiteral(2);
+ ListPartition p4 =
+ Partitions.list(
+ "p4",
+ new Literal[][] {{integerLiteral1, todayLiteral},
{integerLiteral1, tomorrowLiteral}},
+ Collections.emptyMap());
+ ListPartition p5 =
+ Partitions.list(
+ "p5",
+ new Literal[][] {{integerLiteral2, todayLiteral},
{integerLiteral2, tomorrowLiteral}},
+ Collections.emptyMap());
+ partitioning =
+ new Transform[] {
+ Transforms.list(
+ new String[][] {{DORIS_COL_NAME1}, {DORIS_COL_NAME4}}, new
ListPartition[] {p4, p5})
+ };
+ tableCatalog.createTable(
+ tableIdentifier,
+ columns,
+ table_comment,
+ properties,
+ partitioning,
+ distribution,
+ null,
+ indexes);
+ loadedTable = tableCatalog.loadTable(tableIdentifier);
+ ITUtils.assertionsTableInfo(
+ tableName,
+ table_comment,
+ Arrays.asList(columns),
+ properties,
+ indexes,
+ new Transform[] {Transforms.list(new String[][] {{DORIS_COL_NAME1},
{DORIS_COL_NAME4}})},
+ loadedTable);
+
+ // assert partition info
+ tablePartitionOperations = loadedTable.supportPartitions();
+ Map<String, ListPartition> loadedListPartitions =
+ Arrays.stream(tablePartitionOperations.listPartitions())
+ .collect(Collectors.toMap(Partition::name, p -> (ListPartition)
p));
+ assertTrue(loadedListPartitions.size() == 2);
+ assertTrue(loadedListPartitions.containsKey("p4"));
+ assertPartition(p4, loadedListPartitions.get("p4"));
+ assertTrue(loadedListPartitions.containsKey("p5"));
+ assertPartition(p5, loadedListPartitions.get("p5"));
+ }
+
@Test
void testNonPartitionedTable() {
// create a non-partitioned table
diff --git
a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTableOperations.java
b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTableOperations.java
index 62c763f3d..a252a610b 100644
---
a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTableOperations.java
+++
b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTableOperations.java
@@ -18,7 +18,11 @@
*/
package org.apache.gravitino.catalog.doris.operation;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
import com.google.common.collect.Maps;
+import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -26,17 +30,27 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import org.apache.gravitino.catalog.doris.converter.DorisTypeConverter;
import org.apache.gravitino.catalog.jdbc.JdbcColumn;
import org.apache.gravitino.catalog.jdbc.JdbcTable;
+import org.apache.gravitino.catalog.jdbc.converter.JdbcTypeConverter;
+import
org.apache.gravitino.catalog.jdbc.operation.JdbcTablePartitionOperations;
import org.apache.gravitino.integration.test.util.GravitinoITUtils;
import org.apache.gravitino.rel.TableChange;
import org.apache.gravitino.rel.expressions.NamedReference;
import org.apache.gravitino.rel.expressions.distributions.Distribution;
import org.apache.gravitino.rel.expressions.distributions.Distributions;
+import org.apache.gravitino.rel.expressions.literals.Literal;
+import org.apache.gravitino.rel.expressions.literals.Literals;
import org.apache.gravitino.rel.expressions.transforms.Transform;
import org.apache.gravitino.rel.expressions.transforms.Transforms;
import org.apache.gravitino.rel.indexes.Index;
import org.apache.gravitino.rel.indexes.Indexes;
+import org.apache.gravitino.rel.partitions.ListPartition;
+import org.apache.gravitino.rel.partitions.Partition;
+import org.apache.gravitino.rel.partitions.Partitions;
+import org.apache.gravitino.rel.partitions.RangePartition;
import org.apache.gravitino.rel.types.Type;
import org.apache.gravitino.rel.types.Types;
import org.apache.gravitino.utils.RandomNameUtils;
@@ -48,6 +62,7 @@ import org.testcontainers.shaded.org.awaitility.Awaitility;
@Tag("gravitino-docker-test")
public class TestDorisTableOperations extends TestDoris {
+ private static final JdbcTypeConverter TYPE_CONVERTER = new
DorisTypeConverter();
private static final Type VARCHAR_255 = Types.VarCharType.of(255);
private static final Type VARCHAR_1024 = Types.VarCharType.of(1024);
@@ -111,7 +126,7 @@ public class TestDorisTableOperations extends TestDoris {
distribution,
indexes);
List<String> listTables = TABLE_OPERATIONS.listTables(databaseName);
- Assertions.assertTrue(listTables.contains(tableName));
+ assertTrue(listTables.contains(tableName));
JdbcTable load = TABLE_OPERATIONS.load(databaseName, tableName);
assertionsTableInfo(
tableName, tableComment, columns, properties, indexes,
Transforms.EMPTY_TRANSFORM, load);
@@ -444,7 +459,7 @@ public class TestDorisTableOperations extends TestDoris {
}
@Test
- public void testCreateTableWithPartition() {
+ public void testCreatePartitionedTable() {
String tableComment = "partition_table_comment";
JdbcColumn col1 =
JdbcColumn.builder()
@@ -469,7 +484,19 @@ public class TestDorisTableOperations extends TestDoris {
// create table with range partition
String rangePartitionTableName =
GravitinoITUtils.genRandomName("range_partition_table");
- Transform[] rangePartition = new Transform[] {Transforms.range(new
String[] {col4.name()})};
+ LocalDate today = LocalDate.now();
+ LocalDate tomorrow = today.plusDays(1);
+ Literal<LocalDate> todayLiteral = Literals.dateLiteral(today);
+ Literal<LocalDate> tomorrowLiteral = Literals.dateLiteral(tomorrow);
+ RangePartition rangePartition1 = Partitions.range("p1", todayLiteral,
Literals.NULL, null);
+ RangePartition rangePartition2 = Partitions.range("p2", tomorrowLiteral,
todayLiteral, null);
+ RangePartition rangePartition3 = Partitions.range("p3", Literals.NULL,
tomorrowLiteral, null);
+ Transform[] rangePartition =
+ new Transform[] {
+ Transforms.range(
+ new String[] {col4.name()},
+ new RangePartition[] {rangePartition1, rangePartition2,
rangePartition3})
+ };
TABLE_OPERATIONS.create(
databaseName,
rangePartitionTableName,
@@ -486,13 +513,49 @@ public class TestDorisTableOperations extends TestDoris {
columns,
Collections.emptyMap(),
null,
- rangePartition,
+ new Transform[] {Transforms.range(new String[] {col4.name()})},
rangePartitionTable);
+ // assert partition info
+ JdbcTablePartitionOperations tablePartitionOperations =
+ new DorisTablePartitionOperations(
+ DATA_SOURCE, rangePartitionTable, JDBC_EXCEPTION_CONVERTER,
TYPE_CONVERTER);
+ Map<String, RangePartition> loadedRangePartitions =
+ Arrays.stream(tablePartitionOperations.listPartitions())
+ .collect(Collectors.toMap(Partition::name, p -> (RangePartition)
p));
+ assertTrue(loadedRangePartitions.containsKey("p1"));
+ RangePartition actualP1 = loadedRangePartitions.get("p1");
+ assertEquals(todayLiteral, actualP1.upper());
+ assertEquals(Literals.of("0000-01-01", Types.DateType.get()),
actualP1.lower());
+ assertTrue(loadedRangePartitions.containsKey("p2"));
+ RangePartition actualP2 = loadedRangePartitions.get("p2");
+ assertEquals(tomorrowLiteral, actualP2.upper());
+ assertEquals(todayLiteral, actualP2.lower());
+ assertTrue(loadedRangePartitions.containsKey("p3"));
+ RangePartition actualP3 = loadedRangePartitions.get("p3");
+ assertEquals(Literals.of("MAXVALUE", Types.DateType.get()),
actualP3.upper());
+ assertEquals(tomorrowLiteral, actualP3.lower());
+
// create table with list partition
String listPartitionTableName =
GravitinoITUtils.genRandomName("list_partition_table");
+ Literal<Integer> integerLiteral1 = Literals.integerLiteral(1);
+ Literal<Integer> integerLiteral2 = Literals.integerLiteral(2);
+ ListPartition listPartition1 =
+ Partitions.list(
+ "p1",
+ new Literal[][] {{integerLiteral1, todayLiteral},
{integerLiteral1, tomorrowLiteral}},
+ null);
+ ListPartition listPartition2 =
+ Partitions.list(
+ "p2",
+ new Literal[][] {{integerLiteral2, todayLiteral},
{integerLiteral2, tomorrowLiteral}},
+ null);
Transform[] listPartition =
- new Transform[] {Transforms.list(new String[] {col1.name()}, new
String[] {col4.name()})};
+ new Transform[] {
+ Transforms.list(
+ new String[][] {{col1.name()}, {col4.name()}},
+ new ListPartition[] {listPartition1, listPartition2})
+ };
TABLE_OPERATIONS.create(
databaseName,
listPartitionTableName,
@@ -509,7 +572,19 @@ public class TestDorisTableOperations extends TestDoris {
columns,
Collections.emptyMap(),
null,
- listPartition,
+ new Transform[] {Transforms.list(new String[][] {{col1.name()},
{col4.name()}})},
listPartitionTable);
+
+ // assert partition info
+ tablePartitionOperations =
+ new DorisTablePartitionOperations(
+ DATA_SOURCE, listPartitionTable, JDBC_EXCEPTION_CONVERTER,
TYPE_CONVERTER);
+ Map<String, ListPartition> loadedListPartitions =
+ Arrays.stream(tablePartitionOperations.listPartitions())
+ .collect(Collectors.toMap(Partition::name, p -> (ListPartition) p,
(p1, p2) -> p2));
+ assertTrue(loadedListPartitions.containsKey("p1"));
+ assertTrue(Arrays.deepEquals(listPartition1.lists(),
loadedListPartitions.get("p1").lists()));
+ assertTrue(loadedListPartitions.containsKey("p2"));
+ assertTrue(Arrays.deepEquals(listPartition2.lists(),
loadedListPartitions.get("p2").lists()));
}
}
diff --git
a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/utils/TestDorisUtils.java
b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/utils/TestDorisUtils.java
index f1428f2b0..26d011af8 100644
---
a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/utils/TestDorisUtils.java
+++
b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/utils/TestDorisUtils.java
@@ -18,13 +18,21 @@
*/
package org.apache.gravitino.catalog.doris.utils;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
+import org.apache.gravitino.rel.expressions.literals.Literal;
+import org.apache.gravitino.rel.expressions.literals.Literals;
import org.apache.gravitino.rel.expressions.transforms.Transform;
import org.apache.gravitino.rel.expressions.transforms.Transforms;
-import org.junit.jupiter.api.Assertions;
+import org.apache.gravitino.rel.partitions.Partition;
+import org.apache.gravitino.rel.partitions.Partitions;
+import org.apache.gravitino.rel.types.Types;
import org.junit.jupiter.api.Test;
public class TestDorisUtils {
@@ -33,17 +41,17 @@ public class TestDorisUtils {
// Test when properties is null
Map<String, String> properties = null;
String result = DorisUtils.generatePropertiesSql(properties);
- Assertions.assertEquals("", result);
+ assertEquals("", result);
// Test when properties is empty
properties = Collections.emptyMap();
result = DorisUtils.generatePropertiesSql(properties);
- Assertions.assertEquals("", result);
+ assertEquals("", result);
// Test when properties has single entry
properties = Collections.singletonMap("key", "value");
result = DorisUtils.generatePropertiesSql(properties);
- Assertions.assertEquals(" PROPERTIES (\n\"key\"=\"value\"\n)", result);
+ assertEquals(" PROPERTIES (\n\"key\"=\"value\"\n)", result);
// Test when properties has multiple entries
properties = new HashMap<>();
@@ -53,7 +61,7 @@ public class TestDorisUtils {
String expectedStr = " PROPERTIES
(\n\"key1\"=\"value1\",\n\"key2\"=\"value2\"\n)";
result = DorisUtils.generatePropertiesSql(properties);
- Assertions.assertEquals(expectedStr, result);
+ assertEquals(expectedStr, result);
}
@Test
@@ -62,27 +70,27 @@ public class TestDorisUtils {
String createTableSql =
"CREATE TABLE `testTable` (\n`testColumn` STRING NOT NULL COMMENT
'test comment'\n) ENGINE=OLAP\nCOMMENT \"test comment\"";
Map<String, String> result =
DorisUtils.extractPropertiesFromSql(createTableSql);
- Assertions.assertTrue(result.isEmpty());
+ assertTrue(result.isEmpty());
// Test when properties exist
createTableSql =
"CREATE TABLE `testTable` (\n`testColumn` STRING NOT NULL COMMENT
'test comment'\n) ENGINE=OLAP\nCOMMENT \"test comment\"\nPROPERTIES
(\n\"test_property\"=\"test_value\"\n)";
result = DorisUtils.extractPropertiesFromSql(createTableSql);
- Assertions.assertEquals("test_value", result.get("test_property"));
+ assertEquals("test_value", result.get("test_property"));
// Test when multiple properties exist
createTableSql =
"CREATE TABLE `testTable` (\n`testColumn` STRING NOT NULL COMMENT
'test comment'\n) ENGINE=OLAP\nCOMMENT \"test comment\"\nPROPERTIES
(\n\"test_property1\"=\"test_value1\",\n\"test_property2\"=\"test_value2\"\n)";
result = DorisUtils.extractPropertiesFromSql(createTableSql);
- Assertions.assertEquals("test_value1", result.get("test_property1"));
- Assertions.assertEquals("test_value2", result.get("test_property2"));
+ assertEquals("test_value1", result.get("test_property1"));
+ assertEquals("test_value2", result.get("test_property2"));
// test when properties has blank
createTableSql =
"CREATE DATABASE `test`\nPROPERTIES (\n\"property1\" =
\"value1\",\n\"comment\"= \"comment\"\n)";
result = DorisUtils.extractPropertiesFromSql(createTableSql);
- Assertions.assertEquals("value1", result.get("property1"));
- Assertions.assertEquals("comment", result.get("comment"));
+ assertEquals("value1", result.get("property1"));
+ assertEquals("comment", result.get("comment"));
}
@Test
@@ -91,27 +99,76 @@ public class TestDorisUtils {
String createTableSql =
"CREATE TABLE `testTable` (\n`col1` date NOT NULL\n) ENGINE=OLAP\n
PARTITION BY RANGE(`col1`)\n()\n DISTRIBUTED BY HASH(`col1`) BUCKETS 2";
Optional<Transform> transform =
DorisUtils.extractPartitionInfoFromSql(createTableSql);
- Assertions.assertTrue(transform.isPresent());
- Assertions.assertEquals(Transforms.range(new String[] {"col1"}),
transform.get());
+ assertTrue(transform.isPresent());
+ assertEquals(Transforms.range(new String[] {"col1"}), transform.get());
// test list partition
createTableSql =
"CREATE TABLE `testTable` (\n`col1` int(11) NOT NULL\n) ENGINE=OLAP\n
PARTITION BY LIST(`col1`)\n()\n DISTRIBUTED BY HASH(`col1`) BUCKETS 2";
transform = DorisUtils.extractPartitionInfoFromSql(createTableSql);
- Assertions.assertTrue(transform.isPresent());
- Assertions.assertEquals(Transforms.list(new String[][] {{"col1"}}),
transform.get());
+ assertTrue(transform.isPresent());
+ assertEquals(Transforms.list(new String[][] {{"col1"}}), transform.get());
// test multi-column list partition
createTableSql =
"CREATE TABLE `testTable` (\n`col1` date NOT NULL,\n`col2` int(11) NOT
NULL\n) ENGINE=OLAP\n PARTITION BY LIST(`col1`, `col2`)\n()\n DISTRIBUTED BY
HASH(`col1`) BUCKETS 2";
transform = DorisUtils.extractPartitionInfoFromSql(createTableSql);
- Assertions.assertTrue(transform.isPresent());
- Assertions.assertEquals(Transforms.list(new String[][] {{"col1"},
{"col2"}}), transform.get());
+ assertTrue(transform.isPresent());
+ assertEquals(Transforms.list(new String[][] {{"col1"}, {"col2"}}),
transform.get());
// test non-partitioned table
createTableSql =
"CREATE TABLE `testTable` (\n`testColumn` STRING NOT NULL COMMENT
'test comment'\n) ENGINE=OLAP\nCOMMENT \"test comment\"";
transform = DorisUtils.extractPartitionInfoFromSql(createTableSql);
- Assertions.assertFalse(transform.isPresent());
+ assertFalse(transform.isPresent());
+ }
+
+ @Test
+ public void testGeneratePartitionSqlFragment() {
+ // test range partition
+ Partition partition = Partitions.range("p1", Literals.NULL, Literals.NULL,
null);
+ String partitionSqlFragment =
DorisUtils.generatePartitionSqlFragment(partition);
+ assertEquals("PARTITION `p1` VALUES LESS THAN MAXVALUE",
partitionSqlFragment);
+
+ partition =
+ Partitions.range(
+ "p2", Literals.of("2024-07-23", Types.DateType.get()),
Literals.NULL, null);
+ partitionSqlFragment = DorisUtils.generatePartitionSqlFragment(partition);
+ assertEquals("PARTITION `p2` VALUES LESS THAN (\"2024-07-23\")",
partitionSqlFragment);
+
+ partition =
+ Partitions.range(
+ "p3",
+ Literals.of("2024-07-24", Types.DateType.get()),
+ Literals.of("2024-07-23", Types.DateType.get()),
+ null);
+ partitionSqlFragment = DorisUtils.generatePartitionSqlFragment(partition);
+ assertEquals(
+ "PARTITION `p3` VALUES [(\"2024-07-23\"), (\"2024-07-24\"))",
partitionSqlFragment);
+
+ partition =
+ Partitions.range(
+ "p4", Literals.NULL, Literals.of("2024-07-24",
Types.DateType.get()), null);
+ partitionSqlFragment = DorisUtils.generatePartitionSqlFragment(partition);
+ assertEquals("PARTITION `p4` VALUES [(\"2024-07-24\"), (MAXVALUE))",
partitionSqlFragment);
+
+ // test list partition
+ Literal[][] p5values = {{Literals.of("2024-07-24", Types.DateType.get())}};
+ partition = Partitions.list("p5", p5values, Collections.emptyMap());
+ partitionSqlFragment = DorisUtils.generatePartitionSqlFragment(partition);
+ assertEquals("PARTITION `p5` VALUES IN (\"2024-07-24\")",
partitionSqlFragment);
+
+ Literal[][] p6values = {{Literals.integerLiteral(1)},
{Literals.integerLiteral(2)}};
+ partition = Partitions.list("p6", p6values, Collections.emptyMap());
+ partitionSqlFragment = DorisUtils.generatePartitionSqlFragment(partition);
+ assertEquals("PARTITION `p6` VALUES IN (\"1\",\"2\")",
partitionSqlFragment);
+
+ Literal[][] p7values = {
+ {Literals.integerLiteral(1), Literals.integerLiteral(2)},
+ {Literals.integerLiteral(3), Literals.integerLiteral(4)}
+ };
+ partition = Partitions.list("p7", p7values, Collections.emptyMap());
+ partitionSqlFragment = DorisUtils.generatePartitionSqlFragment(partition);
+ assertEquals("PARTITION `p7` VALUES IN ((\"1\",\"2\"),(\"3\",\"4\"))",
partitionSqlFragment);
}
}
diff --git
a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/ITUtils.java
b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/ITUtils.java
index bb97d04a3..596b0c23c 100644
---
a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/ITUtils.java
+++
b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/ITUtils.java
@@ -35,10 +35,15 @@ import java.util.stream.Collectors;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.gravitino.dto.rel.ColumnDTO;
import org.apache.gravitino.dto.rel.expressions.LiteralDTO;
+import org.apache.gravitino.dto.rel.partitions.IdentityPartitionDTO;
+import org.apache.gravitino.dto.rel.partitions.ListPartitionDTO;
+import org.apache.gravitino.dto.rel.partitions.PartitionDTO;
+import org.apache.gravitino.dto.rel.partitions.RangePartitionDTO;
import org.apache.gravitino.rel.Column;
import org.apache.gravitino.rel.Table;
import org.apache.gravitino.rel.expressions.transforms.Transform;
import org.apache.gravitino.rel.indexes.Index;
+import org.apache.gravitino.rel.partitions.Partition;
import org.junit.jupiter.api.Assertions;
public class ITUtils {
@@ -139,5 +144,36 @@ public class ITUtils {
}
}
+ public static void assertPartition(Partition expected, Partition actual) {
+ if (!(expected instanceof PartitionDTO)) {
+ expected = toDTO(expected);
+ }
+ if (!(actual instanceof PartitionDTO)) {
+ actual = toDTO(actual);
+ }
+
+ Assertions.assertEquals(expected.name(), actual.name());
+ Assertions.assertEquals(((PartitionDTO) expected).type(), ((PartitionDTO)
actual).type());
+ if (expected instanceof RangePartitionDTO) {
+ Assertions.assertEquals(
+ ((RangePartitionDTO) expected).upper(), ((RangePartitionDTO)
actual).upper());
+ Assertions.assertEquals(
+ ((RangePartitionDTO) expected).lower(), ((RangePartitionDTO)
actual).lower());
+ } else if (expected instanceof ListPartitionDTO) {
+ Assertions.assertTrue(
+ Arrays.deepEquals(
+ ((ListPartitionDTO) expected).lists(), ((ListPartitionDTO)
actual).lists()));
+ } else if (expected instanceof IdentityPartitionDTO) {
+ Assertions.assertTrue(
+ Arrays.deepEquals(
+ ((IdentityPartitionDTO) expected).fieldNames(),
+ ((IdentityPartitionDTO) actual).fieldNames()));
+ Assertions.assertTrue(
+ Arrays.equals(
+ ((IdentityPartitionDTO) expected).values(),
+ ((IdentityPartitionDTO) actual).values()));
+ }
+ }
+
private ITUtils() {}
}
