This is an automated email from the ASF dual-hosted git repository.
jerryshao pushed a commit to branch branch-1.3
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/branch-1.3 by this push:
new e6184a5eb4 [#11829] fix(doris): support replication allocation
property (#12556)
e6184a5eb4 is described below
commit e6184a5eb43adb315b93224f2bb32cff53b5b266
Author: hutiefang76 <[email protected]>
AuthorDate: Mon Aug 24 13:58:47 2026 +0800
[#11829] fix(doris): support replication allocation property (#12556)
### What changed
Backports the merged main-branch fix from #11877 to branch-1.3. The JDBC
Doris catalog now accepts the Doris 2.1+ replication_allocation property
and avoids automatically adding replication_num=1 when that mutually
exclusive property is already supplied.
### Scope
This backport contains only the original property metadata, validation,
SQL-generation, focused unit coverage, existing Doris integration
coverage, and documentation changes.
### Verification
- git diff --check
- JDK 17: ./gradlew :catalogs:catalog-jdbc-doris:test --tests
org.apache.gravitino.catalog.doris.TestDorisCatalog --tests
org.apache.gravitino.catalog.doris.operation.TestDorisTableOperationsSqlGeneration
-PskipITs
The Docker-backed integration test is retained in the patch and will be
validated by project CI.
Backport of #11877. Closes #11829
---
.../doris/DorisTablePropertiesMetadata.java | 7 +++
.../doris/operation/DorisTableOperations.java | 11 +++-
.../gravitino/catalog/doris/TestDorisCatalog.java | 12 ++++
.../doris/integration/test/CatalogDorisIT.java | 25 ++++++++
.../TestDorisTableOperationsSqlGeneration.java | 66 ++++++++++++++++++++++
docs/jdbc-doris-catalog.md | 1 +
6 files changed, 121 insertions(+), 1 deletion(-)
diff --git
a/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/DorisTablePropertiesMetadata.java
b/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/DorisTablePropertiesMetadata.java
index 4cad4c0351..c19bf15e2b 100644
---
a/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/DorisTablePropertiesMetadata.java
+++
b/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/DorisTablePropertiesMetadata.java
@@ -28,6 +28,7 @@ import org.apache.gravitino.connector.PropertyEntry;
public class DorisTablePropertiesMetadata extends JdbcTablePropertiesMetadata {
public static final String REPLICATION_FACTOR = "replication_num";
+ public static final String REPLICATION_ALLOCATION = "replication_allocation";
public static final int DEFAULT_REPLICATION_FACTOR = 1;
public static final int DEFAULT_REPLICATION_FACTOR_IN_SERVER_SIDE = 3;
@@ -42,6 +43,12 @@ public class DorisTablePropertiesMetadata extends
JdbcTablePropertiesMetadata {
+ " the default value will be used",
false /* immutable */,
DEFAULT_REPLICATION_FACTOR, /* default value */
+ false /* hidden */),
+ PropertyEntry.stringOptionalPropertyEntry(
+ REPLICATION_ALLOCATION,
+ "The replication allocation policy for the table.",
+ false /* immutable */,
+ null /* default value */,
false /* hidden */));
PROPERTIES_METADATA = Maps.uniqueIndex(propertyEntries,
PropertyEntry::getName);
diff --git
a/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTableOperations.java
b/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTableOperations.java
index 926c193267..657dc04c12 100644
---
a/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTableOperations.java
+++
b/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTableOperations.java
@@ -20,6 +20,7 @@ package org.apache.gravitino.catalog.doris.operation;
import static
org.apache.gravitino.catalog.doris.DorisCatalog.DORIS_TABLE_PROPERTIES_META;
import static
org.apache.gravitino.catalog.doris.DorisTablePropertiesMetadata.DEFAULT_REPLICATION_FACTOR_IN_SERVER_SIDE;
+import static
org.apache.gravitino.catalog.doris.DorisTablePropertiesMetadata.REPLICATION_ALLOCATION;
import static
org.apache.gravitino.catalog.doris.DorisTablePropertiesMetadata.REPLICATION_FACTOR;
import static
org.apache.gravitino.catalog.doris.utils.DorisUtils.generatePartitionSqlFragment;
import static
org.apache.gravitino.catalog.jdbc.utils.JdbcConnectorUtils.escapeSqlLiteral;
@@ -179,9 +180,17 @@ public class DorisTableOperations extends
JdbcTableOperations {
resultMap = new HashMap<>(properties);
}
+ Preconditions.checkArgument(
+ !resultMap.containsKey(REPLICATION_FACTOR)
+ || !resultMap.containsKey(REPLICATION_ALLOCATION),
+ "Properties '%s' and '%s' cannot be set at the same time",
+ REPLICATION_FACTOR,
+ REPLICATION_ALLOCATION);
+
// If the backend server is less than
DEFAULT_REPLICATION_FACTOR_IN_SERVER_SIDE (3), we need to
// set the property 'replication_num' to 1 explicitly.
- if (!resultMap.containsKey(REPLICATION_FACTOR)) {
+ if (!resultMap.containsKey(REPLICATION_FACTOR)
+ && !resultMap.containsKey(REPLICATION_ALLOCATION)) {
// Try to check the number of backend servers using `show backends`,
this SQL is supported by
// all versions of Doris
String query = "show backends";
diff --git
a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/TestDorisCatalog.java
b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/TestDorisCatalog.java
index 039a15b0dc..b12e3cf095 100644
---
a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/TestDorisCatalog.java
+++
b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/TestDorisCatalog.java
@@ -19,6 +19,7 @@
package org.apache.gravitino.catalog.doris;
+import static
org.apache.gravitino.catalog.doris.DorisTablePropertiesMetadata.REPLICATION_ALLOCATION;
import static
org.apache.gravitino.catalog.doris.DorisTablePropertiesMetadata.REPLICATION_FACTOR;
import java.util.Map;
@@ -33,6 +34,7 @@ public class TestDorisCatalog {
DorisTablePropertiesMetadata dorisTablePropertiesMetadata = new
DorisTablePropertiesMetadata();
Map<String, PropertyEntry<?>> propertyEntryMap =
dorisTablePropertiesMetadata.specificPropertyEntries();
+ Assertions.assertEquals(2, propertyEntryMap.size());
Assertions.assertTrue(propertyEntryMap.containsKey(REPLICATION_FACTOR));
PropertyEntry<?> propertyEntry = propertyEntryMap.get(REPLICATION_FACTOR);
@@ -41,5 +43,15 @@ public class TestDorisCatalog {
Assertions.assertEquals(
DorisTablePropertiesMetadata.DEFAULT_REPLICATION_FACTOR,
propertyEntry.getDefaultValue());
Assertions.assertFalse(propertyEntry.isHidden());
+
+
Assertions.assertTrue(propertyEntryMap.containsKey(REPLICATION_ALLOCATION));
+ PropertyEntry<?> replicationAllocation =
propertyEntryMap.get(REPLICATION_ALLOCATION);
+ Assertions.assertEquals(REPLICATION_ALLOCATION,
replicationAllocation.getName());
+ Assertions.assertFalse(replicationAllocation.isRequired());
+ Assertions.assertFalse(replicationAllocation.isImmutable());
+ Assertions.assertFalse(replicationAllocation.isReserved());
+ Assertions.assertFalse(replicationAllocation.isHidden());
+ Assertions.assertEquals(String.class, replicationAllocation.getJavaType());
+ Assertions.assertNull(replicationAllocation.getDefaultValue());
}
}
diff --git
a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java
b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java
index e474c10fa7..9d1a90f60b 100644
---
a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java
+++
b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java
@@ -18,6 +18,8 @@
*/
package org.apache.gravitino.catalog.doris.integration.test;
+import static
org.apache.gravitino.catalog.doris.DorisTablePropertiesMetadata.REPLICATION_ALLOCATION;
+import static
org.apache.gravitino.catalog.doris.DorisTablePropertiesMetadata.REPLICATION_FACTOR;
import static
org.apache.gravitino.integration.test.util.ITUtils.assertPartition;
import static
org.apache.gravitino.rel.Column.DEFAULT_VALUE_OF_CURRENT_TIMESTAMP;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -380,6 +382,29 @@ public class CatalogDorisIT extends BaseIT {
renamedTable);
}
+ @Test
+ void testCreateTableWithReplicationAllocation() {
+ NameIdentifier tableIdentifier =
+ NameIdentifier.of(
+ schemaName,
GravitinoITUtils.genRandomName("doris_replication_allocation"));
+ String replicationAllocation = "tag.location.default: 1";
+ Map<String, String> properties = ImmutableMap.of(REPLICATION_ALLOCATION,
replicationAllocation);
+ TableCatalog tableCatalog = catalog.asTableCatalog();
+
+ tableCatalog.createTable(
+ tableIdentifier,
+ createColumns(),
+ table_comment,
+ properties,
+ Transforms.EMPTY_TRANSFORM,
+ createDistribution(),
+ null);
+
+ Table loadedTable = tableCatalog.loadTable(tableIdentifier);
+ assertEquals(replicationAllocation,
loadedTable.properties().get(REPLICATION_ALLOCATION));
+ assertFalse(loadedTable.properties().containsKey(REPLICATION_FACTOR));
+ }
+
@Test
void testDorisIllegalTableName() {
Map<String, String> properties = createTableProperties();
diff --git
a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTableOperationsSqlGeneration.java
b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTableOperationsSqlGeneration.java
index 68b442bbe9..ba7b665c01 100644
---
a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTableOperationsSqlGeneration.java
+++
b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTableOperationsSqlGeneration.java
@@ -18,11 +18,16 @@
*/
package org.apache.gravitino.catalog.doris.operation;
+import static
org.apache.gravitino.catalog.doris.DorisTablePropertiesMetadata.REPLICATION_ALLOCATION;
+import static
org.apache.gravitino.catalog.doris.DorisTablePropertiesMetadata.REPLICATION_FACTOR;
+
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.Statement;
import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
import javax.sql.DataSource;
import org.apache.gravitino.catalog.doris.converter.DorisTypeConverter;
import org.apache.gravitino.catalog.jdbc.JdbcColumn;
@@ -71,6 +76,10 @@ public class TestDorisTableOperationsSqlGeneration {
}
}
+ public void setDataSource(DataSource dataSource) {
+ super.dataSource = dataSource;
+ }
+
public String createTableSql(
String tableName, JdbcColumn[] columns, Distribution distribution) {
return createTableSql(tableName, columns, distribution, "comment");
@@ -495,4 +504,61 @@ public class TestDorisTableOperationsSqlGeneration {
Assertions.assertTrue(DorisTableOperations.isVersionAtLeast("2.1.1", 2, 1,
0));
Assertions.assertFalse(DorisTableOperations.isVersionAtLeast("2.1.0", 2,
1, 1));
}
+
+ @Test
+ public void
testAppendNecessaryPropertiesAddsReplicationNumWhenBackendsAreNotEnough()
+ throws Exception {
+ TestableDorisTableOperations ops = new TestableDorisTableOperations();
+ ops.setDataSource(mockBackendDataSource(1));
+
+ Map<String, String> properties =
ops.appendNecessaryProperties(Collections.emptyMap());
+
+ Assertions.assertEquals("1", properties.get(REPLICATION_FACTOR));
+ }
+
+ @Test
+ public void testAppendNecessaryPropertiesKeepsReplicationAllocation() throws
Exception {
+ TestableDorisTableOperations ops = new TestableDorisTableOperations();
+ ops.setDataSource(mockBackendDataSource(1));
+
+ Map<String, String> properties = new HashMap<>();
+ properties.put(REPLICATION_ALLOCATION, "tag.location.default: 1");
+
+ Map<String, String> result = ops.appendNecessaryProperties(properties);
+
+ Assertions.assertEquals("tag.location.default: 1",
result.get(REPLICATION_ALLOCATION));
+ Assertions.assertFalse(result.containsKey(REPLICATION_FACTOR));
+ }
+
+ @Test
+ public void
testAppendNecessaryPropertiesRejectsConflictingReplicationProperties() {
+ TestableDorisTableOperations ops = new TestableDorisTableOperations();
+ Map<String, String> properties = new HashMap<>();
+ properties.put(REPLICATION_FACTOR, "1");
+ properties.put(REPLICATION_ALLOCATION, "tag.location.default: 1");
+
+ IllegalArgumentException exception =
+ Assertions.assertThrows(
+ IllegalArgumentException.class, () ->
ops.appendNecessaryProperties(properties));
+
+ Assertions.assertEquals(
+ "Properties 'replication_num' and 'replication_allocation' cannot be
set at the same time",
+ exception.getMessage());
+ }
+
+ private static DataSource mockBackendDataSource(int aliveBackendCount)
throws Exception {
+ DataSource dataSource = Mockito.mock(DataSource.class);
+ Connection connection = Mockito.mock(Connection.class);
+ Statement statement = Mockito.mock(Statement.class);
+ ResultSet resultSet = Mockito.mock(ResultSet.class);
+
+ Mockito.when(dataSource.getConnection()).thenReturn(connection);
+ Mockito.when(connection.createStatement()).thenReturn(statement);
+ Mockito.when(statement.executeQuery("show
backends")).thenReturn(resultSet);
+ int[] remainingAliveBackends = new int[] {aliveBackendCount};
+ Mockito.when(resultSet.next()).thenAnswer(invocation ->
remainingAliveBackends[0]-- > 0);
+ Mockito.when(resultSet.getString("Alive")).thenReturn("true");
+
+ return dataSource;
+ }
}
diff --git a/docs/jdbc-doris-catalog.md b/docs/jdbc-doris-catalog.md
index 43706917ac..f362d9c1fb 100644
--- a/docs/jdbc-doris-catalog.md
+++ b/docs/jdbc-doris-catalog.md
@@ -209,6 +209,7 @@ Index[] indexes = new Index[] {
- Doris supports table properties, and you can set them in the table
properties.
- Only supports Doris table properties and doesn't support user-defined
properties.
+- `replication_allocation` is supported for Doris 2.1 and later. Do not set it
together with `replication_num`.
### Table Indexes
