This is an automated email from the ASF dual-hosted git repository.
mchades pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 13f3b3f31 [#3173] fix(doris-catalog): Fix the bug when create doris
table with empty property (#4343)
13f3b3f31 is described below
commit 13f3b3f3118e2a7f8b63aa324e11c688159f4633
Author: Qi Yu <[email protected]>
AuthorDate: Tue Aug 6 19:41:31 2024 +0800
[#3173] fix(doris-catalog): Fix the bug when create doris table with empty
property (#4343)
### What changes were proposed in this pull request?
Automatically set the property `replication_num` to `1` if the number of
backend servers is less than 3, this will mostly happen in the test
environment or Gravitino CI docker image.
### Why are the changes needed?
Improve user experience when they create the Doris catalog with
Gravitino CI Doris image.
Fix: #3173
### Does this PR introduce _any_ user-facing change?
N/A
### How was this patch tested?
Existing UT and IT can cover this change.
---
.../gravitino/catalog/doris/DorisCatalog.java | 9 ++++
.../doris/DorisTablePropertiesMetadata.java | 54 ++++++++++++++++++++++
.../doris/operation/DorisTableOperations.java | 42 +++++++++++++++++
.../gravitino/catalog/doris/TestDorisCatalog.java | 45 ++++++++++++++++++
.../doris/integration/test/CatalogDorisIT.java | 4 +-
.../doris/operation/TestDorisTableOperations.java | 1 -
.../TestDorisTablePartitionOperations.java | 2 +-
docs/gravitino-server-config.md | 2 +-
docs/jdbc-doris-catalog.md | 19 ++++----
.../test/web/ui/CatalogsPageDorisTest.java | 1 -
10 files changed, 163 insertions(+), 16 deletions(-)
diff --git
a/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/DorisCatalog.java
b/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/DorisCatalog.java
index 3034d07f3..0a24908d5 100644
---
a/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/DorisCatalog.java
+++
b/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/DorisCatalog.java
@@ -32,11 +32,15 @@ import
org.apache.gravitino.catalog.jdbc.converter.JdbcTypeConverter;
import org.apache.gravitino.catalog.jdbc.operation.JdbcDatabaseOperations;
import org.apache.gravitino.catalog.jdbc.operation.JdbcTableOperations;
import org.apache.gravitino.connector.CatalogOperations;
+import org.apache.gravitino.connector.PropertiesMetadata;
import org.apache.gravitino.connector.capability.Capability;
/** Implementation of an Apache Doris catalog in Apache Gravitino. */
public class DorisCatalog extends JdbcCatalog {
+ public static final DorisTablePropertiesMetadata DORIS_TABLE_PROPERTIES_META
=
+ new DorisTablePropertiesMetadata();
+
@Override
public String shortName() {
return "jdbc-doris";
@@ -82,4 +86,9 @@ public class DorisCatalog extends JdbcCatalog {
protected JdbcColumnDefaultValueConverter
createJdbcColumnDefaultValueConverter() {
return new DorisColumnDefaultValueConverter();
}
+
+ @Override
+ public PropertiesMetadata tablePropertiesMetadata() throws
UnsupportedOperationException {
+ return DORIS_TABLE_PROPERTIES_META;
+ }
}
diff --git
a/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/DorisTablePropertiesMetadata.java
b/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/DorisTablePropertiesMetadata.java
new file mode 100644
index 000000000..4cad4c035
--- /dev/null
+++
b/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/DorisTablePropertiesMetadata.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.catalog.doris;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
+import java.util.List;
+import java.util.Map;
+import org.apache.gravitino.catalog.jdbc.JdbcTablePropertiesMetadata;
+import org.apache.gravitino.connector.PropertyEntry;
+
+public class DorisTablePropertiesMetadata extends JdbcTablePropertiesMetadata {
+
+ public static final String REPLICATION_FACTOR = "replication_num";
+ public static final int DEFAULT_REPLICATION_FACTOR = 1;
+ public static final int DEFAULT_REPLICATION_FACTOR_IN_SERVER_SIDE = 3;
+
+ private static final Map<String, PropertyEntry<?>> PROPERTIES_METADATA;
+
+ static {
+ List<PropertyEntry<?>> propertyEntries =
+ ImmutableList.of(
+ PropertyEntry.integerOptionalPropertyEntry(
+ REPLICATION_FACTOR,
+ "The number of replications for the table. If not specified
and the number of backend server less than 3,"
+ + " the default value will be used",
+ false /* immutable */,
+ DEFAULT_REPLICATION_FACTOR, /* default value */
+ false /* hidden */));
+
+ PROPERTIES_METADATA = Maps.uniqueIndex(propertyEntries,
PropertyEntry::getName);
+ }
+
+ @Override
+ protected Map<String, PropertyEntry<?>> specificPropertyEntries() {
+ return PROPERTIES_METADATA;
+ }
+}
diff --git
a/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTableOperations.java
b/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTableOperations.java
index 21795d803..27b2e9c68 100644
---
a/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTableOperations.java
+++
b/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTableOperations.java
@@ -18,6 +18,9 @@
*/
package org.apache.gravitino.catalog.doris.operation;
+import static
org.apache.gravitino.catalog.doris.DorisCatalog.DORIS_TABLE_PROPERTIES_META;
+import static
org.apache.gravitino.catalog.doris.DorisTablePropertiesMetadata.DEFAULT_REPLICATION_FACTOR_IN_SERVER_SIDE;
+import static
org.apache.gravitino.catalog.doris.DorisTablePropertiesMetadata.REPLICATION_FACTOR;
import static
org.apache.gravitino.catalog.doris.utils.DorisUtils.generatePartitionSqlFragment;
import static org.apache.gravitino.rel.Column.DEFAULT_VALUE_NOT_SET;
@@ -32,6 +35,7 @@ import java.sql.Statement;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -154,6 +158,7 @@ public class DorisTableOperations extends
JdbcTableOperations {
sqlBuilder.append(" BUCKETS ").append(distribution.number());
}
+ properties = appendNecessaryProperties(properties);
// Add table properties
sqlBuilder.append(NEW_LINE).append(DorisUtils.generatePropertiesSql(properties));
@@ -164,6 +169,43 @@ public class DorisTableOperations extends
JdbcTableOperations {
return result;
}
+ private Map<String, String> appendNecessaryProperties(Map<String, String>
properties) {
+ Map<String, String> resultMap;
+ if (properties == null) {
+ resultMap = new HashMap<>();
+ } else {
+ resultMap = new HashMap<>(properties);
+ }
+
+ // If the backend server is less than
DEFAULT_REPLICATION_FACTOR_IN_SERVER_SIDE (3), we need to
+ // set the property 'replication_num' to 1 explicitly.
+ if (!properties.containsKey(REPLICATION_FACTOR)) {
+ // Try to check the number of backend servers.
+ String query = "select count(*) from information_schema.backends where
Alive = 'true'";
+
+ try (Connection connection = dataSource.getConnection();
+ Statement statement = connection.createStatement();
+ ResultSet resultSet = statement.executeQuery(query)) {
+ while (resultSet.next()) {
+ int backendCount = resultSet.getInt(1);
+ if (backendCount < DEFAULT_REPLICATION_FACTOR_IN_SERVER_SIDE) {
+ resultMap.put(
+ REPLICATION_FACTOR,
+ DORIS_TABLE_PROPERTIES_META
+ .propertyEntries()
+ .get(REPLICATION_FACTOR)
+ .getDefaultValue()
+ .toString());
+ }
+ }
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to get the number of backend
servers", e);
+ }
+ }
+
+ return resultMap;
+ }
+
private static void validateIncrementCol(JdbcColumn[] columns) {
// Get all auto increment column
List<JdbcColumn> autoIncrementCols =
diff --git
a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/TestDorisCatalog.java
b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/TestDorisCatalog.java
new file mode 100644
index 000000000..039a15b0d
--- /dev/null
+++
b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/TestDorisCatalog.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.catalog.doris;
+
+import static
org.apache.gravitino.catalog.doris.DorisTablePropertiesMetadata.REPLICATION_FACTOR;
+
+import java.util.Map;
+import org.apache.gravitino.connector.PropertyEntry;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class TestDorisCatalog {
+
+ @Test
+ void testDorisTablePropertiesMetadata() {
+ DorisTablePropertiesMetadata dorisTablePropertiesMetadata = new
DorisTablePropertiesMetadata();
+ Map<String, PropertyEntry<?>> propertyEntryMap =
+ dorisTablePropertiesMetadata.specificPropertyEntries();
+ Assertions.assertTrue(propertyEntryMap.containsKey(REPLICATION_FACTOR));
+
+ PropertyEntry<?> propertyEntry = propertyEntryMap.get(REPLICATION_FACTOR);
+ Assertions.assertEquals(REPLICATION_FACTOR, propertyEntry.getName());
+ Assertions.assertFalse(propertyEntry.isImmutable());
+ Assertions.assertEquals(
+ DorisTablePropertiesMetadata.DEFAULT_REPLICATION_FACTOR,
propertyEntry.getDefaultValue());
+ Assertions.assertFalse(propertyEntry.isHidden());
+ }
+}
diff --git
a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java
b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java
index 7bc073a0b..fcfad401d 100644
---
a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java
+++
b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java
@@ -205,9 +205,7 @@ public class CatalogDorisIT extends AbstractIT {
}
private Map<String, String> createTableProperties() {
- Map<String, String> properties = Maps.newHashMap();
- properties.put("replication_allocation", "tag.location.default: 1");
- return properties;
+ return ImmutableMap.of();
}
private Distribution createDistribution() {
diff --git
a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTableOperations.java
b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTableOperations.java
index a252a610b..07e6209d9 100644
---
a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTableOperations.java
+++
b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTableOperations.java
@@ -91,7 +91,6 @@ public class TestDorisTableOperations extends TestDoris {
private static Map<String, String> createProperties() {
Map<String, String> properties = Maps.newHashMap();
- properties.put("replication_allocation", "tag.location.default: 1");
return properties;
}
diff --git
a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTablePartitionOperations.java
b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTablePartitionOperations.java
index 0494082bd..40cb3254b 100644
---
a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTablePartitionOperations.java
+++
b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTablePartitionOperations.java
@@ -74,7 +74,7 @@ public class TestDorisTablePartitionOperations extends
TestDoris {
}
private static Map<String, String> createProperties() {
- return ImmutableMap.of("replication_allocation", "tag.location.default:
1");
+ return ImmutableMap.of();
}
@Test
diff --git a/docs/gravitino-server-config.md b/docs/gravitino-server-config.md
index 88e473ded..4f3fba6c7 100644
--- a/docs/gravitino-server-config.md
+++ b/docs/gravitino-server-config.md
@@ -58,7 +58,7 @@ The following table lists the storage configuration items:
| `gravitino.entity.store.kv.deleteAfterTimeMs` | It is deprecated since
Gravitino 0.5.0. Please use `gravitino.entity.store.deleteAfterTimeMs` instead.
| `604800000`(7
days) | No | 0.3.0
|
| `gravitino.entity.store.deleteAfterTimeMs` | The maximum time in
milliseconds that deleted and old-version data is kept. Set to at least 10
minutes and no longer than 30 days.
|
`604800000`(7 days) | No
| 0.5.0 |
| `gravitino.entity.store.versionRetentionCount` | The Count of versions
allowed to be retained, including the current version, used to delete old
versions data. Set to at least 1 and no greater than 10.
| `1`
| No |
0.5.0 |
-| `gravitino.entity.store.relational` | Detailed implementation
of Relational storage. `MySQL` is currently supported, and the implementation
is `JDBCBackend`.
| `JDBCBackend`
| No | 0.5.0
|
+| `gravitino.entity.store.relational` | Detailed implementation
of Relational storage. `H2` and `MySQL` is currently supported, and the
implementation is `JDBCBackend`.
|
`JDBCBackend` | No
| 0.5.0 |
| `gravitino.entity.store.relational.jdbcUrl` | The database url that
the `JDBCBackend` needs to connect to. If you use `MySQL`, you should firstly
initialize the database tables yourself by executing the ddl scripts in the
`${GRAVITINO_HOME}/scripts/mysql/` directory. | `jdbc:h2`
| No |
0.5.0 |
| `gravitino.entity.store.relational.jdbcDriver` | The jdbc driver name
that the `JDBCBackend` needs to use. You should place the driver Jar package in
the `${GRAVITINO_HOME}/libs/` directory.
|
`org.h2.Driver` | Yes if the jdbc connection url is not
`jdbc:h2` | 0.5.0 |
| `gravitino.entity.store.relational.jdbcUser` | The username that the
`JDBCBackend` needs to use when connecting the database. It is required for
`MySQL`.
| `gravitino`
| Yes if the jdbc connection url is not `jdbc:h2` | 0.5.0
|
diff --git a/docs/jdbc-doris-catalog.md b/docs/jdbc-doris-catalog.md
index 9ea70362a..1548455af 100644
--- a/docs/jdbc-doris-catalog.md
+++ b/docs/jdbc-doris-catalog.md
@@ -41,15 +41,16 @@ more details.
Besides the [common catalog
properties](./gravitino-server-config.md#gravitino-catalog-properties-configuration),
the Doris catalog has the following properties:
-| Configuration item | Description
| Default value | Required | Since Version |
-|----------------------|-------------------------------------------------------------------------------------|---------------|----------|---------------|
-| `jdbc-url` | JDBC URL for connecting to the database. For example,
`jdbc:mysql://localhost:9030` | (none) | Yes | 0.5.0 |
-| `jdbc-driver` | The driver of the JDBC connection. For example,
`com.mysql.jdbc.Driver`. | (none) | Yes | 0.5.0 |
-| `jdbc-user` | The JDBC user name.
| (none) | Yes | 0.5.0 |
-| `jdbc-password` | The JDBC password.
| (none) | Yes | 0.5.0 |
-| `jdbc.pool.min-size` | The minimum number of connections in the pool. `2` by
default. | `2` | No | 0.5.0 |
-| `jdbc.pool.max-size` | The maximum number of connections in the pool. `10`
by default. | `10` | No | 0.5.0 |
-
+| Configuration item | Description
| Default value | Required | Since Version |
+|----------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|----------|---------------|
+| `jdbc-url` | JDBC URL for connecting to the database. For example,
`jdbc:mysql://localhost:9030`
| (none) | Yes | 0.5.0 |
+| `jdbc-driver` | The driver of the JDBC connection. For example,
`com.mysql.jdbc.Driver`.
| (none) | Yes | 0.5.0 |
+| `jdbc-user` | The JDBC user name.
| (none) | Yes | 0.5.0 |
+| `jdbc-password` | The JDBC password.
| (none) | Yes | 0.5.0 |
+| `jdbc.pool.min-size` | The minimum number of connections in the pool. `2` by
default.
| `2` | No | 0.5.0 |
+| `jdbc.pool.max-size` | The maximum number of connections in the pool. `10`
by default.
| `10` | No | 0.5.0 |
+| `jdbc.pool.max-size` | The maximum number of connections in the pool. `10`
by default.
| `10` | No | 0.5.0 |
+| `replication_num` | The number of replications for the table. If not
specified and the number of backend servers less than 3, then the default value
is 1; If not specified and the number of backend servers greater or equals to
3, the default value (3) in Doris server will be used. For more, please see the
[doc](https://doris.apache.org/docs/1.2/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE/)
| `1` or `3` | No | 0.6.0 |
Before using the Doris Catalog, you must download the corresponding JDBC
driver to the `catalogs/jdbc-doris/libs` directory.
Gravitino doesn't package the JDBC driver for Doris due to licensing issues.
diff --git
a/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/CatalogsPageDorisTest.java
b/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/CatalogsPageDorisTest.java
index 72a4afbf5..1fcd754b9 100644
---
a/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/CatalogsPageDorisTest.java
+++
b/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/CatalogsPageDorisTest.java
@@ -121,7 +121,6 @@ public class CatalogsPageDorisTest extends AbstractWebIT {
String tableName,
String colName) {
Map<String, String> properties = Maps.newHashMap();
- properties.put("replication_allocation", "tag.location.default: 1");
Column column = Column.of(colName, Types.IntegerType.get(), "column
comment");
Catalog catalog_doris = metalake.loadCatalog(catalogName);
catalog_doris
