This is an automated email from the ASF dual-hosted git repository.
jshao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new a258a46b6 [#4718] fix(iceberg): use unified logic to transform
catalog backend name to handle the renaming of catalog (#4900)
a258a46b6 is described below
commit a258a46b651fb728bba2c69f2cf6b2ac7a0a743f
Author: FANNG <[email protected]>
AuthorDate: Mon Sep 23 10:54:11 2024 +0800
[#4718] fix(iceberg): use unified logic to transform catalog backend name
to handle the renaming of catalog (#4900)
### What changes were proposed in this pull request?
1. Spark,Trino, Iceberg catalog and Iceberg REST server use
`getCatalogBackendName` to get catalog backend name.
2. change the default backend name to catalog backend, like `jdbc`, it
will not change after rename.
### Why are the changes needed?
Fix: #4718
### Does this PR introduce _any_ user-facing change?
Yes
### How was this patch tested?
1. create a jdbc catalog with catalog backend name, check whether can
see the schema after rename
1. create a jdbc catalog without catalog backend name, check whether can
see the schema after rename
---
catalogs/catalog-common/build.gradle.kts | 5 ++
.../lakehouse/iceberg/IcebergPropertiesUtils.java | 20 ++++++++
.../iceberg/TestIcebergPropertiesUtils.java | 55 ++++++++++++++++++++++
.../iceberg/IcebergCatalogOperations.java | 3 --
docs/lakehouse-iceberg-catalog.md | 12 ++---
.../gravitino/iceberg/common/IcebergConfig.java | 5 +-
.../iceberg/common/utils/IcebergCatalogUtil.java | 10 ++--
.../connector/iceberg/GravitinoIcebergCatalog.java | 7 +--
.../iceberg/IcebergCatalogPropertyConverter.java | 22 +++++----
.../catalog/iceberg/IcebergConnectorAdapter.java | 1 -
.../trino/connector/metadata/GravitinoCatalog.java | 3 +-
11 files changed, 110 insertions(+), 33 deletions(-)
diff --git a/catalogs/catalog-common/build.gradle.kts
b/catalogs/catalog-common/build.gradle.kts
index ef3785f7c..c9a76ca90 100644
--- a/catalogs/catalog-common/build.gradle.kts
+++ b/catalogs/catalog-common/build.gradle.kts
@@ -25,4 +25,9 @@ plugins {
dependencies {
implementation(libs.slf4j.api)
implementation(libs.guava)
+
+ testImplementation(libs.junit.jupiter.api)
+ testImplementation(libs.junit.jupiter.params)
+
+ testRuntimeOnly(libs.junit.jupiter.engine)
}
diff --git
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergPropertiesUtils.java
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergPropertiesUtils.java
index 2a92f545e..abe08c57d 100644
---
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergPropertiesUtils.java
+++
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergPropertiesUtils.java
@@ -20,7 +20,9 @@ package org.apache.gravitino.catalog.lakehouse.iceberg;
import java.util.Collections;
import java.util.HashMap;
+import java.util.Locale;
import java.util.Map;
+import java.util.Optional;
import org.apache.gravitino.storage.OSSProperties;
import org.apache.gravitino.storage.S3Properties;
@@ -74,4 +76,22 @@ public class IcebergPropertiesUtils {
});
return icebergProperties;
}
+
+ /**
+ * Get catalog backend name from Gravitino catalog properties.
+ *
+ * @param catalogProperties a map of Gravitino catalog properties.
+ * @return catalog backend name.
+ */
+ public static String getCatalogBackendName(Map<String, String>
catalogProperties) {
+ String backendName =
catalogProperties.get(IcebergConstants.CATALOG_BACKEND_NAME);
+ if (backendName != null) {
+ return backendName;
+ }
+
+ String catalogBackend =
catalogProperties.get(IcebergConstants.CATALOG_BACKEND);
+ return Optional.ofNullable(catalogBackend)
+ .map(s -> s.toLowerCase(Locale.ROOT))
+ .orElse("memory");
+ }
}
diff --git
a/catalogs/catalog-common/src/test/java/org/apache/gravitino/lakehouse/iceberg/TestIcebergPropertiesUtils.java
b/catalogs/catalog-common/src/test/java/org/apache/gravitino/lakehouse/iceberg/TestIcebergPropertiesUtils.java
new file mode 100644
index 000000000..b8efe974d
--- /dev/null
+++
b/catalogs/catalog-common/src/test/java/org/apache/gravitino/lakehouse/iceberg/TestIcebergPropertiesUtils.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.lakehouse.iceberg;
+
+import com.google.common.collect.ImmutableMap;
+import java.util.Map;
+import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants;
+import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergPropertiesUtils;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class TestIcebergPropertiesUtils {
+
+ @Test
+ void testGetCatalogBackendName() {
+ Map<String, String> catalogProperties =
+ ImmutableMap.of(
+ IcebergConstants.CATALOG_BACKEND_NAME, "a",
IcebergConstants.CATALOG_BACKEND, "jdbc");
+ String backendName =
IcebergPropertiesUtils.getCatalogBackendName(catalogProperties);
+ Assertions.assertEquals("a", backendName);
+
+ catalogProperties = ImmutableMap.of(IcebergConstants.CATALOG_BACKEND,
"jdbc");
+ backendName =
IcebergPropertiesUtils.getCatalogBackendName(catalogProperties);
+ Assertions.assertEquals("jdbc", backendName);
+
+ catalogProperties = ImmutableMap.of(IcebergConstants.CATALOG_BACKEND,
"JDBC");
+ backendName =
IcebergPropertiesUtils.getCatalogBackendName(catalogProperties);
+ Assertions.assertEquals("jdbc", backendName);
+
+ catalogProperties = ImmutableMap.of(IcebergConstants.CATALOG_BACKEND,
"hive");
+ backendName =
IcebergPropertiesUtils.getCatalogBackendName(catalogProperties);
+ Assertions.assertEquals("hive", backendName);
+
+ catalogProperties = ImmutableMap.of();
+ backendName =
IcebergPropertiesUtils.getCatalogBackendName(catalogProperties);
+ Assertions.assertEquals("memory", backendName);
+ }
+}
diff --git
a/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java
b/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java
index c42ee0503..bf787df0b 100644
---
a/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java
+++
b/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java
@@ -108,9 +108,6 @@ public class IcebergCatalogOperations implements
CatalogOperations, SupportsSche
Map<String, String> resultConf = Maps.newHashMap(prefixMap);
resultConf.putAll(gravitinoConfig);
resultConf.put("catalog_uuid", info.id().toString());
- if
(!resultConf.containsKey(IcebergCatalogPropertiesMetadata.CATALOG_BACKEND_NAME))
{
- resultConf.put(IcebergCatalogPropertiesMetadata.CATALOG_BACKEND_NAME,
info.name());
- }
IcebergConfig icebergConfig = new IcebergConfig(resultConf);
this.icebergCatalogWrapper = new IcebergCatalogWrapper(icebergConfig);
diff --git a/docs/lakehouse-iceberg-catalog.md
b/docs/lakehouse-iceberg-catalog.md
index 8470da5b2..7552e0f19 100644
--- a/docs/lakehouse-iceberg-catalog.md
+++ b/docs/lakehouse-iceberg-catalog.md
@@ -36,12 +36,12 @@ Builds with Apache Iceberg `1.5.2`. The Apache Iceberg
table format version is `
### Catalog properties
-| Property name | Description
| Default value
| Required | Since Version
|
-|----------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------|-------------------------------------------------------------|---------------|
-| `catalog-backend` | Catalog backend of
Gravitino Iceberg catalog. Supports `hive` or `jdbc` or `rest`.
| (none)
| Yes | 0.2.0
|
-| `uri` | The URI configuration
of the Iceberg catalog. `thrift://127.0.0.1:9083` or
`jdbc:postgresql://127.0.0.1:5432/db_name` or
`jdbc:mysql://127.0.0.1:3306/metastore_db` or `http://127.0.0.1:9001`.
| (none) | Yes
| 0.2.0 |
-| `warehouse` | Warehouse directory of
catalog. `file:///user/hive/warehouse-hive/` for local fs or
`hdfs://namespace/hdfs/path` for HDFS.
|
(none) | Yes
| 0.2.0 |
-| `catalog-backend-name` | The catalog name passed
to underlying Iceberg catalog backend. Catalog name in JDBC backend is used to
isolate namespace and tables.
| Gravitino catalog
name | No | 0.5.2
|
+| Property name | Description
| Default value
| Required
| Since Version |
+|----------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------|-------------------------------------------------------------|---------------|
+| `catalog-backend` | Catalog backend of
Gravitino Iceberg catalog. Supports `hive` or `jdbc` or `rest`.
| (none)
| Yes
| 0.2.0 |
+| `uri` | The URI configuration
of the Iceberg catalog. `thrift://127.0.0.1:9083` or
`jdbc:postgresql://127.0.0.1:5432/db_name` or
`jdbc:mysql://127.0.0.1:3306/metastore_db` or `http://127.0.0.1:9001`.
| (none)
| Yes
| 0.2.0 |
+| `warehouse` | Warehouse directory of
catalog. `file:///user/hive/warehouse-hive/` for local fs or
`hdfs://namespace/hdfs/path` for HDFS.
|
(none)
| Yes | 0.2.0 |
+| `catalog-backend-name` | The catalog name passed
to underlying Iceberg catalog backend. Catalog name in JDBC backend is used to
isolate namespace and tables.
| The property value
of `catalog-backend`, like `jdbc` for JDBC catalog backend. | No
| 0.5.2 |
Any property not defined by Gravitino with `gravitino.bypass.` prefix will
pass to Iceberg catalog properties and HDFS configuration. For example, if
specify `gravitino.bypass.list-all-tables`, `list-all-tables` will pass to
Iceberg catalog properties.
diff --git
a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java
b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java
index 805be399a..64d327c78 100644
---
a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java
+++
b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java
@@ -21,7 +21,6 @@ package org.apache.gravitino.iceberg.common;
import com.google.common.collect.ImmutableMap;
import java.util.Map;
-import java.util.Optional;
import org.apache.commons.lang3.StringUtils;
import org.apache.gravitino.Config;
import org.apache.gravitino.OverwriteDefaultConfig;
@@ -226,8 +225,8 @@ public class IcebergConfig extends Config implements
OverwriteDefaultConfig {
return get(JDBC_DRIVER);
}
- public String getCatalogBackendName(String defaultCatalogBackendName) {
- return
Optional.ofNullable(get(CATALOG_BACKEND_NAME)).orElse(defaultCatalogBackendName);
+ public String getCatalogBackendName() {
+ return IcebergPropertiesUtils.getCatalogBackendName(getAllConfig());
}
public IcebergConfig(Map<String, String> properties) {
diff --git
a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/utils/IcebergCatalogUtil.java
b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/utils/IcebergCatalogUtil.java
index dfbc574be..4e28eb7c8 100644
---
a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/utils/IcebergCatalogUtil.java
+++
b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/utils/IcebergCatalogUtil.java
@@ -53,7 +53,7 @@ public class IcebergCatalogUtil {
private static final Logger LOG =
LoggerFactory.getLogger(IcebergCatalogUtil.class);
private static InMemoryCatalog loadMemoryCatalog(IcebergConfig
icebergConfig) {
- String icebergCatalogName = icebergConfig.getCatalogBackendName("memory");
+ String icebergCatalogName = icebergConfig.getCatalogBackendName();
InMemoryCatalog memoryCatalog = new InMemoryCatalog();
Map<String, String> resultProperties =
icebergConfig.getIcebergCatalogProperties();
resultProperties.put(CatalogProperties.WAREHOUSE_LOCATION, "/tmp");
@@ -64,7 +64,7 @@ public class IcebergCatalogUtil {
private static HiveCatalog loadHiveCatalog(IcebergConfig icebergConfig) {
ClosableHiveCatalog hiveCatalog = new ClosableHiveCatalog();
HdfsConfiguration hdfsConfiguration = new HdfsConfiguration();
- String icebergCatalogName = icebergConfig.getCatalogBackendName("hive");
+ String icebergCatalogName = icebergConfig.getCatalogBackendName();
Map<String, String> properties =
icebergConfig.getIcebergCatalogProperties();
properties.forEach(hdfsConfiguration::set);
@@ -114,7 +114,7 @@ public class IcebergCatalogUtil {
private static JdbcCatalog loadJdbcCatalog(IcebergConfig icebergConfig) {
String driverClassName = icebergConfig.getJdbcDriver();
- String icebergCatalogName = icebergConfig.getCatalogBackendName("jdbc");
+ String icebergCatalogName = icebergConfig.getCatalogBackendName();
Map<String, String> properties =
icebergConfig.getIcebergCatalogProperties();
Preconditions.checkNotNull(
@@ -140,7 +140,7 @@ public class IcebergCatalogUtil {
}
private static Catalog loadRestCatalog(IcebergConfig icebergConfig) {
- String icebergCatalogName = icebergConfig.getCatalogBackendName("rest");
+ String icebergCatalogName = icebergConfig.getCatalogBackendName();
RESTCatalog restCatalog = new RESTCatalog();
HdfsConfiguration hdfsConfiguration = new HdfsConfiguration();
Map<String, String> properties =
icebergConfig.getIcebergCatalogProperties();
@@ -151,7 +151,7 @@ public class IcebergCatalogUtil {
}
private static Catalog loadCustomCatalog(IcebergConfig icebergConfig) {
- String customCatalogName = icebergConfig.getCatalogBackendName("custom");
+ String customCatalogName = icebergConfig.getCatalogBackendName();
String className = icebergConfig.get(IcebergConfig.CATALOG_BACKEND_IMPL);
return CatalogUtil.loadCatalog(
className,
diff --git
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/iceberg/GravitinoIcebergCatalog.java
b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/iceberg/GravitinoIcebergCatalog.java
index 68aaef5ea..ccadc69e4 100644
---
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/iceberg/GravitinoIcebergCatalog.java
+++
b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/iceberg/GravitinoIcebergCatalog.java
@@ -22,7 +22,7 @@ package org.apache.gravitino.spark.connector.iceberg;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.Map;
-import java.util.Optional;
+import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergPropertiesUtils;
import org.apache.gravitino.rel.Table;
import org.apache.gravitino.spark.connector.PropertiesConverter;
import org.apache.gravitino.spark.connector.SparkTransformConverter;
@@ -58,10 +58,7 @@ public class GravitinoIcebergCatalog extends BaseCatalog
@Override
protected TableCatalog createAndInitSparkCatalog(
String name, CaseInsensitiveStringMap options, Map<String, String>
properties) {
- String catalogBackendName =
- Optional.ofNullable(
-
properties.get(IcebergPropertiesConstants.GRAVITINO_ICEBERG_CATALOG_BACKEND_NAME))
- .orElse(name);
+ String catalogBackendName =
IcebergPropertiesUtils.getCatalogBackendName(properties);
Map<String, String> all =
getPropertiesConverter().toSparkCatalogProperties(options, properties);
TableCatalog icebergCatalog = new SparkCatalog();
diff --git
a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergCatalogPropertyConverter.java
b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergCatalogPropertyConverter.java
index 692c3d5ce..c3afb001e 100644
---
a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergCatalogPropertyConverter.java
+++
b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergCatalogPropertyConverter.java
@@ -27,6 +27,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.commons.collections4.bidimap.TreeBidiMap;
import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants;
+import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergPropertiesUtils;
import org.apache.gravitino.catalog.property.PropertyConverter;
import org.apache.gravitino.trino.connector.GravitinoErrorCode;
@@ -345,17 +346,22 @@ public class IcebergCatalogPropertyConverter extends
PropertyConverter {
Map<String, String> jdbcProperties = new HashMap<>();
jdbcProperties.put("iceberg.catalog.type", "jdbc");
- jdbcProperties.put("iceberg.jdbc-catalog.driver-class",
properties.get("jdbc-driver"));
- jdbcProperties.put("iceberg.jdbc-catalog.connection-url",
properties.get("uri"));
- jdbcProperties.put("iceberg.jdbc-catalog.connection-user",
properties.get("jdbc-user"));
- jdbcProperties.put("iceberg.jdbc-catalog.connection-password",
properties.get("jdbc-password"));
- jdbcProperties.put("iceberg.jdbc-catalog.default-warehouse-dir",
properties.get("warehouse"));
+ jdbcProperties.put(
+ "iceberg.jdbc-catalog.driver-class",
+ properties.get(IcebergConstants.GRAVITINO_JDBC_DRIVER));
+ jdbcProperties.put("iceberg.jdbc-catalog.connection-url",
properties.get(IcebergConstants.URI));
+ jdbcProperties.put(
+ "iceberg.jdbc-catalog.connection-user",
+ properties.get(IcebergConstants.GRAVITINO_JDBC_USER));
+ jdbcProperties.put(
+ "iceberg.jdbc-catalog.connection-password",
+ properties.get(IcebergConstants.GRAVITINO_JDBC_PASSWORD));
+ jdbcProperties.put(
+ "iceberg.jdbc-catalog.default-warehouse-dir",
properties.get(IcebergConstants.WAREHOUSE));
- // TODO (yuhui) Optimize the code for retrieve the catalogname
- String catalogName = properties.get("catalog-name");
jdbcProperties.put(
"iceberg.jdbc-catalog.catalog-name",
- properties.getOrDefault(IcebergConstants.CATALOG_BACKEND_NAME,
catalogName));
+ IcebergPropertiesUtils.getCatalogBackendName(properties));
return jdbcProperties;
}
diff --git
a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergConnectorAdapter.java
b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergConnectorAdapter.java
index 1841b4d04..89b8f0198 100644
---
a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergConnectorAdapter.java
+++
b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergConnectorAdapter.java
@@ -46,7 +46,6 @@ public class IcebergConnectorAdapter implements
CatalogConnectorAdapter {
@Override
public Map<String, String> buildInternalConnectorConfig(GravitinoCatalog
catalog)
throws Exception {
- catalog.getProperties().put("catalog-name", catalog.getName());
return
catalogConverter.gravitinoToEngineProperties(catalog.getProperties());
}
diff --git
a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/metadata/GravitinoCatalog.java
b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/metadata/GravitinoCatalog.java
index 124d9aa2d..d59e59e9f 100644
---
a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/metadata/GravitinoCatalog.java
+++
b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/metadata/GravitinoCatalog.java
@@ -28,7 +28,6 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.json.JsonMapper;
import io.trino.spi.TrinoException;
import java.time.Instant;
-import java.util.HashMap;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.apache.gravitino.Catalog;
@@ -60,7 +59,7 @@ public class GravitinoCatalog {
this.metalake = metalake;
this.provider = catalog.provider();
this.name = catalog.name();
- this.properties = new HashMap<>(catalog.properties());
+ this.properties = catalog.properties();
Instant time =
catalog.auditInfo().lastModifiedTime() == null
? catalog.auditInfo().createTime()
