This is an automated email from the ASF dual-hosted git repository.
fanng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 837b53b1c [#4508]feat(iceberg-rest-server): support load custom
catalog as backend in the Iceberg REST service (#4563)
837b53b1c is described below
commit 837b53b1cf4d273756ae98e1ca4e8a9a2e1124d3
Author: theoryxu <[email protected]>
AuthorDate: Mon Aug 19 21:03:40 2024 +0800
[#4508]feat(iceberg-rest-server): support load custom catalog as backend in
the Iceberg REST service (#4563)
### What changes were proposed in this pull request?
support load custom catalog as backend
### Why are the changes needed?
improvement: #4508
### Does this PR introduce _any_ user-facing change?
1. add a property key
### How was this patch tested?
1. add UT
2. manual test
---------
Co-authored-by: theoryxu <[email protected]>
---
.../lakehouse/iceberg/IcebergConstants.java | 1 +
docs/iceberg-rest-service.md | 8 +++++
.../iceberg/common/IcebergCatalogBackend.java | 3 +-
.../gravitino/iceberg/common/IcebergConfig.java | 8 +++++
.../iceberg/common/utils/IcebergCatalogUtil.java | 13 +++++++++
.../common/utils/CustomCatalogForTest.java} | 32 ++++++++++++++++----
.../common/utils/TestIcebergCatalogUtil.java | 34 ++++++++++++++++++++++
7 files changed, 92 insertions(+), 7 deletions(-)
diff --git
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java
index 6e4aae37a..d07a6c487 100644
---
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java
+++
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java
@@ -21,6 +21,7 @@ package org.apache.gravitino.catalog.lakehouse.iceberg;
public class IcebergConstants {
// Iceberg catalog properties constants
public static final String CATALOG_BACKEND = "catalog-backend";
+ public static final String CATALOG_BACKEND_IMPL = "catalog-backend-impl";
public static final String GRAVITINO_JDBC_USER = "jdbc-user";
public static final String ICEBERG_JDBC_USER = "jdbc.user";
diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md
index c1a4a8111..1753cc949 100644
--- a/docs/iceberg-rest-service.md
+++ b/docs/iceberg-rest-service.md
@@ -162,6 +162,14 @@ If you have a JDBC Iceberg catalog prior, you must set
`catalog-backend-name` to
You must download the corresponding JDBC driver to the
`iceberg-rest-server/libs` directory.
:::
+#### Custom backend configuration
+| Configuration item | Description
| Default value | Required | Since Version |
+|------------------------------------------------|---------------------------------------------------------------------------------------------------------------------|------------------|----------|---------------|
+| `gravitino.iceberg-rest.catalog-backend` | The Catalog backend of the
Gravitino Iceberg REST catalog service. Use the value **`custom`** for a Custom
catalog. | `memory` | Yes | 0.2.0 |
+| `gravitino.iceberg-rest.catalog-backend-impl` | The fully-qualified class
name of a custom catalog implementation, only worked if `catalog-backend` is
`custom`. | (none) | No | 0.7.0 |
+
+If you want to use a custom Iceberg Catalog as `catalog-backend`, you can add
a corresponding jar file to the classpath and load a custom Iceberg Catalog
implementation by specifying the `catalog-backend-impl` property.
+
#### Multi catalog support
The Gravitino Iceberg REST server supports multiple catalogs and offers a
configuration-based catalog management system.
diff --git
a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergCatalogBackend.java
b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergCatalogBackend.java
index 63fb07605..4cdedc826 100644
---
a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergCatalogBackend.java
+++
b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergCatalogBackend.java
@@ -22,5 +22,6 @@ public enum IcebergCatalogBackend {
HIVE,
JDBC,
MEMORY,
- REST
+ REST,
+ CUSTOM
}
diff --git
a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java
b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java
index b75fc88d6..bc1e47a06 100644
---
a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java
+++
b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java
@@ -43,6 +43,14 @@ public class IcebergConfig extends Config implements
OverwriteDefaultConfig {
.stringConf()
.createWithDefault("memory");
+ public static final ConfigEntry<String> CATALOG_BACKEND_IMPL =
+ new ConfigBuilder(IcebergConstants.CATALOG_BACKEND_IMPL)
+ .doc(
+ "The fully-qualified class name of a custom catalog
implementation, only worked if `catalog-backend` is `custom`")
+ .version(ConfigConstants.VERSION_0_7_0)
+ .stringConf()
+ .create();
+
public static final ConfigEntry<String> CATALOG_WAREHOUSE =
new ConfigBuilder(IcebergConstants.WAREHOUSE)
.doc("Warehouse directory of catalog")
diff --git
a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/utils/IcebergCatalogUtil.java
b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/utils/IcebergCatalogUtil.java
index f6e8b4e36..dfbc574be 100644
---
a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/utils/IcebergCatalogUtil.java
+++
b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/utils/IcebergCatalogUtil.java
@@ -39,6 +39,7 @@ import
org.apache.gravitino.iceberg.common.authentication.kerberos.KerberosClien
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
import org.apache.iceberg.catalog.Catalog;
import org.apache.iceberg.hive.HiveCatalog;
import org.apache.iceberg.inmemory.InMemoryCatalog;
@@ -149,6 +150,16 @@ public class IcebergCatalogUtil {
return restCatalog;
}
+ private static Catalog loadCustomCatalog(IcebergConfig icebergConfig) {
+ String customCatalogName = icebergConfig.getCatalogBackendName("custom");
+ String className = icebergConfig.get(IcebergConfig.CATALOG_BACKEND_IMPL);
+ return CatalogUtil.loadCatalog(
+ className,
+ customCatalogName,
+ icebergConfig.getIcebergCatalogProperties(),
+ new HdfsConfiguration());
+ }
+
@VisibleForTesting
static Catalog loadCatalogBackend(String catalogType) {
return loadCatalogBackend(
@@ -168,6 +179,8 @@ public class IcebergCatalogUtil {
return loadJdbcCatalog(icebergConfig);
case REST:
return loadRestCatalog(icebergConfig);
+ case CUSTOM:
+ return loadCustomCatalog(icebergConfig);
default:
throw new RuntimeException(
catalogBackend
diff --git
a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergCatalogBackend.java
b/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/utils/CustomCatalogForTest.java
similarity index 54%
copy from
iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergCatalogBackend.java
copy to
iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/utils/CustomCatalogForTest.java
index 63fb07605..6c6492b7d 100644
---
a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergCatalogBackend.java
+++
b/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/utils/CustomCatalogForTest.java
@@ -16,11 +16,31 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.gravitino.iceberg.common;
-public enum IcebergCatalogBackend {
- HIVE,
- JDBC,
- MEMORY,
- REST
+package org.apache.gravitino.iceberg.common.utils;
+
+import java.util.List;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+
+public class CustomCatalogForTest implements Catalog {
+ @Override
+ public List<TableIdentifier> listTables(Namespace namespace) {
+ return null;
+ }
+
+ @Override
+ public boolean dropTable(TableIdentifier identifier, boolean purge) {
+ return false;
+ }
+
+ @Override
+ public void renameTable(TableIdentifier from, TableIdentifier to) {}
+
+ @Override
+ public Table loadTable(TableIdentifier identifier) {
+ return null;
+ }
}
diff --git
a/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/utils/TestIcebergCatalogUtil.java
b/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/utils/TestIcebergCatalogUtil.java
index 289858ae3..29ec0cc15 100644
---
a/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/utils/TestIcebergCatalogUtil.java
+++
b/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/utils/TestIcebergCatalogUtil.java
@@ -78,4 +78,38 @@ public class TestIcebergCatalogUtil {
IcebergCatalogUtil.loadCatalogBackend("other");
});
}
+
+ @Test
+ void testValidLoadCustomCatalog() {
+ Catalog catalog;
+ Map<String, String> config = new HashMap<>();
+
+ config.put(
+ "catalog-backend-impl",
"org.apache.gravitino.iceberg.common.utils.CustomCatalogForTest");
+ catalog =
+ IcebergCatalogUtil.loadCatalogBackend(
+ IcebergCatalogBackend.valueOf("CUSTOM"), new
IcebergConfig(config));
+ Assertions.assertTrue(catalog instanceof CustomCatalogForTest);
+ }
+
+ @Test
+ void testInvalidLoadCustomCatalog() {
+ Assertions.assertThrowsExactly(
+ NullPointerException.class,
+ () ->
+ IcebergCatalogUtil.loadCatalogBackend(
+ IcebergCatalogBackend.valueOf("CUSTOM"), new IcebergConfig(new
HashMap<>())));
+
+ Assertions.assertThrowsExactly(
+ IllegalArgumentException.class,
+ () ->
+ IcebergCatalogUtil.loadCatalogBackend(
+ IcebergCatalogBackend.valueOf("CUSTOM"),
+ new IcebergConfig(
+ new HashMap<String, String>() {
+ {
+ put("catalog-backend-impl", "org.apache.");
+ }
+ })));
+ }
}
