This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 7ebe6e5b80 [#11133] feat(iceberg): Enable table metadata cache by
default and increase default capacity (#11156)
7ebe6e5b80 is described below
commit 7ebe6e5b8059792cef679656fed6c30681dbbb1e
Author: MaSai <[email protected]>
AuthorDate: Mon May 25 11:49:49 2026 +0800
[#11133] feat(iceberg): Enable table metadata cache by default and increase
default capacity (#11156)
### What changes were proposed in this pull request?
1. Default `table-metadata-cache-impl` to
`org.apache.gravitino.iceberg.common.cache.LocalTableMetadataCache` in
`IcebergConfig`.
2. Raise default `table-metadata-cache-capacity` from **200** to
**1000** in `IcebergConfig`.
3. Update IRC and lakehouse-iceberg catalog docs to match the new
defaults.
4. Align `IcebergCatalogPropertiesMetadata` defaults with
`IcebergConfig`.
5. Add `TestIcebergConfig.testTableMetadataCacheDefaults` to assert the
new defaults.
`IcebergCatalogWrapper` already loads the configured cache
implementation when `table-metadata-cache-impl` is set; with the new
config default, deployments no longer silently fall back to
`TableMetadataCache.DUMMY` unless the property is explicitly cleared.
### Why are the changes needed?
Fixes #11133
`TABLE_METADATA_CACHE_IMPL` has no default today, so every deployment
that does not set the property uses `TableMetadataCache.DUMMY` in
`IcebergCatalogWrapper.loadTableMetadataCache` and reads `metadata.json`
from object storage on every `loadTable`. The default capacity of
**200** is also too small for typical enterprise working sets.
### Does this PR introduce _any_ user-facing change?
Yes.
1. `table-metadata-cache-impl` defaults to `LocalTableMetadataCache`
when not configured.
2. `table-metadata-cache-capacity` defaults to **1000** when not
configured.
### How was this patch tested?
- `./gradlew :iceberg:iceberg-common:spotlessApply
:catalogs:catalog-lakehouse-iceberg:spotlessApply`
- `./gradlew :iceberg:iceberg-common:test -PskipITs
-PskipDockerTests=true --tests
"org.apache.gravitino.iceberg.common.TestIcebergConfig"`
---
.../iceberg/IcebergCatalogPropertiesMetadata.java | 9 +-
.../TestIcebergCatalogPropertiesMetadata.java | 75 +++++++
docs/iceberg-rest-service.md | 10 +-
docs/lakehouse-iceberg-catalog.md | 10 +-
.../gravitino/iceberg/common/IcebergConfig.java | 9 +-
.../iceberg/common/ops/IcebergCatalogWrapper.java | 14 +-
.../iceberg/common/TestIcebergConfig.java | 16 ++
.../iceberg/integration/test/IcebergCacheIT.java | 230 +++++++++++++++++++++
8 files changed, 352 insertions(+), 21 deletions(-)
diff --git
a/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java
b/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java
index 7f4735dbf3..49680a512d 100644
---
a/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java
+++
b/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java
@@ -33,6 +33,7 @@ import
org.apache.gravitino.connector.BaseCatalogPropertiesMetadata;
import org.apache.gravitino.connector.PropertyEntry;
import org.apache.gravitino.iceberg.common.authentication.AuthenticationConfig;
import
org.apache.gravitino.iceberg.common.authentication.kerberos.KerberosConfig;
+import org.apache.gravitino.iceberg.common.cache.LocalTableMetadataCache;
import org.apache.gravitino.storage.AzureProperties;
import org.apache.gravitino.storage.OSSProperties;
import org.apache.gravitino.storage.S3Properties;
@@ -125,15 +126,17 @@ public class IcebergCatalogPropertiesMetadata extends
BaseCatalogPropertiesMetad
false /* hidden */),
stringOptionalPropertyEntry(
IcebergConstants.TABLE_METADATA_CACHE_IMPL,
- "Table metadata cache implementation",
+ "Table metadata cache implementation. Set to empty
string(\"\") if "
+ + "catalog-backend is rest catalog, or custom catalog
without the "
+ + "SupportsMetadataLocation interface.",
false /* immutable */,
- null /* defaultValue */,
+ LocalTableMetadataCache.class.getName() /* defaultValue */,
false /* hidden */),
integerOptionalPropertyEntry(
IcebergConstants.TABLE_METADATA_CACHE_CAPACITY,
"Table metadata cache capacity",
false /* immutable */,
- 200 /* defaultValue */,
+ 1000 /* defaultValue */,
false /* hidden */),
integerOptionalPropertyEntry(
IcebergConstants.TABLE_METADATA_CACHE_EXPIRE_MINUTES,
diff --git
a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/TestIcebergCatalogPropertiesMetadata.java
b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/TestIcebergCatalogPropertiesMetadata.java
new file mode 100644
index 0000000000..125323f283
--- /dev/null
+++
b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/TestIcebergCatalogPropertiesMetadata.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.catalog.lakehouse.iceberg;
+
+import com.google.common.collect.ImmutableMap;
+import java.util.Map;
+import org.apache.gravitino.iceberg.common.IcebergConfig;
+import org.apache.gravitino.iceberg.common.cache.LocalTableMetadataCache;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+public class TestIcebergCatalogPropertiesMetadata {
+
+ private IcebergCatalogPropertiesMetadata metadata;
+
+ @BeforeEach
+ void setUp() {
+ metadata = new IcebergCatalogPropertiesMetadata();
+ }
+
+ @Test
+ void testTableMetadataCacheImplDefaultValue() {
+ Assertions.assertEquals(
+ LocalTableMetadataCache.class.getName(),
+ metadata.getDefaultValue(IcebergConstants.TABLE_METADATA_CACHE_IMPL));
+ Assertions.assertEquals(
+ IcebergConfig.TABLE_METADATA_CACHE_IMPL.getDefaultValue(),
+ metadata.getDefaultValue(IcebergConstants.TABLE_METADATA_CACHE_IMPL));
+ }
+
+ @Test
+ void testTableMetadataCacheCapacityDefaultValue() {
+ Assertions.assertEquals(
+ 1000,
metadata.getDefaultValue(IcebergConstants.TABLE_METADATA_CACHE_CAPACITY));
+ Assertions.assertEquals(
+ IcebergConfig.TABLE_METADATA_CACHE_CAPACITY.getDefaultValue(),
+
metadata.getDefaultValue(IcebergConstants.TABLE_METADATA_CACHE_CAPACITY));
+ }
+
+ @Test
+ void testTableMetadataCacheDefaultsViaGetOrDefault() {
+ Map<String, String> catalogProperties =
+ ImmutableMap.of(
+ IcebergCatalogPropertiesMetadata.CATALOG_BACKEND,
+ "hive",
+ IcebergCatalogPropertiesMetadata.URI,
+ "thrift://127.0.0.1:9083",
+ IcebergCatalogPropertiesMetadata.WAREHOUSE,
+ "/tmp/warehouse");
+
+ Assertions.assertEquals(
+ IcebergConfig.TABLE_METADATA_CACHE_IMPL.getDefaultValue(),
+ metadata.getOrDefault(catalogProperties,
IcebergConstants.TABLE_METADATA_CACHE_IMPL));
+ Assertions.assertEquals(
+ IcebergConfig.TABLE_METADATA_CACHE_CAPACITY.getDefaultValue(),
+ metadata.getOrDefault(catalogProperties,
IcebergConstants.TABLE_METADATA_CACHE_CAPACITY));
+ }
+}
diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md
index c89b927e71..9844bc7c69 100644
--- a/docs/iceberg-rest-service.md
+++ b/docs/iceberg-rest-service.md
@@ -563,11 +563,11 @@ You must download the corresponding JDBC driver to the
`iceberg-rest-server/libs
Gravitino features a pluggable cache system for updating or retrieving table
metadata in the cache. It validates the location of table metadata against the
catalog backend to ensure the correctness of cached data.
-| Configuration item | Description
| Default value | Required | Since Version |
-|--------------------------------------------------------------|---------------------------------------------|---------------|----------|---------------|
-| `gravitino.iceberg-rest.table-metadata-cache-impl` | The implement
of the cache. | (none) | No | 1.1.0 |
-| `gravitino.iceberg-rest.table-metadata-cache-capacity` | The capacity
of table metadata cache. | 200 | No | 1.1.0 |
-| `gravitino.iceberg-rest.table-metadata-cache-expire-minutes` | The expire
minutes of table metadata cache. | 60 | No | 1.1.0 |
+| Configuration item | Description
| Default value |
Required | Since Version |
+|--------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------|----------|---------------|
+| `gravitino.iceberg-rest.table-metadata-cache-impl` | The
implementation of the table metadata cache. Set to empty string("") if
`catalog-backend` is `rest` catalog, or `custom` catalog without the
`SupportsMetadataLocation` interface. |
`org.apache.gravitino.iceberg.common.cache.LocalTableMetadataCache` | No
| 1.1.0 |
+| `gravitino.iceberg-rest.table-metadata-cache-capacity` | The capacity
of the table metadata cache.
| 1000
| No | 1.1.0 |
+| `gravitino.iceberg-rest.table-metadata-cache-expire-minutes` | The
expiration time (in minutes) of the table metadata cache.
| 60
| No | 1.1.0 |
Gravitino provides the build-in
`org.apache.gravitino.iceberg.common.cache.LocalTableMetadataCache` to store
the cached data in the memory. You could also implement your custom table
metadata cache by implementing the
`org.apache.gravitino.iceberg.common.cache.TableMetadataCache` interface.
diff --git a/docs/lakehouse-iceberg-catalog.md
b/docs/lakehouse-iceberg-catalog.md
index 252fd7d34f..28c500f29d 100644
--- a/docs/lakehouse-iceberg-catalog.md
+++ b/docs/lakehouse-iceberg-catalog.md
@@ -218,11 +218,11 @@ Users can use the following properties to configure the
security of the catalog
Gravitino features a pluggable cache system for updating or retrieving table
metadata in the cache. It validates the location of table metadata against the
catalog backend to ensure the correctness of cached data.
-| Configuration item | Description
| Default value | Required | Since Version |
-|---------------------------------------|---------------------------------------------|---------------|----------|---------------|
-| `table-metadata-cache-impl` | The implement of the cache.
| (none) | No | 1.1.0 |
-| `table-metadata-cache-capacity` | The capacity of table metadata
cache. | 200 | No | 1.1.0 |
-| `table-metadata-cache-expire-minutes` | The expire minutes of table metadata
cache. | 60 | No | 1.1.0 |
+| Configuration item | Description
| Default
value | Required | Since
Version |
+|---------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------|----------|---------------|
+| `table-metadata-cache-impl` | The implementation of the table
metadata cache. Set to empty string("") if `catalog-backend` is `rest` catalog,
or `custom` catalog without the `SupportsMetadataLocation` interface. |
`org.apache.gravitino.iceberg.common.cache.LocalTableMetadataCache` | No
| 1.1.0 |
+| `table-metadata-cache-capacity` | The capacity of the table metadata
cache.
| 1000
| No | 1.1.0
|
+| `table-metadata-cache-expire-minutes` | The expiration time (in minutes) of
the table metadata cache.
| 60
| No | 1.1.0
|
Gravitino provides the build-in
`org.apache.gravitino.iceberg.common.cache.LocalTableMetadataCache` to store
the cached data in the memory. You could also implement your custom table
metadata cache by implementing the
`org.apache.gravitino.iceberg.common.cache.TableMetadataCache` interface.
diff --git
a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java
b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java
index 4da89e37ef..f5ee8dd512 100644
---
a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java
+++
b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java
@@ -33,6 +33,7 @@ import org.apache.gravitino.config.ConfigBuilder;
import org.apache.gravitino.config.ConfigConstants;
import org.apache.gravitino.config.ConfigEntry;
import org.apache.gravitino.credential.CredentialConstants;
+import org.apache.gravitino.iceberg.common.cache.LocalTableMetadataCache;
import org.apache.gravitino.storage.OSSProperties;
import org.apache.gravitino.storage.S3Properties;
@@ -273,17 +274,19 @@ public class IcebergConfig extends Config implements
OverwriteDefaultConfig {
public static final ConfigEntry<String> TABLE_METADATA_CACHE_IMPL =
new ConfigBuilder(IcebergConstants.TABLE_METADATA_CACHE_IMPL)
- .doc("Table metadata cache implementation")
+ .doc(
+ "Table metadata cache implementation. Set to empty string(\"\")
if catalog-backend "
+ + "is rest catalog, or custom catalog without the
SupportsMetadataLocation interface.")
.version(ConfigConstants.VERSION_1_1_0)
.stringConf()
- .create();
+ .createWithDefault(LocalTableMetadataCache.class.getName());
public static final ConfigEntry<Integer> TABLE_METADATA_CACHE_CAPACITY =
new ConfigBuilder(IcebergConstants.TABLE_METADATA_CACHE_CAPACITY)
.doc("Table metadata cache capacity")
.version(ConfigConstants.VERSION_1_1_0)
.intConf()
- .createWithDefault(200);
+ .createWithDefault(1000);
public static final ConfigEntry<Integer> TABLE_METADATA_CACHE_EXPIRE_MINUTES
=
new ConfigBuilder(IcebergConstants.TABLE_METADATA_CACHE_EXPIRE_MINUTES)
diff --git
a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java
b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java
index 97919418a6..0ab9f215f4 100644
---
a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java
+++
b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java
@@ -457,11 +457,15 @@ public class IcebergCatalogWrapper implements
AutoCloseable {
return TableMetadataCache.DUMMY;
}
- Preconditions.checkArgument(
- catalog instanceof SupportsMetadataLocation,
- "You shouldn't enable Iceberg metadata cache for the catalog %s,"
- + " because the catalog impl does not support get metadata
location.",
- catalog.name());
+ if (!(catalog instanceof SupportsMetadataLocation)) {
+ LOG.warn(
+ "Catalog '{}' does not support the table metadata cache, because the
catalog impl does not "
+ + "support get metadata location. The cache is disabled. Set
'{}' to an empty string "
+ + "(\"\") in the configuration before the next restart.",
+ catalog.name(),
+ IcebergConfig.TABLE_METADATA_CACHE_IMPL.getKey());
+ return TableMetadataCache.DUMMY;
+ }
TableMetadataCache cache =
ClassUtils.loadAndGetInstance(impl,
Thread.currentThread().getContextClassLoader());
diff --git
a/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/TestIcebergConfig.java
b/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/TestIcebergConfig.java
index 86da78ed60..1bd47fcb16 100644
---
a/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/TestIcebergConfig.java
+++
b/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/TestIcebergConfig.java
@@ -21,6 +21,7 @@ package org.apache.gravitino.iceberg.common;
import com.google.common.collect.ImmutableMap;
import java.util.Map;
+import org.apache.gravitino.iceberg.common.cache.LocalTableMetadataCache;
import org.apache.gravitino.server.web.JettyServerConfig;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -63,6 +64,21 @@ public class TestIcebergConfig {
Assertions.assertEquals(1001, jettyServerConfig.getHttpsPort());
}
+ @Test
+ public void testTableMetadataCacheDefaults() {
+ IcebergConfig icebergConfig = new IcebergConfig(ImmutableMap.of());
+ Assertions.assertEquals(
+ LocalTableMetadataCache.class.getName(),
+ icebergConfig.get(IcebergConfig.TABLE_METADATA_CACHE_IMPL));
+ Assertions.assertEquals(1000,
icebergConfig.get(IcebergConfig.TABLE_METADATA_CACHE_CAPACITY));
+ Assertions.assertEquals(
+ IcebergConfig.TABLE_METADATA_CACHE_IMPL.getDefaultValue(),
+ icebergConfig.get(IcebergConfig.TABLE_METADATA_CACHE_IMPL));
+ Assertions.assertEquals(
+ IcebergConfig.TABLE_METADATA_CACHE_CAPACITY.getDefaultValue(),
+ icebergConfig.get(IcebergConfig.TABLE_METADATA_CACHE_CAPACITY));
+ }
+
@Test
public void testDisableRestAuthzConfigKey() {
Map<String, String> propertiesWithNewKey =
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergCacheIT.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergCacheIT.java
new file mode 100644
index 0000000000..cbd36562ed
--- /dev/null
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergCacheIT.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.iceberg.integration.test;
+
+import java.lang.reflect.Method;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergCatalogBackend;
+import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants;
+import org.apache.gravitino.iceberg.common.IcebergConfig;
+import org.apache.gravitino.iceberg.common.cache.LocalTableMetadataCache;
+import org.apache.gravitino.iceberg.common.cache.SupportsMetadataLocation;
+import org.apache.gravitino.iceberg.common.cache.TableMetadataCache;
+import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper;
+import
org.apache.gravitino.iceberg.integration.test.util.IcebergRESTServerManager;
+import org.apache.gravitino.integration.test.container.ContainerSuite;
+import org.apache.gravitino.integration.test.container.HiveContainer;
+import org.apache.gravitino.integration.test.util.GravitinoITUtils;
+import org.apache.gravitino.server.web.JettyServerConfig;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.memory.MemoryCatalogWithMetadataLocationSupport;
+import org.apache.iceberg.rest.requests.CreateNamespaceRequest;
+import org.apache.iceberg.rest.requests.CreateTableRequest;
+import org.apache.iceberg.rest.responses.LoadTableResponse;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+import org.junit.jupiter.api.TestInstance.Lifecycle;
+
+/** Docker IT for default table metadata cache in {@link
IcebergCatalogWrapper}. */
+@Tag("gravitino-docker-test")
+@TestInstance(Lifecycle.PER_CLASS)
+public class IcebergCacheIT {
+
+ private static final ContainerSuite CONTAINER_SUITE =
ContainerSuite.getInstance();
+
+ private static final Schema TABLE_SCHEMA =
+ new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get()));
+
+ private IcebergRESTServerManager restServer;
+
+ @BeforeAll
+ void startDockerEnv() throws Exception {
+ CONTAINER_SUITE.startHiveContainer();
+ restServer = IcebergRESTServerManager.create();
+ Map<String, String> serverConfigs = new HashMap<>();
+ serverConfigs.put(
+ IcebergConfig.ICEBERG_CONFIG_PREFIX +
IcebergConfig.CATALOG_BACKEND.getKey(),
+ IcebergCatalogBackend.HIVE.toString().toLowerCase());
+ serverConfigs.put(
+ IcebergConfig.ICEBERG_CONFIG_PREFIX +
IcebergConfig.CATALOG_URI.getKey(),
+ hiveMetastoreUri());
+ serverConfigs.put(
+ IcebergConfig.ICEBERG_CONFIG_PREFIX +
IcebergConfig.CATALOG_WAREHOUSE.getKey(),
+ hdfsWarehouse("iceberg-rest-cache-it"));
+ restServer.registerCustomConfigs(serverConfigs);
+ restServer.startIcebergRESTServer();
+ }
+
+ @AfterAll
+ void stopDockerEnv() throws Exception {
+ if (restServer != null) {
+ restServer.stopIcebergRESTServer();
+ }
+ }
+
+ @Test
+ void testRestCacheDisabled() throws Exception {
+ int port = restPort();
+ Map<String, String> properties = new HashMap<>();
+ properties.put(
+ IcebergConstants.CATALOG_BACKEND,
IcebergCatalogBackend.REST.name().toLowerCase());
+ properties.put(IcebergConstants.URI,
String.format("http://127.0.0.1:%d/iceberg/";, port));
+ properties.put(IcebergConstants.CATALOG_BACKEND_NAME, "rest_cache_it");
+
+ IcebergConfig icebergConfig = new IcebergConfig(properties);
+ Assertions.assertEquals(
+ LocalTableMetadataCache.class.getName(),
+ icebergConfig.get(IcebergConfig.TABLE_METADATA_CACHE_IMPL));
+
+ try (IcebergCatalogWrapper wrapper = new
IcebergCatalogWrapper(icebergConfig)) {
+ TableMetadataCache cache = metadataCache(wrapper);
+ Assertions.assertSame(TableMetadataCache.DUMMY, cache);
+ }
+ }
+
+ @Test
+ void testCustomNoSupportLocationOff() throws Exception {
+ Map<String, String> properties = new HashMap<>();
+ properties.put(
+ IcebergConstants.CATALOG_BACKEND,
IcebergCatalogBackend.CUSTOM.name().toLowerCase());
+ properties.put(
+ IcebergConstants.CATALOG_BACKEND_IMPL,
CustomNoSupportLocationCatalog.class.getName());
+ properties.put(IcebergConstants.CATALOG_BACKEND_NAME,
"custom_no_support_loc");
+ properties.put(IcebergConstants.URI, hiveMetastoreUri());
+ properties.put(IcebergConstants.WAREHOUSE,
hdfsWarehouse("custom-no-support-loc"));
+
+ IcebergConfig icebergConfig = new IcebergConfig(properties);
+ Assertions.assertEquals(
+ LocalTableMetadataCache.class.getName(),
+ icebergConfig.get(IcebergConfig.TABLE_METADATA_CACHE_IMPL));
+
+ try (IcebergCatalogWrapper wrapper = new
IcebergCatalogWrapper(icebergConfig)) {
+ TableMetadataCache cache = metadataCache(wrapper);
+ Assertions.assertSame(TableMetadataCache.DUMMY, cache);
+ }
+ }
+
+ @Test
+ void testCustomSupportLocationOn() throws Exception {
+ Map<String, String> properties = new HashMap<>();
+ properties.put(
+ IcebergConstants.CATALOG_BACKEND,
IcebergCatalogBackend.CUSTOM.name().toLowerCase());
+ properties.put(
+ IcebergConstants.CATALOG_BACKEND_IMPL,
CustomSupportLocationCatalog.class.getName());
+ properties.put(IcebergConstants.CATALOG_BACKEND_NAME,
"custom_support_loc");
+ properties.put(IcebergConstants.URI, hiveMetastoreUri());
+ properties.put(IcebergConstants.WAREHOUSE,
hdfsWarehouse("custom-support-loc"));
+
+ IcebergConfig icebergConfig = new IcebergConfig(properties);
+ Assertions.assertEquals(
+ LocalTableMetadataCache.class.getName(),
+ icebergConfig.get(IcebergConfig.TABLE_METADATA_CACHE_IMPL));
+ Assertions.assertEquals(1000,
icebergConfig.get(IcebergConfig.TABLE_METADATA_CACHE_CAPACITY));
+
+ Namespace namespace = Namespace.of("cache_ns");
+ TableIdentifier tableId = TableIdentifier.of(namespace, "cache_tbl");
+
+ try (IcebergCatalogWrapper wrapper = new
IcebergCatalogWrapper(icebergConfig)) {
+ TableMetadataCache cache = metadataCache(wrapper);
+ Assertions.assertInstanceOf(LocalTableMetadataCache.class, cache);
+
+
wrapper.createNamespace(CreateNamespaceRequest.builder().withNamespace(namespace).build());
+ LoadTableResponse createResponse =
+ wrapper.createTable(
+ namespace,
+ CreateTableRequest.builder()
+ .withName(tableId.name())
+ .withSchema(TABLE_SCHEMA)
+ .build());
+ Assertions.assertNotNull(createResponse);
+
+ Assertions.assertTrue(cache.getTableMetadata(tableId).isPresent());
+
+ LoadTableResponse loadResponse = wrapper.loadTable(tableId);
+ Assertions.assertNotNull(loadResponse);
+ Assertions.assertTrue(cache.getTableMetadata(tableId).isPresent());
+ }
+ }
+
+ private static String hiveMetastoreUri() {
+ return String.format(
+ "thrift://%s:%d",
+ CONTAINER_SUITE.getHiveContainer().getContainerIpAddress(),
+ HiveContainer.HIVE_METASTORE_PORT);
+ }
+
+ private static String hdfsWarehouse(String suffix) {
+ return GravitinoITUtils.genRandomName(
+ String.format(
+ "hdfs://%s:%d/user/hive/warehouse-%s",
+ CONTAINER_SUITE.getHiveContainer().getContainerIpAddress(),
+ HiveContainer.HDFS_DEFAULTFS_PORT,
+ suffix));
+ }
+
+ private int restPort() {
+ JettyServerConfig jettyServerConfig =
+ JettyServerConfig.fromConfig(
+ restServer.getServerConfig(), IcebergConfig.ICEBERG_CONFIG_PREFIX);
+ return jettyServerConfig.getHttpPort();
+ }
+
+ private static TableMetadataCache metadataCache(IcebergCatalogWrapper
wrapper) throws Exception {
+ Method method =
IcebergCatalogWrapper.class.getDeclaredMethod("getMetadataCache");
+ method.setAccessible(true);
+ return (TableMetadataCache) method.invoke(wrapper);
+ }
+
+ /** Custom catalog that does not support metadata location for cache IT. */
+ public static class CustomNoSupportLocationCatalog implements Catalog {
+
+ @Override
+ public List<TableIdentifier> listTables(Namespace namespace) {
+ return List.of();
+ }
+
+ @Override
+ public boolean dropTable(TableIdentifier identifier, boolean purge) {
+ return false;
+ }
+
+ @Override
+ public void renameTable(TableIdentifier from, TableIdentifier to) {}
+
+ @Override
+ public Table loadTable(TableIdentifier identifier) {
+ return null;
+ }
+ }
+
+ /** Custom catalog that supports metadata location for cache IT. */
+ public static class CustomSupportLocationCatalog extends
MemoryCatalogWithMetadataLocationSupport
+ implements SupportsMetadataLocation {}
+}
