This is an automated email from the ASF dual-hosted git repository.
yuqi1129 pushed a commit to branch branch-1.3
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/branch-1.3 by this push:
new ada141d69d [Cherry-pick to branch-1.3] [#11136] fix(iceberg): add REST
catalog backend HTTP timeout configs (#11270) (#11466)
ada141d69d is described below
commit ada141d69d088cd7f39ee0032b1b8130d8b443c7
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Sun Jun 7 21:39:05 2026 +0800
[Cherry-pick to branch-1.3] [#11136] fix(iceberg): add REST catalog backend
HTTP timeout configs (#11270) (#11466)
**Cherry-pick Information:**
- Original commit: 65c4ed2978119e03744ff500cf0a27de861fc34b
- Target branch: `branch-1.3`
- Status: ✅ Clean cherry-pick (no conflicts)
Co-authored-by: ningsh7 <[email protected]>
---
.../lakehouse/iceberg/IcebergConstants.java | 15 +++++
.../lakehouse/iceberg/IcebergPropertiesUtils.java | 6 ++
.../iceberg/TestIcebergPropertiesUtils.java | 17 +++++
.../iceberg/IcebergCatalogPropertiesMetadata.java | 14 ++++-
.../TestIcebergCatalogPropertiesMetadata.java | 68 ++++++++++++++++++++
dev/docker/iceberg-rest-server/rewrite_config.py | 2 +
docs/iceberg-rest-service.md | 68 ++++++++++----------
docs/lakehouse-iceberg-catalog.md | 12 ++--
.../gravitino/iceberg/common/IcebergConfig.java | 22 +++++++
.../iceberg/common/utils/IcebergCatalogUtil.java | 14 +++++
.../iceberg/common/TestIcebergConfig.java | 72 ++++++++++++++++++++++
.../common/utils/TestIcebergCatalogUtil.java | 43 +++++++++++++
.../iceberg/service/TestCatalogWrapperForREST.java | 24 ++++----
.../provider/TestDynamicIcebergConfigProvider.java | 17 +++++
14 files changed, 346 insertions(+), 48 deletions(-)
diff --git
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java
index 58606f43f3..354153b303 100644
---
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java
+++
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java
@@ -104,6 +104,21 @@ public class IcebergConstants {
public static final String ICEBERG_REST_DEFAULT_CATALOG = "default_catalog";
public static final String ICEBERG_REST_DEFAULT_DYNAMIC_CATALOG_NAME =
"default-catalog-name";
public static final String ICEBERG_REST_DISABLE_REST_AUTHZ =
"disable-rest-authz";
+ /** Gravitino config key for REST catalog backend HTTP connection timeout. */
+ public static final String REST_CATALOG_BACKEND_CLIENT_CONNECTION_TIMEOUT_MS
=
+ "rest-client-connection-timeout-ms";
+
+ /** Gravitino config key for REST catalog backend HTTP socket timeout. */
+ public static final String REST_CATALOG_BACKEND_CLIENT_SOCKET_TIMEOUT_MS =
+ "rest-client-socket-timeout-ms";
+
+ /** Iceberg REST client property key for HTTP connection timeout. */
+ public static final String ICEBERG_REST_CLIENT_CONNECTION_TIMEOUT_MS =
+ "rest.client.connection-timeout-ms";
+
+ /** Iceberg REST client property key for HTTP socket timeout. */
+ public static final String ICEBERG_REST_CLIENT_SOCKET_TIMEOUT_MS =
+ "rest.client.socket-timeout-ms";
public static final String TABLE_METADATA_CACHE_IMPL =
"table-metadata-cache-impl";
public static final String TABLE_METADATA_CACHE_CAPACITY =
"table-metadata-cache-capacity";
diff --git
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergPropertiesUtils.java
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergPropertiesUtils.java
index e333787eb2..c9b96ee5f7 100644
---
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergPropertiesUtils.java
+++
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergPropertiesUtils.java
@@ -79,6 +79,12 @@ public class IcebergPropertiesUtils {
IcebergConstants.TABLE_METADATA_CACHE_EXPIRE_MINUTES,
IcebergConstants.TABLE_METADATA_CACHE_EXPIRE_MINUTES);
map.put(IcebergConstants.DATA_ACCESS,
IcebergConstants.ICEBERG_ACCESS_DELEGATION);
+ map.put(
+ IcebergConstants.REST_CATALOG_BACKEND_CLIENT_CONNECTION_TIMEOUT_MS,
+ IcebergConstants.ICEBERG_REST_CLIENT_CONNECTION_TIMEOUT_MS);
+ map.put(
+ IcebergConstants.REST_CATALOG_BACKEND_CLIENT_SOCKET_TIMEOUT_MS,
+ IcebergConstants.ICEBERG_REST_CLIENT_SOCKET_TIMEOUT_MS);
GRAVITINO_CONFIG_TO_ICEBERG = Collections.unmodifiableMap(map);
diff --git
a/catalogs/catalog-common/src/test/java/org/apache/gravitino/lakehouse/iceberg/TestIcebergPropertiesUtils.java
b/catalogs/catalog-common/src/test/java/org/apache/gravitino/lakehouse/iceberg/TestIcebergPropertiesUtils.java
index a02a750a95..8a60f2de24 100644
---
a/catalogs/catalog-common/src/test/java/org/apache/gravitino/lakehouse/iceberg/TestIcebergPropertiesUtils.java
+++
b/catalogs/catalog-common/src/test/java/org/apache/gravitino/lakehouse/iceberg/TestIcebergPropertiesUtils.java
@@ -40,6 +40,23 @@ public class TestIcebergPropertiesUtils {
"jdbc-schema-version must be translated to jdbc.schema-version for
Iceberg");
}
+ @Test
+ void testRESTCatalogBackendClientTimeoutPropertiesAreMapped() {
+ Map<String, String> gravitinoProps =
+ ImmutableMap.of(
+ IcebergConstants.REST_CATALOG_BACKEND_CLIENT_CONNECTION_TIMEOUT_MS,
+ "1000",
+ IcebergConstants.REST_CATALOG_BACKEND_CLIENT_SOCKET_TIMEOUT_MS,
+ "2000");
+ Map<String, String> icebergProps =
+ IcebergPropertiesUtils.toIcebergCatalogProperties(gravitinoProps);
+
+ Assertions.assertEquals(
+ "1000",
icebergProps.get(IcebergConstants.ICEBERG_REST_CLIENT_CONNECTION_TIMEOUT_MS));
+ Assertions.assertEquals(
+ "2000",
icebergProps.get(IcebergConstants.ICEBERG_REST_CLIENT_SOCKET_TIMEOUT_MS));
+ }
+
@Test
void testGetCatalogBackendName() {
Map<String, String> catalogProperties =
diff --git
a/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java
b/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java
index 49680a512d..2f7900ce8b 100644
---
a/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java
+++
b/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java
@@ -150,7 +150,19 @@ public class IcebergCatalogPropertiesMetadata extends
BaseCatalogPropertiesMetad
+ " remote-signing.",
false,
null,
- false));
+ false),
+ integerOptionalPropertyEntry(
+
IcebergConstants.REST_CATALOG_BACKEND_CLIENT_CONNECTION_TIMEOUT_MS,
+ "HTTP connection timeout in milliseconds for the REST catalog
backend",
+ false /* immutable */,
+ 10000 /* defaultValue */,
+ false /* hidden */),
+ integerOptionalPropertyEntry(
+ IcebergConstants.REST_CATALOG_BACKEND_CLIENT_SOCKET_TIMEOUT_MS,
+ "HTTP socket timeout in milliseconds for the REST catalog
backend",
+ false /* immutable */,
+ 60000 /* defaultValue */,
+ false /* hidden */));
HashMap<String, PropertyEntry<?>> result = Maps.newHashMap();
result.putAll(Maps.uniqueIndex(propertyEntries, PropertyEntry::getName));
result.putAll(KerberosConfig.KERBEROS_PROPERTY_ENTRIES);
diff --git
a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/TestIcebergCatalogPropertiesMetadata.java
b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/TestIcebergCatalogPropertiesMetadata.java
index 125323f283..0faf851030 100644
---
a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/TestIcebergCatalogPropertiesMetadata.java
+++
b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/TestIcebergCatalogPropertiesMetadata.java
@@ -72,4 +72,72 @@ public class TestIcebergCatalogPropertiesMetadata {
IcebergConfig.TABLE_METADATA_CACHE_CAPACITY.getDefaultValue(),
metadata.getOrDefault(catalogProperties,
IcebergConstants.TABLE_METADATA_CACHE_CAPACITY));
}
+
+ @Test
+ void testRESTCatalogBackendClientTimeoutDefaultValues() {
+ Assertions.assertEquals(
+
IcebergConfig.REST_CATALOG_BACKEND_CLIENT_CONNECTION_TIMEOUT_MS.getDefaultValue(),
+ metadata.getDefaultValue(
+
IcebergConstants.REST_CATALOG_BACKEND_CLIENT_CONNECTION_TIMEOUT_MS));
+ Assertions.assertEquals(
+
IcebergConfig.REST_CATALOG_BACKEND_CLIENT_SOCKET_TIMEOUT_MS.getDefaultValue(),
+
metadata.getDefaultValue(IcebergConstants.REST_CATALOG_BACKEND_CLIENT_SOCKET_TIMEOUT_MS));
+ }
+
+ @Test
+ void testRESTCatalogBackendClientTimeoutsViaGetOrDefault() {
+ Map<String, String> catalogProperties =
+ ImmutableMap.of(
+ IcebergCatalogPropertiesMetadata.CATALOG_BACKEND,
+ "rest",
+ IcebergCatalogPropertiesMetadata.URI,
+ "http://127.0.0.1:9001/iceberg";);
+
+ Assertions.assertEquals(
+
IcebergConfig.REST_CATALOG_BACKEND_CLIENT_CONNECTION_TIMEOUT_MS.getDefaultValue(),
+ metadata.getOrDefault(
+ catalogProperties,
IcebergConstants.REST_CATALOG_BACKEND_CLIENT_CONNECTION_TIMEOUT_MS));
+ Assertions.assertEquals(
+
IcebergConfig.REST_CATALOG_BACKEND_CLIENT_SOCKET_TIMEOUT_MS.getDefaultValue(),
+ metadata.getOrDefault(
+ catalogProperties,
IcebergConstants.REST_CATALOG_BACKEND_CLIENT_SOCKET_TIMEOUT_MS));
+
+ catalogProperties =
+ ImmutableMap.of(
+ IcebergCatalogPropertiesMetadata.CATALOG_BACKEND,
+ "rest",
+ IcebergCatalogPropertiesMetadata.URI,
+ "http://127.0.0.1:9001/iceberg";,
+ IcebergConstants.REST_CATALOG_BACKEND_CLIENT_CONNECTION_TIMEOUT_MS,
+ "1234",
+ IcebergConstants.REST_CATALOG_BACKEND_CLIENT_SOCKET_TIMEOUT_MS,
+ "5678");
+
+ Assertions.assertEquals(
+ 1234,
+ metadata.getOrDefault(
+ catalogProperties,
IcebergConstants.REST_CATALOG_BACKEND_CLIENT_CONNECTION_TIMEOUT_MS));
+ Assertions.assertEquals(
+ 5678,
+ metadata.getOrDefault(
+ catalogProperties,
IcebergConstants.REST_CATALOG_BACKEND_CLIENT_SOCKET_TIMEOUT_MS));
+ }
+
+ @Test
+ void testRESTCatalogBackendClientTimeoutPropertiesAreTransformed() {
+ Map<String, String> catalogProperties =
+ ImmutableMap.of(
+ IcebergConstants.REST_CATALOG_BACKEND_CLIENT_CONNECTION_TIMEOUT_MS,
+ "1234",
+ IcebergConstants.REST_CATALOG_BACKEND_CLIENT_SOCKET_TIMEOUT_MS,
+ "5678");
+
+ Map<String, String> transformedProperties =
metadata.transformProperties(catalogProperties);
+
+ Assertions.assertEquals(
+ "1234",
+
transformedProperties.get(IcebergConstants.ICEBERG_REST_CLIENT_CONNECTION_TIMEOUT_MS));
+ Assertions.assertEquals(
+ "5678",
transformedProperties.get(IcebergConstants.ICEBERG_REST_CLIENT_SOCKET_TIMEOUT_MS));
+ }
}
diff --git a/dev/docker/iceberg-rest-server/rewrite_config.py
b/dev/docker/iceberg-rest-server/rewrite_config.py
index 908516e9a7..d6ffcf7ec9 100755
--- a/dev/docker/iceberg-rest-server/rewrite_config.py
+++ b/dev/docker/iceberg-rest-server/rewrite_config.py
@@ -29,6 +29,8 @@ env_map = {
"GRAVITINO_ICEBERG_REST_JDBC_USER": "jdbc-user",
"GRAVITINO_ICEBERG_REST_JDBC_PASSWORD": "jdbc-password",
"GRAVITINO_ICEBERG_REST_WAREHOUSE": "warehouse",
+ "GRAVITINO_ICEBERG_REST_REST_CLIENT_CONNECTION_TIMEOUT_MS":
"rest-client-connection-timeout-ms",
+ "GRAVITINO_ICEBERG_REST_REST_CLIENT_SOCKET_TIMEOUT_MS":
"rest-client-socket-timeout-ms",
# Credentials
"GRAVITINO_ICEBERG_REST_CREDENTIAL_PROVIDERS": "credential-providers",
# GCS
diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md
index afb6c17ef2..03f984a865 100644
--- a/docs/iceberg-rest-service.md
+++ b/docs/iceberg-rest-service.md
@@ -159,6 +159,8 @@ By default, when the backend catalog is a REST catalog,
IRC1 skips authorization
| `gravitino.iceberg-rest.catalog-backend` | The
Catalog backend of the Gravitino Iceberg REST catalog service. Use the value
**`rest`** for the REST catalog backend. |
`memory` | Yes | 0.2.0 |
| `gravitino.iceberg-rest.uri` | The
Iceberg REST catalog URI (IRC2), such as `http://127.0.0.1:9001/iceberg`.
|
(none) | Yes | 0.2.0 |
| `gravitino.iceberg-rest.warehouse` | The
catalog name in the Iceberg REST spec. Set to a specific catalog name, or leave
empty to use the default catalog on IRC2. |
(none) | No | 0.2.0 |
+| `gravitino.iceberg-rest.rest-client-connection-timeout-ms` | The
HTTP connection timeout in milliseconds for IRC1 requests to the REST catalog
backend. |
`10000` | No | 1.3.0 |
+| `gravitino.iceberg-rest.rest-client-socket-timeout-ms` | The
HTTP socket timeout in milliseconds for IRC1 requests to the REST catalog
backend.
| `60000` | No | 1.3.0 |
| `gravitino.iceberg-rest.data-access` | Data
access mode exposed to Iceberg REST clients via `/v1/config`. Supported values:
`vended-credentials`, `remote-signing`. |
(none) | No | 1.3.0 |
| `gravitino.iceberg-rest.disable-rest-authz` |
Whether IRC1 disables authorization when the target backend catalog is a REST
catalog. Set to `false` if you want IRC1 to enforce authorization before
proxying. | `true` | No | 1.3.0 |
@@ -704,38 +706,40 @@ docker run -d -p 9001:9001
apache/gravitino-iceberg-rest:latest
Gravitino Iceberg REST server in the Docker image could access local storage
by default, you could set the following environment variables if the storage is
cloud/remote storage like S3, refer to [storage section](#storage) for more
details.
-| Environment Variable | Configuration
items | Since version |
-|----------------------------------------------------------|------------------------------------------------------|------------------|
-| `GRAVITINO_ICEBERG_REST_HOST` |
`gravitino.iceberg-rest.host` | 1.3.0 |
-| `GRAVITINO_ICEBERG_REST_HTTP_PORT` |
`gravitino.iceberg-rest.httpPort` | 1.1.0 |
-| `GRAVITINO_ICEBERG_REST_URI` |
`gravitino.iceberg-rest.uri` | 1.3.0 |
-| `GRAVITINO_ICEBERG_REST_IO_IMPL` |
`gravitino.iceberg-rest.io-impl` | 1.3.0 |
-| `GRAVITINO_ICEBERG_REST_CATALOG_BACKEND` |
`gravitino.iceberg-rest.catalog-backend` | 1.3.0 |
-| `GRAVITINO_ICEBERG_REST_JDBC_DRIVER` |
`gravitino.iceberg-rest.jdbc-driver` | 1.3.0 |
-| `GRAVITINO_ICEBERG_REST_JDBC_USER` |
`gravitino.iceberg-rest.jdbc-user` | 1.3.0 |
-| `GRAVITINO_ICEBERG_REST_JDBC_PASSWORD` |
`gravitino.iceberg-rest.jdbc-password` | 1.3.0 |
-| `GRAVITINO_ICEBERG_REST_WAREHOUSE` |
`gravitino.iceberg-rest.warehouse` | 1.3.0 |
-| `GRAVITINO_ICEBERG_REST_CREDENTIAL_PROVIDERS` |
`gravitino.iceberg-rest.credential-providers` | 1.3.0 |
-| `GRAVITINO_ICEBERG_REST_GCS_SERVICE_ACCOUNT_FILE` |
`gravitino.iceberg-rest.gcs-service-account-file` | 1.3.0 |
-| `GRAVITINO_ICEBERG_REST_S3_ACCESS_KEY` |
`gravitino.iceberg-rest.s3-access-key-id` | 1.3.0 |
-| `GRAVITINO_ICEBERG_REST_S3_SECRET_KEY` |
`gravitino.iceberg-rest.s3-secret-access-key` | 1.3.0 |
-| `GRAVITINO_ICEBERG_REST_S3_ENDPOINT` |
`gravitino.iceberg-rest.s3-endpoint` | 1.3.0 |
-| `GRAVITINO_ICEBERG_REST_S3_REGION` |
`gravitino.iceberg-rest.s3-region` | 1.3.0 |
-| `GRAVITINO_ICEBERG_REST_S3_PATH_STYLE_ACCESS` |
`gravitino.iceberg-rest.s3-path-style-access` | 1.3.0 |
-| `GRAVITINO_ICEBERG_REST_S3_ROLE_ARN` |
`gravitino.iceberg-rest.s3-role-arn` | 1.3.0 |
-| `GRAVITINO_ICEBERG_REST_S3_EXTERNAL_ID` |
`gravitino.iceberg-rest.s3-external-id` | 1.3.0 |
-| `GRAVITINO_ICEBERG_REST_S3_TOKEN_SERVICE_ENDPOINT` |
`gravitino.iceberg-rest.s3-token-service-endpoint` | 1.3.0 |
-| `GRAVITINO_ICEBERG_REST_AZURE_STORAGE_ACCOUNT_NAME` |
`gravitino.iceberg-rest.azure-storage-account-name` | 1.3.0 |
-| `GRAVITINO_ICEBERG_REST_AZURE_STORAGE_ACCOUNT_KEY` |
`gravitino.iceberg-rest.azure-storage-account-key` | 1.3.0 |
-| `GRAVITINO_ICEBERG_REST_AZURE_TENANT_ID` |
`gravitino.iceberg-rest.azure-tenant-id` | 1.3.0 |
-| `GRAVITINO_ICEBERG_REST_AZURE_CLIENT_ID` |
`gravitino.iceberg-rest.azure-client-id` | 1.3.0 |
-| `GRAVITINO_ICEBERG_REST_AZURE_CLIENT_SECRET` |
`gravitino.iceberg-rest.azure-client-secret` | 1.3.0 |
-| `GRAVITINO_ICEBERG_REST_OSS_ACCESS_KEY` |
`gravitino.iceberg-rest.oss-access-key-id` | 1.3.0 |
-| `GRAVITINO_ICEBERG_REST_OSS_SECRET_KEY` |
`gravitino.iceberg-rest.oss-secret-access-key` | 1.3.0 |
-| `GRAVITINO_ICEBERG_REST_OSS_ENDPOINT` |
`gravitino.iceberg-rest.oss-endpoint` | 1.3.0 |
-| `GRAVITINO_ICEBERG_REST_OSS_REGION` |
`gravitino.iceberg-rest.oss-region` | 1.3.0 |
-| `GRAVITINO_ICEBERG_REST_OSS_ROLE_ARN` |
`gravitino.iceberg-rest.oss-role-arn` | 1.3.0 |
-| `GRAVITINO_ICEBERG_REST_OSS_EXTERNAL_ID` |
`gravitino.iceberg-rest.oss-external-id` | 1.3.0 |
+| Environment Variable |
Configuration items | Since version |
+|----------------------------------------------------------------|------------------------------------------------------------|---------------|
+| `GRAVITINO_ICEBERG_REST_HOST` |
`gravitino.iceberg-rest.host` | 1.3.0 |
+| `GRAVITINO_ICEBERG_REST_HTTP_PORT` |
`gravitino.iceberg-rest.httpPort` | 1.1.0 |
+| `GRAVITINO_ICEBERG_REST_URI` |
`gravitino.iceberg-rest.uri` | 1.3.0 |
+| `GRAVITINO_ICEBERG_REST_IO_IMPL` |
`gravitino.iceberg-rest.io-impl` | 1.3.0 |
+| `GRAVITINO_ICEBERG_REST_CATALOG_BACKEND` |
`gravitino.iceberg-rest.catalog-backend` | 1.3.0 |
+| `GRAVITINO_ICEBERG_REST_JDBC_DRIVER` |
`gravitino.iceberg-rest.jdbc-driver` | 1.3.0 |
+| `GRAVITINO_ICEBERG_REST_JDBC_USER` |
`gravitino.iceberg-rest.jdbc-user` | 1.3.0 |
+| `GRAVITINO_ICEBERG_REST_JDBC_PASSWORD` |
`gravitino.iceberg-rest.jdbc-password` | 1.3.0 |
+| `GRAVITINO_ICEBERG_REST_WAREHOUSE` |
`gravitino.iceberg-rest.warehouse` | 1.3.0 |
+| `GRAVITINO_ICEBERG_REST_REST_CLIENT_CONNECTION_TIMEOUT_MS` |
`gravitino.iceberg-rest.rest-client-connection-timeout-ms` | 1.3.0 |
+| `GRAVITINO_ICEBERG_REST_REST_CLIENT_SOCKET_TIMEOUT_MS` |
`gravitino.iceberg-rest.rest-client-socket-timeout-ms` | 1.3.0 |
+| `GRAVITINO_ICEBERG_REST_CREDENTIAL_PROVIDERS` |
`gravitino.iceberg-rest.credential-providers` | 1.3.0 |
+| `GRAVITINO_ICEBERG_REST_GCS_SERVICE_ACCOUNT_FILE` |
`gravitino.iceberg-rest.gcs-service-account-file` | 1.3.0 |
+| `GRAVITINO_ICEBERG_REST_S3_ACCESS_KEY` |
`gravitino.iceberg-rest.s3-access-key-id` | 1.3.0 |
+| `GRAVITINO_ICEBERG_REST_S3_SECRET_KEY` |
`gravitino.iceberg-rest.s3-secret-access-key` | 1.3.0 |
+| `GRAVITINO_ICEBERG_REST_S3_ENDPOINT` |
`gravitino.iceberg-rest.s3-endpoint` | 1.3.0 |
+| `GRAVITINO_ICEBERG_REST_S3_REGION` |
`gravitino.iceberg-rest.s3-region` | 1.3.0 |
+| `GRAVITINO_ICEBERG_REST_S3_PATH_STYLE_ACCESS` |
`gravitino.iceberg-rest.s3-path-style-access` | 1.3.0 |
+| `GRAVITINO_ICEBERG_REST_S3_ROLE_ARN` |
`gravitino.iceberg-rest.s3-role-arn` | 1.3.0 |
+| `GRAVITINO_ICEBERG_REST_S3_EXTERNAL_ID` |
`gravitino.iceberg-rest.s3-external-id` | 1.3.0 |
+| `GRAVITINO_ICEBERG_REST_S3_TOKEN_SERVICE_ENDPOINT` |
`gravitino.iceberg-rest.s3-token-service-endpoint` | 1.3.0 |
+| `GRAVITINO_ICEBERG_REST_AZURE_STORAGE_ACCOUNT_NAME` |
`gravitino.iceberg-rest.azure-storage-account-name` | 1.3.0 |
+| `GRAVITINO_ICEBERG_REST_AZURE_STORAGE_ACCOUNT_KEY` |
`gravitino.iceberg-rest.azure-storage-account-key` | 1.3.0 |
+| `GRAVITINO_ICEBERG_REST_AZURE_TENANT_ID` |
`gravitino.iceberg-rest.azure-tenant-id` | 1.3.0 |
+| `GRAVITINO_ICEBERG_REST_AZURE_CLIENT_ID` |
`gravitino.iceberg-rest.azure-client-id` | 1.3.0 |
+| `GRAVITINO_ICEBERG_REST_AZURE_CLIENT_SECRET` |
`gravitino.iceberg-rest.azure-client-secret` | 1.3.0 |
+| `GRAVITINO_ICEBERG_REST_OSS_ACCESS_KEY` |
`gravitino.iceberg-rest.oss-access-key-id` | 1.3.0 |
+| `GRAVITINO_ICEBERG_REST_OSS_SECRET_KEY` |
`gravitino.iceberg-rest.oss-secret-access-key` | 1.3.0 |
+| `GRAVITINO_ICEBERG_REST_OSS_ENDPOINT` |
`gravitino.iceberg-rest.oss-endpoint` | 1.3.0 |
+| `GRAVITINO_ICEBERG_REST_OSS_REGION` |
`gravitino.iceberg-rest.oss-region` | 1.3.0 |
+| `GRAVITINO_ICEBERG_REST_OSS_ROLE_ARN` |
`gravitino.iceberg-rest.oss-role-arn` | 1.3.0 |
+| `GRAVITINO_ICEBERG_REST_OSS_EXTERNAL_ID` |
`gravitino.iceberg-rest.oss-external-id` | 1.3.0 |
The below environment is deprecated; use the corresponding configuration items
instead.
diff --git a/docs/lakehouse-iceberg-catalog.md
b/docs/lakehouse-iceberg-catalog.md
index 6744337748..4e098da075 100644
--- a/docs/lakehouse-iceberg-catalog.md
+++ b/docs/lakehouse-iceberg-catalog.md
@@ -71,11 +71,13 @@ If you are using multiple JDBC catalog backends, setting
`jdbc-initialize` to tr
For the REST catalog backend, `warehouse` identifies the catalog in the
Iceberg REST spec. In the Gravitino Iceberg REST server, `warehouse` maps to
the catalog name. An empty value means the default catalog.
-`data-access` controls how the Iceberg REST client accesses table data when
using a REST backend:
+The following properties tune REST backend behavior:
-| Property name | Description
| Default value |
Required | Since Version |
-|----------------|-------------------------------------------------------------------------------------------------------------------------|---------------|----------|---------------|
-| `data-access` | Data access mode for REST catalog backend. Supported values
are `vended-credentials` and `remote-signing`. | (none) |
No | 1.3.0 |
+| Property name | Description
| Default
value | Required | Since Version |
+|---------------------------------------|------------------------------------------------------------------------------------------------------------|---------------|----------|---------------|
+| `data-access` | Data access mode for REST catalog
backend. Supported values are `vended-credentials` and `remote-signing`. |
(none) | No | 1.3.0 |
+| `rest-client-connection-timeout-ms` | The HTTP connection timeout in
milliseconds for requests to the REST catalog backend. |
10000 | No | 1.3.0 |
+| `rest-client-socket-timeout-ms` | The HTTP socket timeout in
milliseconds for requests to the REST catalog backend.
| 60000 | No | 1.3.0 |
- `vended-credentials`: request credential vending from the Iceberg REST
server.
- `remote-signing`: Gravitino doesn't support this mode yet.
@@ -92,6 +94,8 @@ curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \
"properties": {
"catalog-backend": "rest",
"uri": "http://localhost:9001/iceberg";,
+ "rest-client-connection-timeout-ms": "10000",
+ "rest-client-socket-timeout-ms": "60000",
"data-access": "vended-credentials"
}
}' http://localhost:8090/api/metalakes/metalake/catalogs
diff --git
a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java
b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java
index aeba5f01a8..305bf95242 100644
---
a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java
+++
b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java
@@ -230,6 +230,28 @@ public class IcebergConfig extends Config implements
OverwriteDefaultConfig {
.booleanConf()
.createWithDefault(true);
+ /** HTTP connection timeout in milliseconds for the REST catalog backend. */
+ public static final ConfigEntry<Integer>
REST_CATALOG_BACKEND_CLIENT_CONNECTION_TIMEOUT_MS =
+ new
ConfigBuilder(IcebergConstants.REST_CATALOG_BACKEND_CLIENT_CONNECTION_TIMEOUT_MS)
+ .alternatives(
+
Collections.singletonList(IcebergConstants.ICEBERG_REST_CLIENT_CONNECTION_TIMEOUT_MS))
+ .doc("HTTP connection timeout in milliseconds for the REST catalog
backend.")
+ .version(ConfigConstants.VERSION_1_3_0)
+ .intConf()
+ .checkValue(value -> value > 0,
ConfigConstants.POSITIVE_NUMBER_ERROR_MSG)
+ .createWithDefault(10000);
+
+ /** HTTP socket timeout in milliseconds for the REST catalog backend. */
+ public static final ConfigEntry<Integer>
REST_CATALOG_BACKEND_CLIENT_SOCKET_TIMEOUT_MS =
+ new
ConfigBuilder(IcebergConstants.REST_CATALOG_BACKEND_CLIENT_SOCKET_TIMEOUT_MS)
+ .alternatives(
+
Collections.singletonList(IcebergConstants.ICEBERG_REST_CLIENT_SOCKET_TIMEOUT_MS))
+ .doc("HTTP socket timeout in milliseconds for the REST catalog
backend.")
+ .version(ConfigConstants.VERSION_1_3_0)
+ .intConf()
+ .checkValue(value -> value > 0,
ConfigConstants.POSITIVE_NUMBER_ERROR_MSG)
+ .createWithDefault(60000);
+
public static final ConfigEntry<String> GRAVITINO_URI =
new ConfigBuilder(IcebergConstants.GRAVITINO_URI)
.doc(
diff --git
a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/utils/IcebergCatalogUtil.java
b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/utils/IcebergCatalogUtil.java
index 8af0c6325c..0d2d5462d0 100644
---
a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/utils/IcebergCatalogUtil.java
+++
b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/utils/IcebergCatalogUtil.java
@@ -145,6 +145,7 @@ public class IcebergCatalogUtil {
// REST catalog must use forward access token from the user request
properties.put(AuthProperties.AUTH_TYPE,
UserPrincipalForwardingAuthManager.class.getName());
+ applyRestCatalogHttpTimeoutProperties(icebergConfig, properties);
properties.forEach(hdfsConfiguration::set);
restCatalog.setConf(hdfsConfiguration);
@@ -166,6 +167,19 @@ public class IcebergCatalogUtil {
properties.putIfAbsent(IcebergConstants.IO_IMPL,
ResolvingFileIO.class.getName());
}
+ @VisibleForTesting
+ static void applyRestCatalogHttpTimeoutProperties(
+ IcebergConfig icebergConfig, Map<String, String> properties) {
+ properties.put(
+ IcebergConstants.ICEBERG_REST_CLIENT_CONNECTION_TIMEOUT_MS,
+ String.valueOf(
+
icebergConfig.get(IcebergConfig.REST_CATALOG_BACKEND_CLIENT_CONNECTION_TIMEOUT_MS)));
+ properties.put(
+ IcebergConstants.ICEBERG_REST_CLIENT_SOCKET_TIMEOUT_MS,
+ String.valueOf(
+
icebergConfig.get(IcebergConfig.REST_CATALOG_BACKEND_CLIENT_SOCKET_TIMEOUT_MS)));
+ }
+
@VisibleForTesting
static Catalog loadCatalogBackend(String catalogType) {
return loadCatalogBackend(
diff --git
a/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/TestIcebergConfig.java
b/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/TestIcebergConfig.java
index ad8faaa236..aa6ed3fd67 100644
---
a/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/TestIcebergConfig.java
+++
b/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/TestIcebergConfig.java
@@ -21,6 +21,7 @@ package org.apache.gravitino.iceberg.common;
import com.google.common.collect.ImmutableMap;
import java.util.Map;
+import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants;
import org.apache.gravitino.iceberg.common.cache.LocalTableMetadataCache;
import org.apache.gravitino.server.web.JettyServerConfig;
import org.junit.jupiter.api.Assertions;
@@ -99,4 +100,75 @@ public class TestIcebergConfig {
Assertions.assertEquals(5,
config.get(IcebergConfig.ASYNC_CLEANUP_MAX_ATTEMPTS));
Assertions.assertEquals(720,
config.get(IcebergConfig.ASYNC_CLEANUP_RETENTION_HOURS));
}
+
+ @Test
+ public void testRESTCatalogBackendClientTimeoutDefaults() {
+ IcebergConfig icebergConfig = new IcebergConfig(ImmutableMap.of());
+
+ Assertions.assertEquals(
+ 10000,
icebergConfig.get(IcebergConfig.REST_CATALOG_BACKEND_CLIENT_CONNECTION_TIMEOUT_MS));
+ Assertions.assertEquals(
+ 60000,
icebergConfig.get(IcebergConfig.REST_CATALOG_BACKEND_CLIENT_SOCKET_TIMEOUT_MS));
+ Assertions.assertEquals(
+
IcebergConfig.REST_CATALOG_BACKEND_CLIENT_CONNECTION_TIMEOUT_MS.getDefaultValue(),
+
icebergConfig.get(IcebergConfig.REST_CATALOG_BACKEND_CLIENT_CONNECTION_TIMEOUT_MS));
+ Assertions.assertEquals(
+
IcebergConfig.REST_CATALOG_BACKEND_CLIENT_SOCKET_TIMEOUT_MS.getDefaultValue(),
+
icebergConfig.get(IcebergConfig.REST_CATALOG_BACKEND_CLIENT_SOCKET_TIMEOUT_MS));
+ }
+
+ @Test
+ public void testRESTCatalogBackendClientTimeoutConfigKeys() {
+ IcebergConfig icebergConfig =
+ new IcebergConfig(
+ ImmutableMap.of(
+
IcebergConfig.REST_CATALOG_BACKEND_CLIENT_CONNECTION_TIMEOUT_MS.getKey(),
+ "1234",
+
IcebergConfig.REST_CATALOG_BACKEND_CLIENT_SOCKET_TIMEOUT_MS.getKey(),
+ "5678"));
+
+ Assertions.assertEquals(
+ 1234,
icebergConfig.get(IcebergConfig.REST_CATALOG_BACKEND_CLIENT_CONNECTION_TIMEOUT_MS));
+ Assertions.assertEquals(
+ 5678,
icebergConfig.get(IcebergConfig.REST_CATALOG_BACKEND_CLIENT_SOCKET_TIMEOUT_MS));
+ }
+
+ @Test
+ public void testRESTCatalogBackendClientTimeoutIcebergPropertyAliases() {
+ IcebergConfig icebergConfig =
+ new IcebergConfig(
+ ImmutableMap.of(
+ IcebergConstants.ICEBERG_REST_CLIENT_CONNECTION_TIMEOUT_MS,
+ "2345",
+ IcebergConstants.ICEBERG_REST_CLIENT_SOCKET_TIMEOUT_MS,
+ "6789"));
+
+ Assertions.assertEquals(
+ 2345,
icebergConfig.get(IcebergConfig.REST_CATALOG_BACKEND_CLIENT_CONNECTION_TIMEOUT_MS));
+ Assertions.assertEquals(
+ 6789,
icebergConfig.get(IcebergConfig.REST_CATALOG_BACKEND_CLIENT_SOCKET_TIMEOUT_MS));
+ }
+
+ @Test
+ public void testRESTCatalogBackendClientTimeoutRejectsNonPositiveValues() {
+ IcebergConfig zeroConnectionTimeoutConfig =
+ new IcebergConfig(
+ ImmutableMap.of(
+
IcebergConfig.REST_CATALOG_BACKEND_CLIENT_CONNECTION_TIMEOUT_MS.getKey(), "0"));
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ zeroConnectionTimeoutConfig.get(
+
IcebergConfig.REST_CATALOG_BACKEND_CLIENT_CONNECTION_TIMEOUT_MS));
+
+ IcebergConfig negativeSocketTimeoutConfig =
+ new IcebergConfig(
+ ImmutableMap.of(
+
IcebergConfig.REST_CATALOG_BACKEND_CLIENT_SOCKET_TIMEOUT_MS.getKey(), "-1"));
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ negativeSocketTimeoutConfig.get(
+ IcebergConfig.REST_CATALOG_BACKEND_CLIENT_SOCKET_TIMEOUT_MS));
+ }
}
diff --git
a/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/utils/TestIcebergCatalogUtil.java
b/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/utils/TestIcebergCatalogUtil.java
index bf0fbd8e2c..0bf3745538 100644
---
a/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/utils/TestIcebergCatalogUtil.java
+++
b/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/utils/TestIcebergCatalogUtil.java
@@ -225,4 +225,47 @@ public class TestIcebergCatalogUtil {
Assertions.assertEquals(
"org.apache.iceberg.aws.s3.S3FileIO",
properties.get(IcebergConstants.IO_IMPL));
}
+
+ @Test
+ void testApplyRestCatalogHttpTimeoutPropertiesUsesDefaults() {
+ Map<String, String> properties = new HashMap<>();
+
+ IcebergCatalogUtil.applyRestCatalogHttpTimeoutProperties(
+ new IcebergConfig(new HashMap<>()), properties);
+
+ Assertions.assertEquals(
+ "10000",
properties.get(IcebergConstants.ICEBERG_REST_CLIENT_CONNECTION_TIMEOUT_MS));
+ Assertions.assertEquals(
+ "60000",
properties.get(IcebergConstants.ICEBERG_REST_CLIENT_SOCKET_TIMEOUT_MS));
+ }
+
+ @Test
+ void testApplyRestCatalogHttpTimeoutPropertiesUsesConfiguredValues() {
+ Map<String, String> config = new HashMap<>();
+
config.put(IcebergConfig.REST_CATALOG_BACKEND_CLIENT_CONNECTION_TIMEOUT_MS.getKey(),
"1234");
+
config.put(IcebergConfig.REST_CATALOG_BACKEND_CLIENT_SOCKET_TIMEOUT_MS.getKey(),
"5678");
+ Map<String, String> properties = new HashMap<>();
+
+ IcebergCatalogUtil.applyRestCatalogHttpTimeoutProperties(new
IcebergConfig(config), properties);
+
+ Assertions.assertEquals(
+ "1234",
properties.get(IcebergConstants.ICEBERG_REST_CLIENT_CONNECTION_TIMEOUT_MS));
+ Assertions.assertEquals(
+ "5678",
properties.get(IcebergConstants.ICEBERG_REST_CLIENT_SOCKET_TIMEOUT_MS));
+ }
+
+ @Test
+ void testApplyRestCatalogHttpTimeoutPropertiesUsesIcebergPropertyAliases() {
+ Map<String, String> config = new HashMap<>();
+ config.put(IcebergConstants.ICEBERG_REST_CLIENT_CONNECTION_TIMEOUT_MS,
"2345");
+ config.put(IcebergConstants.ICEBERG_REST_CLIENT_SOCKET_TIMEOUT_MS, "6789");
+ Map<String, String> properties = new HashMap<>();
+
+ IcebergCatalogUtil.applyRestCatalogHttpTimeoutProperties(new
IcebergConfig(config), properties);
+
+ Assertions.assertEquals(
+ "2345",
properties.get(IcebergConstants.ICEBERG_REST_CLIENT_CONNECTION_TIMEOUT_MS));
+ Assertions.assertEquals(
+ "6789",
properties.get(IcebergConstants.ICEBERG_REST_CLIENT_SOCKET_TIMEOUT_MS));
+ }
}
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/TestCatalogWrapperForREST.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/TestCatalogWrapperForREST.java
index ce845ce020..e372170585 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/TestCatalogWrapperForREST.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/TestCatalogWrapperForREST.java
@@ -549,17 +549,15 @@ public class TestCatalogWrapperForREST {
RESTCatalog restCatalog = mock(RESTCatalog.class);
when(restCatalog.properties())
.thenReturn(
- ImmutableMap.of(
- IcebergConstants.URI,
- "http://merged-from-remote-config:9999";,
- IcebergConstants.IO_IMPL,
- "org.apache.iceberg.aws.s3.S3FileIO",
- IcebergConstants.ICEBERG_S3_ENDPOINT,
- "http://localhost:9000";,
- IcebergConstants.ICEBERG_ACCESS_DELEGATION,
- "vended-credentials",
- IcebergConstants.WAREHOUSE,
- "s3://remote/warehouse"));
+ ImmutableMap.<String, String>builder()
+ .put(IcebergConstants.URI,
"http://merged-from-remote-config:9999";)
+ .put(IcebergConstants.IO_IMPL,
"org.apache.iceberg.aws.s3.S3FileIO")
+ .put(IcebergConstants.ICEBERG_S3_ENDPOINT,
"http://localhost:9000";)
+ .put(IcebergConstants.ICEBERG_ACCESS_DELEGATION,
"vended-credentials")
+ .put(IcebergConstants.WAREHOUSE, "s3://remote/warehouse")
+
.put(IcebergConstants.ICEBERG_REST_CLIENT_CONNECTION_TIMEOUT_MS, "10000")
+ .put(IcebergConstants.ICEBERG_REST_CLIENT_SOCKET_TIMEOUT_MS,
"60000")
+ .build());
// FederatedCatalogWrapper sources the client config from the remote
RESTCatalog's properties().
CatalogWrapperForREST wrapper = new StaticCatalogWrapperForREST("test",
config, restCatalog);
@@ -571,6 +569,10 @@ public class TestCatalogWrapperForREST {
"http://localhost:9000";,
configToClients.get(IcebergConstants.ICEBERG_S3_ENDPOINT));
Assertions.assertEquals(
"vended-credentials",
configToClients.get(IcebergConstants.ICEBERG_ACCESS_DELEGATION));
+ Assertions.assertFalse(
+
configToClients.containsKey(IcebergConstants.ICEBERG_REST_CLIENT_CONNECTION_TIMEOUT_MS));
+ Assertions.assertFalse(
+
configToClients.containsKey(IcebergConstants.ICEBERG_REST_CLIENT_SOCKET_TIMEOUT_MS));
}
@Test
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/provider/TestDynamicIcebergConfigProvider.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/provider/TestDynamicIcebergConfigProvider.java
index 2177778734..93f5be1b5f 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/provider/TestDynamicIcebergConfigProvider.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/provider/TestDynamicIcebergConfigProvider.java
@@ -248,6 +248,9 @@ public class TestDynamicIcebergConfigProvider {
catalogProperties.put("catalog.backend-name", "custom_backend");
catalogProperties.put("gravitino.bypass.custom-k1", "custom-v1");
catalogProperties.put("custom-k2", "custom-v2");
+ catalogProperties.put(
+ IcebergConstants.REST_CATALOG_BACKEND_CLIENT_CONNECTION_TIMEOUT_MS,
"1234");
+
catalogProperties.put(IcebergConstants.REST_CATALOG_BACKEND_CLIENT_SOCKET_TIMEOUT_MS,
"5678");
IcebergConfig icebergConfig =
DynamicIcebergConfigProvider.getIcebergConfigFromCatalogProperties(catalogProperties);
@@ -256,6 +259,20 @@ public class TestDynamicIcebergConfigProvider {
Assertions.assertTrue(icebergConfig.getIcebergCatalogProperties().containsKey("custom-k2"));
Assertions.assertEquals(
icebergConfig.getIcebergCatalogProperties().get("catalog.backend-name"),
"custom_backend");
+ Assertions.assertEquals(
+ 1234,
icebergConfig.get(IcebergConfig.REST_CATALOG_BACKEND_CLIENT_CONNECTION_TIMEOUT_MS));
+ Assertions.assertEquals(
+ 5678,
icebergConfig.get(IcebergConfig.REST_CATALOG_BACKEND_CLIENT_SOCKET_TIMEOUT_MS));
+ Assertions.assertEquals(
+ "1234",
+ icebergConfig
+ .getIcebergCatalogProperties()
+ .get(IcebergConstants.ICEBERG_REST_CLIENT_CONNECTION_TIMEOUT_MS));
+ Assertions.assertEquals(
+ "5678",
+ icebergConfig
+ .getIcebergCatalogProperties()
+ .get(IcebergConstants.ICEBERG_REST_CLIENT_SOCKET_TIMEOUT_MS));
}
@Test
