This is an automated email from the ASF dual-hosted git repository. yuqi1129 pushed a commit to branch improve/job-uri-validation in repository https://gitbox.apache.org/repos/asf/gravitino.git
commit d3dd0a94751e61d4f8529e4c48a3c91c118ae933 Author: yuqi <[email protected]> AuthorDate: Fri Jun 5 11:30:49 2026 +0800 feat: make local remote URI fetch configurable --- .../gravitino/catalog/hive/HiveConstants.java | 2 + .../fileset/FilesetCatalogPropertiesMetadata.java | 12 +++ .../catalog/hive/HiveCatalogOperations.java | 3 + .../hive/HiveCatalogPropertiesMetadata.java | 14 +++ .../catalog/hive/TestHiveCatalogOperations.java | 4 +- .../iceberg/IcebergCatalogPropertiesMetadata.java | 26 ++--- .../paimon/PaimonCatalogPropertiesMetadata.java | 21 ++-- .../authentication/kerberos/FetchFileUtils.java | 11 ++ .../authentication/kerberos/KerberosClient.java | 6 +- .../authentication/kerberos/KerberosConfig.java | 29 +++++ .../catalog/hadoop/fs/kerberos/FetchFileUtils.java | 15 +++ .../catalog/hadoop/fs/kerberos/KerberosClient.java | 7 +- .../catalog/hadoop/fs/kerberos/KerberosConfig.java | 18 ++++ .../gravitino/hive/kerberos/FetchFileUtils.java | 18 ++++ .../gravitino/hive/kerberos/KerberosClient.java | 7 +- .../gravitino/hive/kerberos/KerberosConfig.java | 18 ++++ .../hive/kerberos/TestFetchFileUtils.java | 56 ++++++++++ .../apache/gravitino/utils/RemoteUriValidator.java | 93 ++++++++++++++++ .../gravitino/utils/TestRemoteUriValidator.java | 91 ++++++++++++++++ .../main/java/org/apache/gravitino/Configs.java | 14 +++ .../java/org/apache/gravitino/job/JobManager.java | 117 ++++++++++----------- .../org/apache/gravitino/job/TestJobManager.java | 83 +++++++-------- docs/apache-hive-catalog.md | 28 ++--- docs/fileset-catalog.md | 17 +-- docs/gravitino-server-config.md | 12 +++ docs/iceberg-rest-service.md | 19 ++-- docs/lakehouse-hudi-catalog.md | 17 +-- docs/lakehouse-iceberg-catalog.md | 19 ++-- docs/lakehouse-paimon-catalog.md | 51 ++++----- .../authentication/kerberos/FetchFileUtils.java | 19 ++++ .../authentication/kerberos/KerberosClient.java | 7 +- .../authentication/kerberos/KerberosConfig.java | 29 +++++ 32 files changed, 680 insertions(+), 203 deletions(-) diff --git a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/hive/HiveConstants.java b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/hive/HiveConstants.java index e8459ba2a7..5c1a94c4c8 100644 --- a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/hive/HiveConstants.java +++ b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/hive/HiveConstants.java @@ -30,6 +30,8 @@ public class HiveConstants { public static final String PRINCIPAL = "kerberos.principal"; public static final String CHECK_INTERVAL_SEC = "kerberos.check-interval-sec"; public static final String FETCH_TIMEOUT_SEC = "kerberos.keytab-fetch-timeout-sec"; + public static final String KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS = + "kerberos.keytab-fetch-allow-local-address"; public static final String LIST_ALL_TABLES = "list-all-tables"; // table properties diff --git a/catalogs/catalog-fileset/src/main/java/org/apache/gravitino/catalog/fileset/FilesetCatalogPropertiesMetadata.java b/catalogs/catalog-fileset/src/main/java/org/apache/gravitino/catalog/fileset/FilesetCatalogPropertiesMetadata.java index 57c5a79842..dddff9359b 100644 --- a/catalogs/catalog-fileset/src/main/java/org/apache/gravitino/catalog/fileset/FilesetCatalogPropertiesMetadata.java +++ b/catalogs/catalog-fileset/src/main/java/org/apache/gravitino/catalog/fileset/FilesetCatalogPropertiesMetadata.java @@ -24,6 +24,7 @@ import static org.apache.gravitino.catalog.hadoop.fs.kerberos.AuthenticationConf import static org.apache.gravitino.catalog.hadoop.fs.kerberos.AuthenticationConfig.KERBEROS_DEFAULT_IMPERSONATION_ENABLE; import static org.apache.gravitino.catalog.hadoop.fs.kerberos.KerberosConfig.CHECK_INTERVAL_SEC_KEY; import static org.apache.gravitino.catalog.hadoop.fs.kerberos.KerberosConfig.FETCH_TIMEOUT_SEC_KEY; +import static org.apache.gravitino.catalog.hadoop.fs.kerberos.KerberosConfig.KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_KEY; import static org.apache.gravitino.catalog.hadoop.fs.kerberos.KerberosConfig.KEY_TAB_URI_KEY; import static org.apache.gravitino.catalog.hadoop.fs.kerberos.KerberosConfig.PRINCIPAL_KEY; import static org.apache.gravitino.file.Fileset.LOCATION_NAME_UNKNOWN; @@ -120,6 +121,17 @@ public class FilesetCatalogPropertiesMetadata extends BaseCatalogPropertiesMetad false /* immutable */, 60 /* defaultValue */, false /* hidden */)) + .put( + KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_KEY, + PropertyEntry.booleanPropertyEntry( + KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_KEY, + "Whether to allow Kerberos keytab fetch from local or private addresses from " + + "the Gravitino server side. This is disabled by default to prevent SSRF.", + false /* required */, + false /* immutable */, + false /* defaultValue */, + false /* hidden */, + false /* reserved */)) .build(); public static final Map<String, PropertyEntry<?>> AUTHENTICATION_PROPERTY_ENTRIES = diff --git a/catalogs/catalog-hive/src/main/java/org/apache/gravitino/catalog/hive/HiveCatalogOperations.java b/catalogs/catalog-hive/src/main/java/org/apache/gravitino/catalog/hive/HiveCatalogOperations.java index 87825ca913..e7af12b62c 100644 --- a/catalogs/catalog-hive/src/main/java/org/apache/gravitino/catalog/hive/HiveCatalogOperations.java +++ b/catalogs/catalog-hive/src/main/java/org/apache/gravitino/catalog/hive/HiveCatalogOperations.java @@ -19,6 +19,7 @@ package org.apache.gravitino.catalog.hive; import static org.apache.gravitino.catalog.hive.HiveCatalogPropertiesMetadata.IMPERSONATION_ENABLE; +import static org.apache.gravitino.catalog.hive.HiveCatalogPropertiesMetadata.KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS; import static org.apache.gravitino.catalog.hive.HiveCatalogPropertiesMetadata.KEY_TAB_URI; import static org.apache.gravitino.catalog.hive.HiveCatalogPropertiesMetadata.LIST_ALL_TABLES; import static org.apache.gravitino.catalog.hive.HiveCatalogPropertiesMetadata.METASTORE_URIS; @@ -30,6 +31,7 @@ import static org.apache.gravitino.catalog.hive.TableType.EXTERNAL_TABLE; import static org.apache.gravitino.connector.BaseCatalog.CATALOG_BYPASS_PREFIX; import static org.apache.gravitino.hive.HiveTable.SUPPORT_TABLE_TYPES; import static org.apache.gravitino.hive.kerberos.AuthenticationConfig.IMPERSONATION_ENABLE_KEY; +import static org.apache.gravitino.hive.kerberos.KerberosConfig.KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_KEY; import static org.apache.gravitino.hive.kerberos.KerberosConfig.KEY_TAB_URI_KEY; import static org.apache.gravitino.hive.kerberos.KerberosConfig.PRINCIPAL_KEY; @@ -123,6 +125,7 @@ public class HiveCatalogOperations METASTORE_URIS, HIVE_METASTORE_URIS, IMPERSONATION_ENABLE, IMPERSONATION_ENABLE_KEY, KEY_TAB_URI, KEY_TAB_URI_KEY, + KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS, KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_KEY, PRINCIPAL, PRINCIPAL_KEY); /** diff --git a/catalogs/catalog-hive/src/main/java/org/apache/gravitino/catalog/hive/HiveCatalogPropertiesMetadata.java b/catalogs/catalog-hive/src/main/java/org/apache/gravitino/catalog/hive/HiveCatalogPropertiesMetadata.java index 2577bdf944..6213eb20e8 100644 --- a/catalogs/catalog-hive/src/main/java/org/apache/gravitino/catalog/hive/HiveCatalogPropertiesMetadata.java +++ b/catalogs/catalog-hive/src/main/java/org/apache/gravitino/catalog/hive/HiveCatalogPropertiesMetadata.java @@ -48,6 +48,9 @@ public class HiveCatalogPropertiesMetadata extends BaseCatalogPropertiesMetadata public static final String FETCH_TIMEOUT_SEC = HiveConstants.FETCH_TIMEOUT_SEC; + public static final String KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS = + HiveConstants.KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS; + public static final String LIST_ALL_TABLES = HiveConstants.LIST_ALL_TABLES; public static final boolean DEFAULT_LIST_ALL_TABLES = false; @@ -110,6 +113,17 @@ public class HiveCatalogPropertiesMetadata extends BaseCatalogPropertiesMetadata FETCH_TIMEOUT_SEC, PropertyEntry.integerOptionalPropertyEntry( FETCH_TIMEOUT_SEC, "The timeout to fetch key tab", false, 60, false)) + .put( + KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS, + PropertyEntry.booleanPropertyEntry( + KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS, + "Whether to allow Kerberos keytab fetch from local or private addresses from " + + "the Gravitino server side. This is disabled by default to prevent SSRF.", + false /* required */, + false /* immutable */, + false /* defaultValue */, + false /* hidden */, + false /* reserved */)) .put( LIST_ALL_TABLES, PropertyEntry.booleanPropertyEntry( diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/TestHiveCatalogOperations.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/TestHiveCatalogOperations.java index a8ab3142d9..c091cef4f3 100644 --- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/TestHiveCatalogOperations.java +++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/TestHiveCatalogOperations.java @@ -29,6 +29,7 @@ import static org.apache.gravitino.catalog.hive.HiveCatalogPropertiesMetadata.CL import static org.apache.gravitino.catalog.hive.HiveCatalogPropertiesMetadata.DEFAULT_CATALOG; import static org.apache.gravitino.catalog.hive.HiveCatalogPropertiesMetadata.FETCH_TIMEOUT_SEC; import static org.apache.gravitino.catalog.hive.HiveCatalogPropertiesMetadata.IMPERSONATION_ENABLE; +import static org.apache.gravitino.catalog.hive.HiveCatalogPropertiesMetadata.KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS; import static org.apache.gravitino.catalog.hive.HiveCatalogPropertiesMetadata.KEY_TAB_URI; import static org.apache.gravitino.catalog.hive.HiveCatalogPropertiesMetadata.LIST_ALL_TABLES; import static org.apache.gravitino.catalog.hive.HiveCatalogPropertiesMetadata.METASTORE_URIS; @@ -88,7 +89,7 @@ class TestHiveCatalogOperations { Map<String, PropertyEntry<?>> propertyEntryMap = HIVE_PROPERTIES_METADATA.catalogPropertiesMetadata().propertyEntries(); - Assertions.assertEquals(18, propertyEntryMap.size()); + Assertions.assertEquals(19, propertyEntryMap.size()); Assertions.assertTrue(propertyEntryMap.containsKey(METASTORE_URIS)); Assertions.assertTrue(propertyEntryMap.containsKey(Catalog.PROPERTY_PACKAGE)); Assertions.assertTrue(propertyEntryMap.containsKey(BaseCatalog.CATALOG_OPERATION_IMPL)); @@ -108,6 +109,7 @@ class TestHiveCatalogOperations { Assertions.assertFalse(propertyEntryMap.get(PRINCIPAL).isRequired()); Assertions.assertFalse(propertyEntryMap.get(CHECK_INTERVAL_SEC).isRequired()); Assertions.assertFalse(propertyEntryMap.get(FETCH_TIMEOUT_SEC).isRequired()); + Assertions.assertFalse(propertyEntryMap.get(KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS).isRequired()); Assertions.assertFalse(propertyEntryMap.get(CLOUD_NAME).isRequired()); Assertions.assertFalse(propertyEntryMap.get(CLOUD_NAME).isImmutable()); Assertions.assertFalse(propertyEntryMap.get(CLOUD_REGION_CODE).isRequired()); diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java index 49680a512d..df2312374e 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java @@ -49,19 +49,19 @@ public class IcebergCatalogPropertiesMetadata extends BaseCatalogPropertiesMetad private static final Map<String, PropertyEntry<?>> PROPERTIES_METADATA; public static final Map<String, String> KERBEROS_CONFIGURATION_FOR_HIVE_BACKEND = - ImmutableMap.of( - KerberosConfig.PRINCIPAL_KEY, - KerberosConfig.PRINCIPAL_KEY, - KerberosConfig.KET_TAB_URI_KEY, - KerberosConfig.KET_TAB_URI_KEY, - KerberosConfig.CHECK_INTERVAL_SEC_KEY, - KerberosConfig.CHECK_INTERVAL_SEC_KEY, - KerberosConfig.FETCH_TIMEOUT_SEC_KEY, - KerberosConfig.FETCH_TIMEOUT_SEC_KEY, - AuthenticationConfig.IMPERSONATION_ENABLE_KEY, - AuthenticationConfig.IMPERSONATION_ENABLE_KEY, - AuthenticationConfig.AUTH_TYPE_KEY, - AuthenticationConfig.AUTH_TYPE_KEY); + ImmutableMap.<String, String>builder() + .put(KerberosConfig.PRINCIPAL_KEY, KerberosConfig.PRINCIPAL_KEY) + .put(KerberosConfig.KET_TAB_URI_KEY, KerberosConfig.KET_TAB_URI_KEY) + .put(KerberosConfig.CHECK_INTERVAL_SEC_KEY, KerberosConfig.CHECK_INTERVAL_SEC_KEY) + .put(KerberosConfig.FETCH_TIMEOUT_SEC_KEY, KerberosConfig.FETCH_TIMEOUT_SEC_KEY) + .put( + KerberosConfig.KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_KEY, + KerberosConfig.KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_KEY) + .put( + AuthenticationConfig.IMPERSONATION_ENABLE_KEY, + AuthenticationConfig.IMPERSONATION_ENABLE_KEY) + .put(AuthenticationConfig.AUTH_TYPE_KEY, AuthenticationConfig.AUTH_TYPE_KEY) + .build(); static { List<PropertyEntry<?>> propertyEntries = diff --git a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalogPropertiesMetadata.java b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalogPropertiesMetadata.java index b82061ac36..16c3b153d7 100644 --- a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalogPropertiesMetadata.java +++ b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalogPropertiesMetadata.java @@ -85,17 +85,16 @@ public class PaimonCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada .build(); private static final Map<String, PropertyEntry<?>> PROPERTIES_METADATA; public static final Map<String, String> KERBEROS_CONFIGURATION = - ImmutableMap.of( - KerberosConfig.PRINCIPAL_KEY, - KerberosConfig.PRINCIPAL_KEY, - KerberosConfig.KEY_TAB_URI_KEY, - KerberosConfig.KEY_TAB_URI_KEY, - KerberosConfig.CHECK_INTERVAL_SEC_KEY, - KerberosConfig.CHECK_INTERVAL_SEC_KEY, - KerberosConfig.FETCH_TIMEOUT_SEC_KEY, - KerberosConfig.FETCH_TIMEOUT_SEC_KEY, - AuthenticationConfig.AUTH_TYPE_KEY, - AuthenticationConfig.AUTH_TYPE_KEY); + ImmutableMap.<String, String>builder() + .put(KerberosConfig.PRINCIPAL_KEY, KerberosConfig.PRINCIPAL_KEY) + .put(KerberosConfig.KEY_TAB_URI_KEY, KerberosConfig.KEY_TAB_URI_KEY) + .put(KerberosConfig.CHECK_INTERVAL_SEC_KEY, KerberosConfig.CHECK_INTERVAL_SEC_KEY) + .put(KerberosConfig.FETCH_TIMEOUT_SEC_KEY, KerberosConfig.FETCH_TIMEOUT_SEC_KEY) + .put( + KerberosConfig.KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_KEY, + KerberosConfig.KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_KEY) + .put(AuthenticationConfig.AUTH_TYPE_KEY, AuthenticationConfig.AUTH_TYPE_KEY) + .build(); public static final Map<String, String> S3_CONFIGURATION = ImmutableMap.of( diff --git a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/authentication/kerberos/FetchFileUtils.java b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/authentication/kerberos/FetchFileUtils.java index 29cfdc37c5..03b4aeb8f4 100644 --- a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/authentication/kerberos/FetchFileUtils.java +++ b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/authentication/kerberos/FetchFileUtils.java @@ -25,6 +25,7 @@ import java.net.URISyntaxException; import java.nio.file.Files; import java.util.Optional; import org.apache.commons.io.FileUtils; +import org.apache.gravitino.utils.RemoteUriValidator; public class FetchFileUtils { @@ -32,6 +33,12 @@ public class FetchFileUtils { public static void fetchFileFromUri(String fileUri, File destFile, int timeout) throws IOException { + fetchFileFromUri(fileUri, destFile, timeout, false /* allowLocalAddressForRemoteUri */); + } + + public static void fetchFileFromUri( + String fileUri, File destFile, int timeout, boolean allowLocalAddressForRemoteUri) + throws IOException { try { URI uri = new URI(fileUri); String scheme = Optional.ofNullable(uri.getScheme()).orElse("file"); @@ -40,6 +47,10 @@ public class FetchFileUtils { case "http": case "https": case "ftp": + RemoteUriValidator.validate( + uri, + allowLocalAddressForRemoteUri, + String.format("'%s' to true", KerberosConfig.KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_KEY)); FileUtils.copyURLToFile(uri.toURL(), destFile, timeout * 1000, timeout * 1000); break; diff --git a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/authentication/kerberos/KerberosClient.java b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/authentication/kerberos/KerberosClient.java index d8ae6c6438..c8bb7a3c18 100644 --- a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/authentication/kerberos/KerberosClient.java +++ b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/authentication/kerberos/KerberosClient.java @@ -100,7 +100,11 @@ public class KerberosClient implements Closeable { } int fetchKeytabFileTimeout = kerberosConfig.getFetchTimeoutSec(); - FetchFileUtils.fetchFileFromUri(keyTabUri, keytabFile, fetchKeytabFileTimeout); + FetchFileUtils.fetchFileFromUri( + keyTabUri, + keytabFile, + fetchKeytabFileTimeout, + kerberosConfig.allowKeytabFetchLocalAddress()); return keytabFile; } diff --git a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/authentication/kerberos/KerberosConfig.java b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/authentication/kerberos/KerberosConfig.java index 93f46f7270..d99078a738 100644 --- a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/authentication/kerberos/KerberosConfig.java +++ b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/authentication/kerberos/KerberosConfig.java @@ -37,6 +37,9 @@ public class KerberosConfig extends AuthenticationConfig { public static final String FETCH_TIMEOUT_SEC_KEY = "authentication.kerberos.keytab-fetch-timeout-sec"; + public static final String KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_KEY = + "authentication.kerberos.keytab-fetch-allow-local-address"; + public static final ConfigEntry<String> PRINCIPAL_ENTRY = new ConfigBuilder(PRINCIPAL_KEY) .doc("The principal of the Kerberos connection") @@ -69,6 +72,17 @@ public class KerberosConfig extends AuthenticationConfig { .checkValue(value -> value > 0, ConfigConstants.POSITIVE_NUMBER_ERROR_MSG) .createWithDefault(2); + public static final ConfigEntry<Boolean> KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_ENTRY = + new ConfigBuilder(KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_KEY) + .doc( + "Whether to allow the Kerberos keytab URI to resolve to local, private, link-local, " + + "or cloud metadata addresses from the Gravitino server side. This is disabled " + + "by default to prevent SSRF. Set it to true only when the URI is trusted and " + + "must be fetched from local or private addresses.") + .version(ConfigConstants.VERSION_1_3_0) + .booleanConf() + .createWithDefault(false); + public KerberosConfig(Map<String, String> properties) { super(properties); loadFromMap(properties, k -> true); @@ -90,6 +104,10 @@ public class KerberosConfig extends AuthenticationConfig { return get(FETCH_TIMEOUT_SEC_ENTRY); } + public boolean allowKeytabFetchLocalAddress() { + return get(KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_ENTRY); + } + public static final Map<String, PropertyEntry<?>> KERBEROS_PROPERTY_ENTRIES = new ImmutableMap.Builder<String, PropertyEntry<?>>() .put( @@ -124,5 +142,16 @@ public class KerberosConfig extends AuthenticationConfig { false /* immutable */, 60 /* defaultValue */, false /* hidden */)) + .put( + KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_KEY, + PropertyEntry.booleanPropertyEntry( + KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_KEY, + "Whether to allow Kerberos keytab fetch from local or private addresses from " + + "the Gravitino server side. This is disabled by default to prevent SSRF.", + false /* required */, + false /* immutable */, + false /* defaultValue */, + false /* hidden */, + false /* reserved */)) .build(); } diff --git a/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/kerberos/FetchFileUtils.java b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/kerberos/FetchFileUtils.java index d014345c54..6d9fae3cd0 100644 --- a/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/kerberos/FetchFileUtils.java +++ b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/kerberos/FetchFileUtils.java @@ -25,6 +25,7 @@ import java.net.URISyntaxException; import java.nio.file.Files; import java.util.Optional; import org.apache.commons.io.FileUtils; +import org.apache.gravitino.utils.RemoteUriValidator; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -35,6 +36,16 @@ public class FetchFileUtils { public static void fetchFileFromUri( String fileUri, File destFile, int timeout, Configuration conf) throws IOException { + fetchFileFromUri(fileUri, destFile, timeout, conf, false /* allowLocalAddressForRemoteUri */); + } + + public static void fetchFileFromUri( + String fileUri, + File destFile, + int timeout, + Configuration conf, + boolean allowLocalAddressForRemoteUri) + throws IOException { try { URI uri = new URI(fileUri); String scheme = Optional.ofNullable(uri.getScheme()).orElse("file"); @@ -43,6 +54,10 @@ public class FetchFileUtils { case "http": case "https": case "ftp": + RemoteUriValidator.validate( + uri, + allowLocalAddressForRemoteUri, + String.format("'%s' to true", KerberosConfig.KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_KEY)); FileUtils.copyURLToFile(uri.toURL(), destFile, timeout * 1000, timeout * 1000); break; diff --git a/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/kerberos/KerberosClient.java b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/kerberos/KerberosClient.java index eb93f2ab72..3cef659bc2 100644 --- a/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/kerberos/KerberosClient.java +++ b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/kerberos/KerberosClient.java @@ -122,7 +122,12 @@ public class KerberosClient implements Closeable { } int fetchKeytabFileTimeout = kerberosConfig.getFetchTimeoutSec(); - FetchFileUtils.fetchFileFromUri(keyTabUri, keytabFile, fetchKeytabFileTimeout, hadoopConf); + FetchFileUtils.fetchFileFromUri( + keyTabUri, + keytabFile, + fetchKeytabFileTimeout, + hadoopConf, + kerberosConfig.allowKeytabFetchLocalAddress()); return keytabFile; } diff --git a/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/kerberos/KerberosConfig.java b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/kerberos/KerberosConfig.java index 146fcf6324..eb2f2c4bef 100644 --- a/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/kerberos/KerberosConfig.java +++ b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/kerberos/KerberosConfig.java @@ -39,6 +39,9 @@ public class KerberosConfig extends AuthenticationConfig { public static final String FETCH_TIMEOUT_SEC_KEY = "authentication.kerberos.keytab-fetch-timeout-sec"; + public static final String KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_KEY = + "authentication.kerberos.keytab-fetch-allow-local-address"; + public static final ConfigEntry<String> PRINCIPAL_ENTRY = new ConfigBuilder(PRINCIPAL_KEY) .doc("The principal of the Kerberos connection") @@ -71,6 +74,17 @@ public class KerberosConfig extends AuthenticationConfig { .checkValue(value -> value > 0, ConfigConstants.POSITIVE_NUMBER_ERROR_MSG) .createWithDefault(2); + public static final ConfigEntry<Boolean> KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_ENTRY = + new ConfigBuilder(KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_KEY) + .doc( + "Whether to allow the Kerberos keytab URI to resolve to local, private, link-local, " + + "or cloud metadata addresses from the Gravitino server side. This is disabled " + + "by default to prevent SSRF. Set it to true only when the URI is trusted and " + + "must be fetched from local or private addresses.") + .version(ConfigConstants.VERSION_1_3_0) + .booleanConf() + .createWithDefault(false); + public KerberosConfig(Map<String, String> properties, Configuration configuration) { super(properties, configuration); loadFromHdfsConfiguration(configuration); @@ -99,4 +113,8 @@ public class KerberosConfig extends AuthenticationConfig { public int getFetchTimeoutSec() { return get(FETCH_TIMEOUT_SEC_ENTRY); } + + public boolean allowKeytabFetchLocalAddress() { + return get(KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_ENTRY); + } } diff --git a/catalogs/hive-metastore-common/src/main/java/org/apache/gravitino/hive/kerberos/FetchFileUtils.java b/catalogs/hive-metastore-common/src/main/java/org/apache/gravitino/hive/kerberos/FetchFileUtils.java index 7149e1c82d..543ca61549 100644 --- a/catalogs/hive-metastore-common/src/main/java/org/apache/gravitino/hive/kerberos/FetchFileUtils.java +++ b/catalogs/hive-metastore-common/src/main/java/org/apache/gravitino/hive/kerberos/FetchFileUtils.java @@ -27,6 +27,7 @@ import java.nio.file.StandardCopyOption; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import org.apache.commons.io.FileUtils; +import org.apache.gravitino.utils.RemoteUriValidator; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -55,6 +56,16 @@ public class FetchFileUtils { public static void fetchFileFromUri( String fileUri, File destFile, int timeout, Configuration conf) throws IOException { + fetchFileFromUri(fileUri, destFile, timeout, conf, false /* allowLocalAddressForRemoteUri */); + } + + public static void fetchFileFromUri( + String fileUri, + File destFile, + int timeout, + Configuration conf, + boolean allowLocalAddressForRemoteUri) + throws IOException { try { URI uri = new URI(fileUri); String scheme = Optional.ofNullable(uri.getScheme()).orElse("file"); @@ -63,6 +74,13 @@ public class FetchFileUtils { case "http": case "https": case "ftp": + RemoteUriValidator.validate( + uri, + allowLocalAddressForRemoteUri, + String.format( + "'%s' to true, or set 'kerberos.keytab-fetch-allow-local-address' to true for " + + "Hive catalog properties", + KerberosConfig.KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_KEY)); FileUtils.copyURLToFile(uri.toURL(), destFile, timeout * 1000, timeout * 1000); break; diff --git a/catalogs/hive-metastore-common/src/main/java/org/apache/gravitino/hive/kerberos/KerberosClient.java b/catalogs/hive-metastore-common/src/main/java/org/apache/gravitino/hive/kerberos/KerberosClient.java index cb115048e4..ca90d4cdf8 100644 --- a/catalogs/hive-metastore-common/src/main/java/org/apache/gravitino/hive/kerberos/KerberosClient.java +++ b/catalogs/hive-metastore-common/src/main/java/org/apache/gravitino/hive/kerberos/KerberosClient.java @@ -173,7 +173,12 @@ public class KerberosClient implements java.io.Closeable { } File keytabFile = new File(path); int fetchKeytabFileTimeout = kerberosConfig.getFetchTimeoutSec(); - FetchFileUtils.fetchFileFromUri(keyTabUri, keytabFile, fetchKeytabFileTimeout, hadoopConf); + FetchFileUtils.fetchFileFromUri( + keyTabUri, + keytabFile, + fetchKeytabFileTimeout, + hadoopConf, + kerberosConfig.allowKeytabFetchLocalAddress()); return keytabFile; } diff --git a/catalogs/hive-metastore-common/src/main/java/org/apache/gravitino/hive/kerberos/KerberosConfig.java b/catalogs/hive-metastore-common/src/main/java/org/apache/gravitino/hive/kerberos/KerberosConfig.java index 4495e0d7fc..f1567534b5 100644 --- a/catalogs/hive-metastore-common/src/main/java/org/apache/gravitino/hive/kerberos/KerberosConfig.java +++ b/catalogs/hive-metastore-common/src/main/java/org/apache/gravitino/hive/kerberos/KerberosConfig.java @@ -38,6 +38,9 @@ public class KerberosConfig extends AuthenticationConfig { public static final String FETCH_TIMEOUT_SEC_KEY = "authentication.kerberos.keytab-fetch-timeout-sec"; + public static final String KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_KEY = + "authentication.kerberos.keytab-fetch-allow-local-address"; + public static final ConfigEntry<String> PRINCIPAL_ENTRY = new ConfigBuilder(PRINCIPAL_KEY) .doc("The principal of the Kerberos connection") @@ -70,6 +73,17 @@ public class KerberosConfig extends AuthenticationConfig { .checkValue(value -> value > 0, ConfigConstants.POSITIVE_NUMBER_ERROR_MSG) .createWithDefault(60); + public static final ConfigEntry<Boolean> KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_ENTRY = + new ConfigBuilder(KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_KEY) + .doc( + "Whether to allow the Kerberos keytab URI to resolve to local, private, link-local, " + + "or cloud metadata addresses from the Gravitino server side. This is disabled " + + "by default to prevent SSRF. Set it to true only when the URI is trusted and " + + "must be fetched from local or private addresses.") + .version(ConfigConstants.VERSION_1_3_0) + .booleanConf() + .createWithDefault(false); + public KerberosConfig(Properties properties, Configuration configuration) { super(properties, configuration); loadFromMap((Map) properties, k -> true); @@ -90,4 +104,8 @@ public class KerberosConfig extends AuthenticationConfig { public int getFetchTimeoutSec() { return get(FETCH_TIMEOUT_SEC_ENTRY); } + + public boolean allowKeytabFetchLocalAddress() { + return get(KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_ENTRY); + } } diff --git a/catalogs/hive-metastore-common/src/test/java/org/apache/gravitino/hive/kerberos/TestFetchFileUtils.java b/catalogs/hive-metastore-common/src/test/java/org/apache/gravitino/hive/kerberos/TestFetchFileUtils.java index c9a4bbec28..eb5900b23a 100644 --- a/catalogs/hive-metastore-common/src/test/java/org/apache/gravitino/hive/kerberos/TestFetchFileUtils.java +++ b/catalogs/hive-metastore-common/src/test/java/org/apache/gravitino/hive/kerberos/TestFetchFileUtils.java @@ -18,7 +18,11 @@ */ package org.apache.gravitino.hive.kerberos; +import com.sun.net.httpserver.HttpServer; import java.io.File; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.ArrayList; import java.util.List; @@ -116,4 +120,56 @@ public class TestFetchFileUtils { FetchFileUtils.fetchFileFromUri(srcFileB.toURI().toString(), destFile, 10, conf); Assertions.assertEquals(srcFileB.toPath(), Files.readSymbolicLink(destFile.toPath())); } + + @Test + public void testRemoteFetchShouldBlockLocalhostByDefault() { + File destFile = new File(tempDir, "blocked"); + + IllegalArgumentException exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + FetchFileUtils.fetchFileFromUri( + "http://127.0.0.1:8090/keytab";, destFile, 10, new Configuration())); + + Assertions.assertTrue(exception.getMessage().contains("Gravitino server side")); + Assertions.assertTrue( + exception.getMessage().contains(KerberosConfig.KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_KEY)); + } + + @Test + public void testRemoteFetchShouldAllowLocalhostWhenConfigured() throws Exception { + File destFile = new File(tempDir, "allowed"); + HttpServer server = createLoopbackHttpServer("keytab"); + + try { + server.start(); + int port = server.getAddress().getPort(); + + FetchFileUtils.fetchFileFromUri( + String.format("http://127.0.0.1:%d/keytab";, port), + destFile, + 10, + new Configuration(), + true); + + Assertions.assertEquals("keytab", Files.readString(destFile.toPath())); + } finally { + server.stop(0); + } + } + + private HttpServer createLoopbackHttpServer(String response) throws Exception { + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext( + "/keytab", + exchange -> { + byte[] bytes = response.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, bytes.length); + try (OutputStream outputStream = exchange.getResponseBody()) { + outputStream.write(bytes); + } + }); + return server; + } } diff --git a/common/src/main/java/org/apache/gravitino/utils/RemoteUriValidator.java b/common/src/main/java/org/apache/gravitino/utils/RemoteUriValidator.java new file mode 100644 index 0000000000..233720d363 --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/utils/RemoteUriValidator.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.utils; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.URI; + +/** Validates remote URI hosts before server-side downloads. */ +public final class RemoteUriValidator { + + private RemoteUriValidator() {} + + /** + * Resolves the host in the given URI and rejects local or private addresses unless explicitly + * allowed. + * + * @param uri The remote URI to validate. + * @param allowLocalAddress Whether local and private addresses are allowed. + * @param allowLocalAddressHint The configuration hint that enables local and private addresses. + * @throws IOException If host resolution fails. + * @throws IllegalArgumentException If the URI has no host or resolves to a blocked address. + */ + public static void validate(URI uri, boolean allowLocalAddress, String allowLocalAddressHint) + throws IOException { + String host = uri.getHost(); + if (host == null) { + throw new IllegalArgumentException("URI has no host: " + uri); + } + + if (allowLocalAddress) { + return; + } + + InetAddress[] addresses = InetAddress.getAllByName(host); + for (InetAddress address : addresses) { + if (isBlockedAddress(address)) { + throw new IllegalArgumentException( + String.format( + "URI '%s' resolves to blocked address %s from the Gravitino server side. " + + "Access to local, private, link-local, multicast, unspecified, and cloud " + + "metadata addresses is disabled by default to prevent SSRF. If this URI is " + + "trusted and this access is required, set %s.", + uri, address.getHostAddress(), allowLocalAddressHint)); + } + } + } + + private static boolean isBlockedAddress(InetAddress address) { + if (address.isLoopbackAddress() + || address.isLinkLocalAddress() + || address.isSiteLocalAddress() + || address.isMulticastAddress() + || address.isAnyLocalAddress()) { + return true; + } + + byte[] bytes = address.getAddress(); + if (isCloudMetadataAddress(bytes)) { + return true; + } + + return isIpv6UniqueLocalAddress(bytes); + } + + private static boolean isCloudMetadataAddress(byte[] bytes) { + return bytes.length == 4 + && (bytes[0] & 0xFF) == 100 + && (bytes[1] & 0xFF) == 100 + && (bytes[2] & 0xFF) == 100 + && (bytes[3] & 0xFF) == 200; + } + + private static boolean isIpv6UniqueLocalAddress(byte[] bytes) { + return bytes.length == 16 && ((bytes[0] & 0xFE) == 0xFC); + } +} diff --git a/common/src/test/java/org/apache/gravitino/utils/TestRemoteUriValidator.java b/common/src/test/java/org/apache/gravitino/utils/TestRemoteUriValidator.java new file mode 100644 index 0000000000..5a250aed6b --- /dev/null +++ b/common/src/test/java/org/apache/gravitino/utils/TestRemoteUriValidator.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.utils; + +import java.net.URI; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestRemoteUriValidator { + private static final String ALLOW_LOCAL_ADDRESS_CONFIG = "test.allow-local-address"; + + @Test + public void testRejectLocalAddressesByDefault() { + IllegalArgumentException exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + RemoteUriValidator.validate( + new URI("http://127.0.0.1/";), false, ALLOW_LOCAL_ADDRESS_CONFIG)); + Assertions.assertTrue(exception.getMessage().contains("Gravitino server side")); + Assertions.assertTrue(exception.getMessage().contains(ALLOW_LOCAL_ADDRESS_CONFIG)); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + RemoteUriValidator.validate( + new URI("http://localhost/";), false, ALLOW_LOCAL_ADDRESS_CONFIG)); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + RemoteUriValidator.validate( + new URI("http://169.254.169.254/";), false, ALLOW_LOCAL_ADDRESS_CONFIG)); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + RemoteUriValidator.validate( + new URI("http://10.0.0.1/";), false, ALLOW_LOCAL_ADDRESS_CONFIG)); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + RemoteUriValidator.validate( + new URI("http://172.16.0.1/";), false, ALLOW_LOCAL_ADDRESS_CONFIG)); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + RemoteUriValidator.validate( + new URI("http://192.168.0.1/";), false, ALLOW_LOCAL_ADDRESS_CONFIG)); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + RemoteUriValidator.validate( + new URI("http://100.100.100.200/";), false, ALLOW_LOCAL_ADDRESS_CONFIG)); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + RemoteUriValidator.validate( + new URI("http://[fd00::1]/";), false, ALLOW_LOCAL_ADDRESS_CONFIG)); + } + + @Test + public void testAllowLocalAddressesWhenConfigured() { + Assertions.assertDoesNotThrow( + () -> + RemoteUriValidator.validate( + new URI("http://127.0.0.1/";), true, ALLOW_LOCAL_ADDRESS_CONFIG)); + Assertions.assertDoesNotThrow( + () -> + RemoteUriValidator.validate( + new URI("http://localhost/";), true, ALLOW_LOCAL_ADDRESS_CONFIG)); + Assertions.assertDoesNotThrow( + () -> + RemoteUriValidator.validate( + new URI("http://192.168.0.1/";), true, ALLOW_LOCAL_ADDRESS_CONFIG)); + } +} diff --git a/core/src/main/java/org/apache/gravitino/Configs.java b/core/src/main/java/org/apache/gravitino/Configs.java index 516c2c9106..bd3a7578cf 100644 --- a/core/src/main/java/org/apache/gravitino/Configs.java +++ b/core/src/main/java/org/apache/gravitino/Configs.java @@ -523,6 +523,20 @@ public class Configs { .checkValue(value -> value > 0, ConfigConstants.POSITIVE_NUMBER_ERROR_MSG) .createWithDefault(5 * 60 * 1000L); // Default is 5 minutes + public static final String JOB_REMOTE_URI_ALLOW_LOCAL_ADDRESS_KEY = + "gravitino.job.remoteUri.allowLocalAddress"; + + public static final ConfigEntry<Boolean> JOB_REMOTE_URI_ALLOW_LOCAL_ADDRESS = + new ConfigBuilder(JOB_REMOTE_URI_ALLOW_LOCAL_ADDRESS_KEY) + .doc( + "Whether to allow job file remote URIs to resolve to local, private, link-local, " + + "or cloud metadata addresses from the Gravitino server side. This is disabled " + + "by default to prevent SSRF. Set it to true only when the URI is trusted and " + + "must be fetched from local or private addresses.") + .version(ConfigConstants.VERSION_1_3_0) + .booleanConf() + .createWithDefault(false); + public static final ConfigEntry<String> SCHEMA_SEPARATOR = new ConfigBuilder("gravitino.schema.separator") .doc( diff --git a/core/src/main/java/org/apache/gravitino/job/JobManager.java b/core/src/main/java/org/apache/gravitino/job/JobManager.java index 2be112fd25..7c7a846073 100644 --- a/core/src/main/java/org/apache/gravitino/job/JobManager.java +++ b/core/src/main/java/org/apache/gravitino/job/JobManager.java @@ -25,7 +25,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import java.io.File; import java.io.IOException; -import java.net.InetAddress; import java.net.URI; import java.nio.file.Files; import java.time.Instant; @@ -65,6 +64,7 @@ import org.apache.gravitino.storage.IdGenerator; import org.apache.gravitino.utils.NameIdentifierUtil; import org.apache.gravitino.utils.NamespaceUtil; import org.apache.gravitino.utils.PrincipalUtils; +import org.apache.gravitino.utils.RemoteUriValidator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -99,6 +99,8 @@ public class JobManager implements JobOperationDispatcher { private final long jobStagingDirKeepTimeInMs; + private final boolean allowLocalAddressForRemoteUri; + @VisibleForTesting final ScheduledExecutorService cleanUpExecutor; @VisibleForTesting final ScheduledExecutorService statusPullExecutor; @@ -134,6 +136,7 @@ public class JobManager implements JobOperationDispatcher { } this.jobStagingDirKeepTimeInMs = config.get(Configs.JOB_STAGING_DIR_KEEP_TIME_IN_MS); + this.allowLocalAddressForRemoteUri = config.get(Configs.JOB_REMOTE_URI_ALLOW_LOCAL_ADDRESS); if (jobStagingDirKeepTimeInMs < JOB_STAGING_DIR_CLEANUP_MIN_TIME_IN_MS) { LOG.warn( "The job staging directory keep time is set to {} ms, the number is too small, " @@ -436,7 +439,9 @@ public class JobManager implements JobOperationDispatcher { // Create a JobTemplate by replacing the template parameters with the jobConf values, and // also downloading any necessary files from the URIs specified in the job template. - JobTemplate jobTemplate = createRuntimeJobTemplate(jobTemplateEntity, jobConf, jobStagingDir); + JobTemplate jobTemplate = + createRuntimeJobTemplate( + jobTemplateEntity, jobConf, jobStagingDir, allowLocalAddressForRemoteUri); // Submit the job template to the job executor String jobExecutionId; @@ -670,13 +675,26 @@ public class JobManager implements JobOperationDispatcher { @VisibleForTesting public static JobTemplate createRuntimeJobTemplate( JobTemplateEntity jobTemplateEntity, Map<String, String> jobConf, File stagingDir) { + return createRuntimeJobTemplate( + jobTemplateEntity, jobConf, stagingDir, false /* allowLocalAddressForRemoteUri */); + } + + @VisibleForTesting + static JobTemplate createRuntimeJobTemplate( + JobTemplateEntity jobTemplateEntity, + Map<String, String> jobConf, + File stagingDir, + boolean allowLocalAddressForRemoteUri) { String name = jobTemplateEntity.name(); String comment = jobTemplateEntity.comment(); JobTemplateEntity.TemplateContent content = jobTemplateEntity.templateContent(); String executable = fetchFileFromUri( - replacePlaceholder(content.executable(), jobConf), stagingDir, TIMEOUT_IN_MS); + replacePlaceholder(content.executable(), jobConf), + stagingDir, + TIMEOUT_IN_MS, + allowLocalAddressForRemoteUri); List<String> args = content.arguments().stream() @@ -702,7 +720,10 @@ public class JobManager implements JobOperationDispatcher { .map( script -> fetchFileFromUri( - replacePlaceholder(script, jobConf), stagingDir, TIMEOUT_IN_MS)) + replacePlaceholder(script, jobConf), + stagingDir, + TIMEOUT_IN_MS, + allowLocalAddressForRemoteUri)) .collect(Collectors.toList()); return ShellJobTemplate.builder() @@ -723,7 +744,11 @@ public class JobManager implements JobOperationDispatcher { content.jars().stream() .map( jar -> - fetchFileFromUri(replacePlaceholder(jar, jobConf), stagingDir, TIMEOUT_IN_MS)) + fetchFileFromUri( + replacePlaceholder(jar, jobConf), + stagingDir, + TIMEOUT_IN_MS, + allowLocalAddressForRemoteUri)) .collect(Collectors.toList()); List<String> files = @@ -731,7 +756,10 @@ public class JobManager implements JobOperationDispatcher { .map( file -> fetchFileFromUri( - replacePlaceholder(file, jobConf), stagingDir, TIMEOUT_IN_MS)) + replacePlaceholder(file, jobConf), + stagingDir, + TIMEOUT_IN_MS, + allowLocalAddressForRemoteUri)) .collect(Collectors.toList()); List<String> archives = @@ -739,7 +767,10 @@ public class JobManager implements JobOperationDispatcher { .map( archive -> fetchFileFromUri( - replacePlaceholder(archive, jobConf), stagingDir, TIMEOUT_IN_MS)) + replacePlaceholder(archive, jobConf), + stagingDir, + TIMEOUT_IN_MS, + allowLocalAddressForRemoteUri)) .collect(Collectors.toList()); Map<String, String> configs = @@ -793,13 +824,27 @@ public class JobManager implements JobOperationDispatcher { @VisibleForTesting static List<String> fetchFilesFromUri(List<String> uris, File stagingDir, int timeoutInMs) { + return fetchFilesFromUri( + uris, stagingDir, timeoutInMs, false /* allowLocalAddressForRemoteUri */); + } + + @VisibleForTesting + static List<String> fetchFilesFromUri( + List<String> uris, File stagingDir, int timeoutInMs, boolean allowLocalAddressForRemoteUri) { return uris.stream() - .map(uri -> fetchFileFromUri(uri, stagingDir, timeoutInMs)) + .map(uri -> fetchFileFromUri(uri, stagingDir, timeoutInMs, allowLocalAddressForRemoteUri)) .collect(Collectors.toList()); } @VisibleForTesting static String fetchFileFromUri(String uri, File stagingDir, int timeoutInMs) { + return fetchFileFromUri( + uri, stagingDir, timeoutInMs, false /* allowLocalAddressForRemoteUri */); + } + + @VisibleForTesting + static String fetchFileFromUri( + String uri, File stagingDir, int timeoutInMs, boolean allowLocalAddressForRemoteUri) { try { URI fileUri = new URI(uri); String scheme = Optional.ofNullable(fileUri.getScheme()).orElse("file"); @@ -809,7 +854,10 @@ public class JobManager implements JobOperationDispatcher { case "http": case "https": case "ftp": - validateRemoteUri(fileUri); + RemoteUriValidator.validate( + fileUri, + allowLocalAddressForRemoteUri, + String.format("'%s' to true", Configs.JOB_REMOTE_URI_ALLOW_LOCAL_ADDRESS_KEY)); FileUtils.copyURLToFile(fileUri.toURL(), destFile, timeoutInMs, timeoutInMs); break; @@ -832,57 +880,6 @@ public class JobManager implements JobOperationDispatcher { } } - /** - * Resolves the host in the given URI and rejects addresses that should not be reachable from the - * server (loopback, link-local, RFC-1918 private ranges, IPv6 ULA, cloud metadata endpoints). - * This is a defence-in-depth measure against Server-Side Request Forgery (SSRF). - * - * <p><b>Note on DNS TOCTOU:</b> This method resolves the hostname once at validation time. {@code - * FileUtils.copyURLToFile()} will re-resolve it when opening the connection, so a precisely-timed - * DNS-rebinding attack could theoretically bypass this check. Complete protection against DNS - * rebinding requires network-level egress controls (e.g. firewall rules) in addition to this - * application-layer validation. - */ - @VisibleForTesting - static void validateRemoteUri(URI uri) throws IOException { - String host = uri.getHost(); - if (host == null) { - throw new IllegalArgumentException("URI has no host: " + uri); - } - InetAddress[] addresses = InetAddress.getAllByName(host); - for (InetAddress address : addresses) { - if (isBlockedAddress(address)) { - throw new IllegalArgumentException( - String.format( - "URI '%s' resolves to blocked address %s, access denied (SSRF prevention)", - uri, address.getHostAddress())); - } - } - } - - private static boolean isBlockedAddress(InetAddress address) { - // Covers loopback (127.x.x.x / ::1), link-local (169.254.x.x / fe80::/10 — includes AWS/GCP/ - // Azure metadata), RFC-1918 private (10.x / 172.16-31.x / 192.168.x), multicast, unspecified. - if (address.isLoopbackAddress() - || address.isLinkLocalAddress() - || address.isSiteLocalAddress() - || address.isMulticastAddress() - || address.isAnyLocalAddress()) { - return true; - } - byte[] b = address.getAddress(); - // Alibaba Cloud / Oracle Cloud metadata endpoint: 100.100.100.200 - if (b.length == 4 - && (b[0] & 0xFF) == 100 - && (b[1] & 0xFF) == 100 - && (b[2] & 0xFF) == 100 - && (b[3] & 0xFF) == 200) { - return true; - } - // IPv6 Unique Local Addresses (RFC 4193): fc00::/7 — not covered by isSiteLocalAddress() - return b.length == 16 && ((b[0] & 0xFE) == 0xFC); - } - @VisibleForTesting JobTemplateEntity updateJobTemplateEntity( NameIdentifier jobTemplateIdent, diff --git a/core/src/test/java/org/apache/gravitino/job/TestJobManager.java b/core/src/test/java/org/apache/gravitino/job/TestJobManager.java index 692d70d010..7e8d693f93 100644 --- a/core/src/test/java/org/apache/gravitino/job/TestJobManager.java +++ b/core/src/test/java/org/apache/gravitino/job/TestJobManager.java @@ -31,9 +31,13 @@ import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import com.sun.net.httpserver.HttpServer; import java.io.File; import java.io.IOException; -import java.net.URI; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.nio.file.Path; import java.time.Instant; import java.util.Collections; @@ -902,6 +906,20 @@ public class TestJobManager { .build(); } + private HttpServer createLoopbackHttpServer(String response) throws IOException { + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext( + "/artifact.jar", + exchange -> { + byte[] bytes = response.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, bytes.length); + try (OutputStream outputStream = exchange.getResponseBody()) { + outputStream.write(bytes); + } + }); + return server; + } + @Test public void testFetchFileFromUriWithMissingLocalFileShouldFail() throws IOException { File stagingDir = new File(testStagingDir); @@ -925,7 +943,7 @@ public class TestJobManager { Assertions.assertThrows( RuntimeException.class, () -> JobManager.fetchFileFromUri("http://127.0.0.1:8090/configs";, stagingDir, 1000)); - Assertions.assertTrue(e1.getCause().getMessage().contains("SSRF prevention")); + assertRemoteUriBlockedMessage(e1); // AWS / GCP / Azure cloud-metadata endpoint (link-local 169.254.x.x) RuntimeException e2 = @@ -934,64 +952,47 @@ public class TestJobManager { () -> JobManager.fetchFileFromUri( "http://169.254.169.254/latest/meta-data/";, stagingDir, 1000)); - Assertions.assertTrue(e2.getCause().getMessage().contains("SSRF prevention")); + assertRemoteUriBlockedMessage(e2); // RFC-1918 private range RuntimeException e3 = Assertions.assertThrows( RuntimeException.class, () -> JobManager.fetchFileFromUri("http://192.168.1.1/";, stagingDir, 1000)); - Assertions.assertTrue(e3.getCause().getMessage().contains("SSRF prevention")); + assertRemoteUriBlockedMessage(e3); // Alibaba Cloud / Oracle Cloud metadata endpoint RuntimeException e4 = Assertions.assertThrows( RuntimeException.class, () -> JobManager.fetchFileFromUri("http://100.100.100.200/";, stagingDir, 1000)); - Assertions.assertTrue(e4.getCause().getMessage().contains("SSRF prevention")); + assertRemoteUriBlockedMessage(e4); } @Test - public void testValidateRemoteUri() throws Exception { - // Loopback - Assertions.assertThrows( - IllegalArgumentException.class, - () -> JobManager.validateRemoteUri(new URI("http://127.0.0.1/";))); - - // Link-local (cloud metadata) - Assertions.assertThrows( - IllegalArgumentException.class, - () -> JobManager.validateRemoteUri(new URI("http://169.254.169.254/";))); - - // RFC-1918 private 10.x.x.x - Assertions.assertThrows( - IllegalArgumentException.class, - () -> JobManager.validateRemoteUri(new URI("http://10.0.0.1/";))); - - // RFC-1918 private 172.16.x.x - Assertions.assertThrows( - IllegalArgumentException.class, - () -> JobManager.validateRemoteUri(new URI("http://172.16.0.1/";))); + public void testFetchFileFromUriShouldAllowLocalhostWhenConfigured() throws Exception { + File stagingDir = new File(testStagingDir); + Assertions.assertTrue(stagingDir.mkdirs() || stagingDir.exists()); + HttpServer server = createLoopbackHttpServer("job artifact"); - // RFC-1918 private 192.168.x.x - Assertions.assertThrows( - IllegalArgumentException.class, - () -> JobManager.validateRemoteUri(new URI("http://192.168.0.1/";))); + try { + server.start(); + int port = server.getAddress().getPort(); - // Alibaba Cloud metadata - Assertions.assertThrows( - IllegalArgumentException.class, - () -> JobManager.validateRemoteUri(new URI("http://100.100.100.200/";))); + String fetchedFile = + JobManager.fetchFileFromUri( + String.format("http://127.0.0.1:%d/artifact.jar";, port), stagingDir, 1000, true); - // localhost resolves to loopback - Assertions.assertThrows( - IllegalArgumentException.class, - () -> JobManager.validateRemoteUri(new URI("http://localhost/";))); + Assertions.assertEquals("job artifact", Files.readString(Path.of(fetchedFile))); + } finally { + server.stop(0); + } + } - // IPv6 Unique Local Address (RFC 4193): fc00::/7 - Assertions.assertThrows( - IllegalArgumentException.class, - () -> JobManager.validateRemoteUri(new URI("http://[fd00::1]/";))); + private static void assertRemoteUriBlockedMessage(RuntimeException exception) { + Assertions.assertTrue(exception.getCause().getMessage().contains("Gravitino server side")); + Assertions.assertTrue( + exception.getCause().getMessage().contains(Configs.JOB_REMOTE_URI_ALLOW_LOCAL_ADDRESS_KEY)); } @Test diff --git a/docs/apache-hive-catalog.md b/docs/apache-hive-catalog.md index f27fec46ab..32f9a633b9 100644 --- a/docs/apache-hive-catalog.md +++ b/docs/apache-hive-catalog.md @@ -29,19 +29,20 @@ The Hive catalog supports creating, updating, and deleting databases and tables Besides the [common catalog properties](./gravitino-server-config.md#apache-gravitino-catalog-properties-configuration), the Hive catalog has the following properties: -| Property Name | Description | Default Value | Required | Since Version | -|------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|------------------------------|---------------| -| `metastore.uris` | The Hive metastore service URIs, separate multiple addresses with commas. Such as `thrift://127.0.0.1:9083` | (none) | Yes | 0.2.0 | -| `client.pool-size` | The maximum number of Hive metastore clients in the pool for Gravitino. | 1 | No | 0.2.0 | -| `gravitino.bypass.` | Property name with this prefix passed down to the underlying HMS client for use. Such as `gravitino.bypass.hive.metastore.failure.retries = 3` indicate 3 times of retries upon failure of Thrift metastore calls | (none) | No | 0.2.0 | -| `client.pool-cache.eviction-interval-ms` | The cache pool eviction interval. | 300000 | No | 0.4.0 | -| `impersonation-enable` | Enable user impersonation for Hive catalog. | false | No | 0.4.0 | -| `kerberos.principal` | The Kerberos principal for the catalog. You should configure `gravitino.bypass.hadoop.security.authentication`, `gravitino.bypass.hive.metastore.kerberos.principal` and `gravitino.bypass.hive.metastore.sasl.enabled`if you want to use Kerberos. | (none) | required if you use kerberos | 0.4.0 | -| `kerberos.keytab-uri` | The uri of key tab for the catalog. Now supported protocols are `https`, `http`, `ftp`, `file`. | (none) | required if you use kerberos | 0.4.0 | -| `kerberos.check-interval-sec` | The interval to check validness of the principal | 60 | No | 0.4.0 | -| `kerberos.keytab-fetch-timeout-sec` | The timeout to fetch key tab | 60 | No | 0.4.0 | -| `list-all-tables` | Whether to list all tables in a database, including non-Hive tables such as Iceberg, Paimon, and Hudi. When false, non-Hive tables are filtered out on a best-effort basis; see the note below for known limitations. | false | No | 0.5.1 | -| `default.catalog` | The default catalog name for the Hive3 metastore backend; this configuration is ignored when using a Hive2 metastore. | hive | No | 1.1.0 | +| Property Name | Description | Default Value | Required | Since Version | +|---------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|------------------------------|---------------| +| `metastore.uris` | The Hive metastore service URIs, separate multiple addresses with commas. Such as `thrift://127.0.0.1:9083` | (none) | Yes | 0.2.0 | +| `client.pool-size` | The maximum number of Hive metastore clients in the pool for Gravitino. | 1 | No | 0.2.0 | +| `gravitino.bypass.` | Property name with this prefix passed down to the underlying HMS client for use. Such as `gravitino.bypass.hive.metastore.failure.retries = 3` indicate 3 times of retries upon failure of Thrift metastore calls | (none) | No | 0.2.0 | +| `client.pool-cache.eviction-interval-ms` | The cache pool eviction interval. | 300000 | No | 0.4.0 | +| `impersonation-enable` | Enable user impersonation for Hive catalog. | false | No | 0.4.0 | +| `kerberos.principal` | The Kerberos principal for the catalog. You should configure `gravitino.bypass.hadoop.security.authentication`, `gravitino.bypass.hive.metastore.kerberos.principal` and `gravitino.bypass.hive.metastore.sasl.enabled`if you want to use Kerberos. | (none) | required if you use kerberos | 0.4.0 | +| `kerberos.keytab-uri` | The uri of key tab for the catalog. Now supported protocols are `https`, `http`, `ftp`, `file`. | (none) | required if you use kerberos | 0.4.0 | +| `kerberos.check-interval-sec` | The interval to check validness of the principal | 60 | No | 0.4.0 | +| `kerberos.keytab-fetch-timeout-sec` | The timeout to fetch key tab | 60 | No | 0.4.0 | +| `kerberos.keytab-fetch-allow-local-address` | Whether to allow the Kerberos keytab URI to resolve to local, private, link-local, or cloud metadata addresses from the Gravitino server side. Enable this only for trusted URIs that must be fetched from local or private addresses. | false | No | 1.3.0 | +| `list-all-tables` | Whether to list all tables in a database, including non-Hive tables such as Iceberg, Paimon, and Hudi. When false, non-Hive tables are filtered out on a best-effort basis; see the note below for known limitations. | false | No | 0.5.1 | +| `default.catalog` | The default catalog name for the Hive3 metastore backend; this configuration is ignored when using a Hive2 metastore. | hive | No | 1.1.0 | :::note When `list-all-tables=false`, the Hive catalog removes the following on a best-effort basis: @@ -248,4 +249,3 @@ Refer to [Manage view metadata using Gravitino](./manage-view-metadata-using-gra To create a Hive catalog with S3 storage, you can refer to the [Hive catalog with S3](./hive-catalog-with-s3.md) documentation. No special configurations are required for the Hive catalog to work with S3 storage. The only difference is the storage location of the files, which is in S3. You can use `location` to specify the S3 path for the database or table. - diff --git a/docs/fileset-catalog.md b/docs/fileset-catalog.md index fbc5113239..b63733602b 100644 --- a/docs/fileset-catalog.md +++ b/docs/fileset-catalog.md @@ -53,14 +53,15 @@ Please refer to [Credential vending](./security/credential-vending.md) for more Apart from the above properties, to access fileset like HDFS fileset, you need to configure the following extra properties. -| Property Name | Description | Default Value | Required | Since Version | -|----------------------------------------------------|-------------------------------------------------------------------------------------------------|---------------|-------------------------------------------------------------|---------------| -| `authentication.impersonation-enable` | Whether to enable impersonation for the Fileset catalog. | `false` | No | 0.5.1 | -| `authentication.type` | The type of authentication for Fileset catalog, currently we only support `kerberos`, `simple`. | `simple` | No | 0.5.1 | -| `authentication.kerberos.principal` | The principal of the Kerberos authentication | (none) | required if the value of `authentication.type` is Kerberos. | 0.5.1 | -| `authentication.kerberos.keytab-uri` | The URI of The keytab for the Kerberos authentication. | (none) | required if the value of `authentication.type` is Kerberos. | 0.5.1 | -| `authentication.kerberos.check-interval-sec` | The check interval of Kerberos credential for Fileset catalog. | 60 | No | 0.5.1 | -| `authentication.kerberos.keytab-fetch-timeout-sec` | The fetch timeout of retrieving Kerberos keytab from `authentication.kerberos.keytab-uri`. | 60 | No | 0.5.1 | +| Property Name | Description | Default Value | Required | Since Version | +|------------------------------------------------------------|-------------------------------------------------------------------------------------------------|---------------|-------------------------------------------------------------|---------------| +| `authentication.impersonation-enable` | Whether to enable impersonation for the Fileset catalog. | `false` | No | 0.5.1 | +| `authentication.type` | The type of authentication for Fileset catalog, currently we only support `kerberos`, `simple`. | `simple` | No | 0.5.1 | +| `authentication.kerberos.principal` | The principal of the Kerberos authentication | (none) | required if the value of `authentication.type` is Kerberos. | 0.5.1 | +| `authentication.kerberos.keytab-uri` | The URI of The keytab for the Kerberos authentication. | (none) | required if the value of `authentication.type` is Kerberos. | 0.5.1 | +| `authentication.kerberos.check-interval-sec` | The check interval of Kerberos credential for Fileset catalog. | 60 | No | 0.5.1 | +| `authentication.kerberos.keytab-fetch-timeout-sec` | The fetch timeout of retrieving Kerberos keytab from `authentication.kerberos.keytab-uri`. | 60 | No | 0.5.1 | +| `authentication.kerberos.keytab-fetch-allow-local-address` | Whether to allow the Kerberos keytab URI to resolve to local, private, link-local, or cloud metadata addresses from the Gravitino server side. Enable this only for trusted URIs that must be fetched from local or private addresses. | false | No | 1.3.0 | The `config.resources` property allows users to specify custom configuration files. diff --git a/docs/gravitino-server-config.md b/docs/gravitino-server-config.md index 2aec2d9183..37617d68d2 100644 --- a/docs/gravitino-server-config.md +++ b/docs/gravitino-server-config.md @@ -104,6 +104,18 @@ gravitino.cache.lockSegments=16 - `gravitino.cache.expireTimeInMs`: Controls the cache TTL in milliseconds. - If `gravitino.cache.enableStats` is enabled, Gravitino will log cache statistics (hit count, miss count, load failures, etc.) every 5 minutes at the Info level. +### Job configuration + +The following table lists the job configuration items: + +| Configuration item | Description | Default value | Required | Since version | +|---------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------|----------|---------------| +| `gravitino.job.stagingDir` | Directory for managing staging files when running jobs. | `/tmp/gravitino/jobs/staging` | No | 1.0.0 | +| `gravitino.job.executor` | The executor to run jobs. By default it is `local`; users can implement their own executor and set it here. | `local` | No | 1.0.0 | +| `gravitino.job.stagingDirKeepTimeInMs` | The time in milliseconds to keep the staging files of the finished job in the job staging directory. The minimum recommended value is 10 minutes if you are not testing. | `604800000` (7 days) | No | 1.0.0 | +| `gravitino.job.statusPullIntervalInMs` | The interval in milliseconds to pull the job status from the job executor. The minimum recommended value is 1 minute if you are not testing. | `300000` (5 minutes) | No | 1.0.0 | +| `gravitino.job.remoteUri.allowLocalAddress` | Whether to allow job file remote URIs to resolve to local, private, link-local, or cloud metadata addresses from the Gravitino server side. This is disabled by default to prevent server-side request forgery (SSRF). Enable it only for trusted URIs when existing jobs must fetch from local or private addresses. | `false` | No | 1.3.0 | + #### Eviction strategies Gravitino supports multiple eviction strategies including capacity-based, weight-based, and time-based (TTL) eviction. The following describes how they work with Caffeine: diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md index d25832d6cb..578297a791 100644 --- a/docs/iceberg-rest-service.md +++ b/docs/iceberg-rest-service.md @@ -372,15 +372,16 @@ Please refer to [HTTPS Configuration](./security/how-to-use-https/#apache-iceber For JDBC backend, you can use the `gravitino.iceberg-rest.jdbc-user` and `gravitino.iceberg-rest.jdbc-password` to authenticate the JDBC connection. For Hive backend, you can use the `gravitino.iceberg-rest.authentication.type` to specify the authentication type, and use the `gravitino.iceberg-rest.authentication.kerberos.principal` and `gravitino.iceberg-rest.authentication.kerberos.keytab-uri` to authenticate the Kerberos connection. The detailed configuration items are as follows: -| Configuration item | Description | Default value | Required [...] -|---------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------ [...] -| `gravitino.iceberg-rest.authentication.type` | The type of authentication for Iceberg rest catalog backend. This configuration only applicable for for Hive backend, and only supports `Kerberos`, `simple` currently. As for JDBC backend, only username/password authentication was supported now. | `simple` | No [...] -| `gravitino.iceberg-rest.authentication.impersonation-enable` | Whether to enable impersonation for the Iceberg catalog | `false` | No [...] -| `gravitino.iceberg-rest.hive.metastore.sasl.enabled` | Whether to enable SASL authentication protocol when connect to Kerberos Hive metastore. | `false` | No, This value should be true in most case(Some will use SSL protocol, but it rather rare) if the value of `gravitino.iceberg-rest.authentication.typ [...] -| `gravitino.iceberg-rest.authentication.kerberos.principal` | The principal of the Kerberos authentication | (none) | required if the value of `gravitino.iceberg-rest.authentication.type` is Kerberos. [...] -| `gravitino.iceberg-rest.authentication.kerberos.keytab-uri` | The URI of The keytab for the Kerberos authentication. | (none) | required if the value of `gravitino.iceberg-rest.authentication.type` is Kerberos. [...] -| `gravitino.iceberg-rest.authentication.kerberos.check-interval-sec` | The check interval of Kerberos credential for Iceberg catalog. | 60 | No [...] -| `gravitino.iceberg-rest.authentication.kerberos.keytab-fetch-timeout-sec` | The fetch timeout of retrieving Kerberos keytab from `authentication.kerberos.keytab-uri`. | 60 | No [...] +| Configuration item | Description | Default value | Required [...] +|-----------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------- [...] +| `gravitino.iceberg-rest.authentication.type` | The type of authentication for Iceberg rest catalog backend. This configuration only applicable for for Hive backend, and only supports `Kerberos`, `simple` currently. As for JDBC backend, only username/password authentication was supported now. | `simple` | No [...] +| `gravitino.iceberg-rest.authentication.impersonation-enable` | Whether to enable impersonation for the Iceberg catalog | `false` | No [...] +| `gravitino.iceberg-rest.hive.metastore.sasl.enabled` | Whether to enable SASL authentication protocol when connect to Kerberos Hive metastore. | `false` | No, This value should be true in most case(Some will use SSL protocol, but it rather rare) if the value of `gravitino.iceberg-rest.authentica [...] +| `gravitino.iceberg-rest.authentication.kerberos.principal` | The principal of the Kerberos authentication | (none) | required if the value of `gravitino.iceberg-rest.authentication.type` is Kerberos. [...] +| `gravitino.iceberg-rest.authentication.kerberos.keytab-uri` | The URI of The keytab for the Kerberos authentication. | (none) | required if the value of `gravitino.iceberg-rest.authentication.type` is Kerberos. [...] +| `gravitino.iceberg-rest.authentication.kerberos.check-interval-sec` | The check interval of Kerberos credential for Iceberg catalog. | 60 | No [...] +| `gravitino.iceberg-rest.authentication.kerberos.keytab-fetch-timeout-sec` | The fetch timeout of retrieving Kerberos keytab from `authentication.kerberos.keytab-uri`. | 60 | No [...] +| `gravitino.iceberg-rest.authentication.kerberos.keytab-fetch-allow-local-address` | Whether to allow the Kerberos keytab URI to resolve to local, private, link-local, or cloud metadata addresses from the Gravitino Iceberg REST server side. Enable this only for trusted URIs that must be fetched from local or private addresses. | false | No [...] ### Credential vending diff --git a/docs/lakehouse-hudi-catalog.md b/docs/lakehouse-hudi-catalog.md index 1bf6c59116..a1c3b79c2a 100644 --- a/docs/lakehouse-hudi-catalog.md +++ b/docs/lakehouse-hudi-catalog.md @@ -44,14 +44,15 @@ Tested and verified with Apache Hudi `0.15.0`. Users can use the following properties to configure the security of the catalog backend if needed. For example, if you are using a Kerberos Hive catalog backend, you must set `authentication.type` to `Kerberos` and provide `authentication.kerberos.principal` and `authentication.kerberos.keytab-uri`. -| Property name | Description | Default value | Required | Since Version | -|----------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|-------------------------------------------------------------|-------------------| -| `authentication.type` | The type of authentication for hudi catalog backend. This configuration only applicable for for hms backend, and only supports `kerberos`, `simple` currently. | `simple` | No | 1.0.0 | -| `authentication.impersonation-enable` | Whether to enable impersonation for the hudi catalog | `false` | No | 1.0.0 | -| `authentication.kerberos.principal` | The principal of the Kerberos authentication | (none) | required if the value of `authentication.type` is kerberos. | 1.0.0 | -| `authentication.kerberos.keytab-uri` | The URI of The keytab for the Kerberos authentication. | (none) | required if the value of `authentication.type` is kerberos. | 1.0.0 | -| `authentication.kerberos.check-interval-sec` | The check interval of Kerberos credential for hudi catalog. | 60 | No | 1.0.0 | -| `authentication.kerberos.keytab-fetch-timeout-sec` | The fetch timeout of retrieving Kerberos keytab from `authentication.kerberos.keytab-uri`. | 60 | No | 1.0.0 | +| Property name | Description | Default value | Required | Since Version | +|------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|-------------------------------------------------------------|-------------------| +| `authentication.type` | The type of authentication for hudi catalog backend. This configuration only applicable for for hms backend, and only supports `kerberos`, `simple` currently. | `simple` | No | 1.0.0 | +| `authentication.impersonation-enable` | Whether to enable impersonation for the hudi catalog | `false` | No | 1.0.0 | +| `authentication.kerberos.principal` | The principal of the Kerberos authentication | (none) | required if the value of `authentication.type` is kerberos. | 1.0.0 | +| `authentication.kerberos.keytab-uri` | The URI of The keytab for the Kerberos authentication. | (none) | required if the value of `authentication.type` is kerberos. | 1.0.0 | +| `authentication.kerberos.check-interval-sec` | The check interval of Kerberos credential for hudi catalog. | 60 | No | 1.0.0 | +| `authentication.kerberos.keytab-fetch-timeout-sec` | The fetch timeout of retrieving Kerberos keytab from `authentication.kerberos.keytab-uri`. | 60 | No | 1.0.0 | +| `authentication.kerberos.keytab-fetch-allow-local-address` | Whether to allow the Kerberos keytab URI to resolve to local, private, link-local, or cloud metadata addresses from the Gravitino server side. Enable this only for trusted URIs that must be fetched from local or private addresses. | false | No | 1.3.0 | Property name with this prefix passed down to the underlying backend client for use. Such as `gravitino.bypass.hive.metastore.kerberos.principal=XXXX`、`gravitino.bypass.hadoop.security.authentication=kerberos`、`gravitino.bypass.hive.metastore.sasl.enabled=ture` And so on. diff --git a/docs/lakehouse-iceberg-catalog.md b/docs/lakehouse-iceberg-catalog.md index 203b055048..0cfe6f9122 100644 --- a/docs/lakehouse-iceberg-catalog.md +++ b/docs/lakehouse-iceberg-catalog.md @@ -204,15 +204,16 @@ Please set the `warehouse` parameter to `{storage_prefix}://{bucket_name}/${pref Users can use the following properties to configure the security of the catalog backend if needed. For example, if you are using a Kerberos Hive catalog backend, you must set `authentication.type` to `Kerberos` and provide `authentication.kerberos.principal` and `authentication.kerberos.keytab-uri`. -| Property name | Description | Default value | Required | Since Version | -|----------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------| -| `authentication.type` | The type of authentication for Iceberg catalog backend. This configuration only applicable for for Hive backend, and only supports `Kerberos`, `simple` currently. As for JDBC backend, only username/password authentication was supported now. | `simple` | No | 0.6.0-incubating | -| `authentication.impersonation-enable` | Whether to enable impersonation for the Iceberg catalog | `false` | No | 0.6.0-incubating | -| `hive.metastore.sasl.enabled` | Whether to enable SASL authentication protocol when connect to Kerberos Hive metastore. This is a raw Hive configuration | `false` | No, This value should be true in most case(Some will use SSL protocol, but it rather rare) if the value of `gravitino.iceberg-rest.authentication.type` is Kerberos. | 0.6.0-incubating | -| `authentication.kerberos.principal` | The principal of the Kerberos authentication | (none) | required if the value of `authentication.type` is Kerberos. | 0.6.0-incubating | -| `authentication.kerberos.keytab-uri` | The URI of The keytab for the Kerberos authentication. | (none) | required if the value of `authentication.type` is Kerberos. | 0.6.0-incubating | -| `authentication.kerberos.check-interval-sec` | The check interval of Kerberos credential for Iceberg catalog. | 60 | No | 0.6.0-incubating | -| `authentication.kerberos.keytab-fetch-timeout-sec` | The fetch timeout of retrieving Kerberos keytab from `authentication.kerberos.keytab-uri`. | 60 | No | 0.6.0-incubating | +| Property name | Description | Default value | Required | Sin [...] +|------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------|---- [...] +| `authentication.type` | The type of authentication for Iceberg catalog backend. This configuration only applicable for for Hive backend, and only supports `Kerberos`, `simple` currently. As for JDBC backend, only username/password authentication was supported now. | `simple` | No | 0.6 [...] +| `authentication.impersonation-enable` | Whether to enable impersonation for the Iceberg catalog | `false` | No | 0.6 [...] +| `hive.metastore.sasl.enabled` | Whether to enable SASL authentication protocol when connect to Kerberos Hive metastore. This is a raw Hive configuration | `false` | No, This value should be true in most case(Some will use SSL protocol, but it rather rare) if the value of `gravitino.iceberg-rest.authentication.type` is Kerberos. | 0.6 [...] +| `authentication.kerberos.principal` | The principal of the Kerberos authentication | (none) | required if the value of `authentication.type` is Kerberos. | 0.6 [...] +| `authentication.kerberos.keytab-uri` | The URI of The keytab for the Kerberos authentication. | (none) | required if the value of `authentication.type` is Kerberos. | 0.6 [...] +| `authentication.kerberos.check-interval-sec` | The check interval of Kerberos credential for Iceberg catalog. | 60 | No | 0.6 [...] +| `authentication.kerberos.keytab-fetch-timeout-sec` | The fetch timeout of retrieving Kerberos keytab from `authentication.kerberos.keytab-uri`. | 60 | No | 0.6 [...] +| `authentication.kerberos.keytab-fetch-allow-local-address` | Whether to allow the Kerberos keytab URI to resolve to local, private, link-local, or cloud metadata addresses from the Gravitino server side. Enable this only for trusted URIs that must be fetched from local or private addresses. | false | No [...] #### Table metadata cache diff --git a/docs/lakehouse-paimon-catalog.md b/docs/lakehouse-paimon-catalog.md index 3de913e2c5..6000fe0ae7 100644 --- a/docs/lakehouse-paimon-catalog.md +++ b/docs/lakehouse-paimon-catalog.md @@ -29,31 +29,32 @@ Builds with Apache Paimon `1.2`. ### Catalog properties -| Property name | Description | Default value | Required [...] -|----------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------- [...] -| `catalog-backend` | Catalog backend of Gravitino Paimon catalog. Supports `filesystem`, `jdbc`, `hive` and `rest`. | (none) | Yes [...] -| `uri` | The URI configuration of the Paimon catalog. `thrift://127.0.0.1:9083` or `jdbc:postgresql://127.0.0.1:5432/db_name` or `jdbc:mysql://127.0.0.1:3306/metastore_db`. It is optional for `FilesystemCatalog`. | (none) | required if the value of `catalog-backend` is not `filesystem`. [...] -| `warehouse` | Warehouse directory of catalog. `file:///user/hive/warehouse-paimon/` for local fs, `hdfs://namespace/hdfs/path` for HDFS , `s3://{bucket-name}/path/` for S3 or `oss://{bucket-name}/path` for Aliyun OSS | (none) | Yes [...] -| `catalog-backend-name` | The catalog name passed to underlying Paimon catalog backend. | The property value of `catalog-backend`, like `jdbc` for JDBC catalog backend. | No [...] -| `authentication.type` | The type of authentication for Paimon catalog backend, currently Gravitino only supports `Kerberos` and `simple`. | `simple` | No [...] -| `hive.metastore.sasl.enabled` | Whether to enable SASL authentication protocol when connect to Kerberos Hive metastore. This is a raw Hive configuration | `false` | No, This value should be true in most case(Some will use SSL protocol, but it rather rare) if the value of `gravitino.iceberg-rest.authentication.type [...] -| `authentication.kerberos.principal` | The principal of the Kerberos authentication. | (none) | required if the value of `authentication.type` is Kerberos. [...] -| `authentication.kerberos.keytab-uri` | The URI of The keytab for the Kerberos authentication. | (none) | required if the value of `authentication.type` is Kerberos. [...] -| `authentication.kerberos.check-interval-sec` | The check interval of Kerberos credential for Paimon catalog. | 60 | No [...] -| `authentication.kerberos.keytab-fetch-timeout-sec` | The fetch timeout of retrieving Kerberos keytab from `authentication.kerberos.keytab-uri`. | 60 | No [...] -| `oss-endpoint` | The endpoint of the Aliyun OSS. | (none) | required if the value of `warehouse` is a OSS path [...] -| `oss-access-key-id` | The access key of the Aliyun OSS. | (none) | required if the value of `warehouse` is a OSS path [...] -| `oss-secret-access-key` | The secret key the Aliyun OSS. | (none) | required if the value of `warehouse` is a OSS path [...] -| `s3-endpoint` | The endpoint of the AWS S3. | (none) | required if the value of `warehouse` is a S3 path [...] -| `s3-access-key-id` | The access key of the AWS S3. | (none) | required if the value of `warehouse` is a S3 path [...] -| `s3-secret-access-key` | The secret key of the AWS S3. | (none) | required if the value of `warehouse` is a S3 path [...] -| `token-provider` | The token provider type for Paimon catalog backend. | Token provider could be `bear` or `dlf`. | required if the value of `catalog-backend` is `rest`. [...] -| `token` | The bear token for Paimon REST catalog authentication. | (none) | required if the value of `token-provider` is `bear`. [...] -| `dlf-access-key-id` | The access key ID for Aliyun DLF (Data Lake Formation). | (none) | required if the value of `catalog-backend` is `rest` and accessing Aliyun DLF Paimon REST server. [...] -| `dlf-access-key-secret` | The access key secret for Aliyun DLF. | (none) | required if the value of `catalog-backend` is `rest` and accessing Aliyun DLF Paimon REST server. [...] -| `dlf-security-token` | The security token for Aliyun DLF. | (none) | No [...] -| `dlf-token-path` | The token path for Aliyun DLF. | (none) | No [...] -| `dlf-token-loader` | The token loader for Aliyun DLF. | (none) | No [...] +| Property name | Description | Default value | Required [...] +|------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------- [...] +| `catalog-backend` | Catalog backend of Gravitino Paimon catalog. Supports `filesystem`, `jdbc`, `hive` and `rest`. | (none) | Yes [...] +| `uri` | The URI configuration of the Paimon catalog. `thrift://127.0.0.1:9083` or `jdbc:postgresql://127.0.0.1:5432/db_name` or `jdbc:mysql://127.0.0.1:3306/metastore_db`. It is optional for `FilesystemCatalog`. | (none) | required if the value of `catalog-backend` is not `filesystem`. [...] +| `warehouse` | Warehouse directory of catalog. `file:///user/hive/warehouse-paimon/` for local fs, `hdfs://namespace/hdfs/path` for HDFS , `s3://{bucket-name}/path/` for S3 or `oss://{bucket-name}/path` for Aliyun OSS | (none) | Yes [...] +| `catalog-backend-name` | The catalog name passed to underlying Paimon catalog backend. | The property value of `catalog-backend`, like `jdbc` for JDBC catalog backend. | No [...] +| `authentication.type` | The type of authentication for Paimon catalog backend, currently Gravitino only supports `Kerberos` and `simple`. | `simple` | No [...] +| `hive.metastore.sasl.enabled` | Whether to enable SASL authentication protocol when connect to Kerberos Hive metastore. This is a raw Hive configuration | `false` | No, This value should be true in most case(Some will use SSL protocol, but it rather rare) if the value of `gravitino.iceberg-rest.authenticat [...] +| `authentication.kerberos.principal` | The principal of the Kerberos authentication. | (none) | required if the value of `authentication.type` is Kerberos. [...] +| `authentication.kerberos.keytab-uri` | The URI of The keytab for the Kerberos authentication. | (none) | required if the value of `authentication.type` is Kerberos. [...] +| `authentication.kerberos.check-interval-sec` | The check interval of Kerberos credential for Paimon catalog. | 60 | No [...] +| `authentication.kerberos.keytab-fetch-timeout-sec` | The fetch timeout of retrieving Kerberos keytab from `authentication.kerberos.keytab-uri`. | 60 | No [...] +| `authentication.kerberos.keytab-fetch-allow-local-address` | Whether to allow the Kerberos keytab URI to resolve to local, private, link-local, or cloud metadata addresses from the Gravitino server side. Enable this only for trusted URIs that must be fetched from local or private addresses. | false | No [...] +| `oss-endpoint` | The endpoint of the Aliyun OSS. | (none) | required if the value of `warehouse` is a OSS path [...] +| `oss-access-key-id` | The access key of the Aliyun OSS. | (none) | required if the value of `warehouse` is a OSS path [...] +| `oss-secret-access-key` | The secret key the Aliyun OSS. | (none) | required if the value of `warehouse` is a OSS path [...] +| `s3-endpoint` | The endpoint of the AWS S3. | (none) | required if the value of `warehouse` is a S3 path [...] +| `s3-access-key-id` | The access key of the AWS S3. | (none) | required if the value of `warehouse` is a S3 path [...] +| `s3-secret-access-key` | The secret key of the AWS S3. | (none) | required if the value of `warehouse` is a S3 path [...] +| `token-provider` | The token provider type for Paimon catalog backend. | Token provider could be `bear` or `dlf`. | required if the value of `catalog-backend` is `rest`. [...] +| `token` | The bear token for Paimon REST catalog authentication. | (none) | required if the value of `token-provider` is `bear`. [...] +| `dlf-access-key-id` | The access key ID for Aliyun DLF (Data Lake Formation). | (none) | required if the value of `catalog-backend` is `rest` and accessing Aliyun DLF Paimon REST server. [...] +| `dlf-access-key-secret` | The access key secret for Aliyun DLF. | (none) | required if the value of `catalog-backend` is `rest` and accessing Aliyun DLF Paimon REST server. [...] +| `dlf-security-token` | The security token for Aliyun DLF. | (none) | No [...] +| `dlf-token-path` | The token path for Aliyun DLF. | (none) | No [...] +| `dlf-token-loader` | The token loader for Aliyun DLF. | (none) | No [...] :::note - If you want to use the `oss` or `s3` warehouse, you need to place related jars in the `catalogs/lakehouse-paimon/lib` directory, more information can be found in the [Paimon S3](https://paimon.apache.org/docs/1.2/maintenance/filesystems/#s3). diff --git a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/authentication/kerberos/FetchFileUtils.java b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/authentication/kerberos/FetchFileUtils.java index 96d91765a2..bba3dabd51 100644 --- a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/authentication/kerberos/FetchFileUtils.java +++ b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/authentication/kerberos/FetchFileUtils.java @@ -25,6 +25,7 @@ import java.net.URISyntaxException; import java.nio.file.Files; import java.util.Optional; import org.apache.commons.io.FileUtils; +import org.apache.gravitino.utils.RemoteUriValidator; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -35,6 +36,16 @@ public class FetchFileUtils { public static void fetchFileFromUri( String fileUri, File destFile, int timeout, Configuration conf) throws IOException { + fetchFileFromUri(fileUri, destFile, timeout, conf, false /* allowLocalAddressForRemoteUri */); + } + + public static void fetchFileFromUri( + String fileUri, + File destFile, + int timeout, + Configuration conf, + boolean allowLocalAddressForRemoteUri) + throws IOException { try { URI uri = new URI(fileUri); String scheme = Optional.ofNullable(uri.getScheme()).orElse("file"); @@ -43,6 +54,14 @@ public class FetchFileUtils { case "http": case "https": case "ftp": + RemoteUriValidator.validate( + uri, + allowLocalAddressForRemoteUri, + String.format( + "'%s' to true, or set 'gravitino.iceberg-rest.%s' to true for the Iceberg REST " + + "service", + KerberosConfig.KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_KEY, + KerberosConfig.KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_KEY)); FileUtils.copyURLToFile(uri.toURL(), destFile, timeout * 1000, timeout * 1000); break; diff --git a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/authentication/kerberos/KerberosClient.java b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/authentication/kerberos/KerberosClient.java index 12b612fa82..5cfbc9936c 100644 --- a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/authentication/kerberos/KerberosClient.java +++ b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/authentication/kerberos/KerberosClient.java @@ -120,7 +120,12 @@ public class KerberosClient implements Closeable { // TODO: Make the configuration int fetchKeytabFileTimeout = kerberosConfig.getFetchTimeoutSec(); - FetchFileUtils.fetchFileFromUri(keyTabUri, keytabFile, fetchKeytabFileTimeout, hadoopConf); + FetchFileUtils.fetchFileFromUri( + keyTabUri, + keytabFile, + fetchKeytabFileTimeout, + hadoopConf, + kerberosConfig.allowKeytabFetchLocalAddress()); return keytabFile; } diff --git a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/authentication/kerberos/KerberosConfig.java b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/authentication/kerberos/KerberosConfig.java index 78959c9932..d5bb4c2049 100644 --- a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/authentication/kerberos/KerberosConfig.java +++ b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/authentication/kerberos/KerberosConfig.java @@ -38,6 +38,9 @@ public class KerberosConfig extends AuthenticationConfig { public static final String FETCH_TIMEOUT_SEC_KEY = "authentication.kerberos.keytab-fetch-timeout-sec"; + public static final String KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_KEY = + "authentication.kerberos.keytab-fetch-allow-local-address"; + public static final String GRAVITINO_KEYTAB_FORMAT = "keytabs/gravitino-lakehouse-iceberg-%s-keytab"; @@ -75,6 +78,17 @@ public class KerberosConfig extends AuthenticationConfig { .checkValue(value -> value > 0, ConfigConstants.POSITIVE_NUMBER_ERROR_MSG) .createWithDefault(2); + public static final ConfigEntry<Boolean> KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_ENTRY = + new ConfigBuilder(KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_KEY) + .doc( + "Whether to allow the Kerberos keytab URI to resolve to local, private, link-local, " + + "or cloud metadata addresses from the Gravitino server side. This is disabled " + + "by default to prevent SSRF. Set it to true only when the URI is trusted and " + + "must be fetched from local or private addresses.") + .version(ConfigConstants.VERSION_1_3_0) + .booleanConf() + .createWithDefault(false); + public KerberosConfig(Map<String, String> properties) { super(properties); loadFromMap(properties, k -> true); @@ -101,6 +115,10 @@ public class KerberosConfig extends AuthenticationConfig { return get(FETCH_TIMEOUT_SEC_ENTRY); } + public boolean allowKeytabFetchLocalAddress() { + return get(KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_ENTRY); + } + public static final Map<String, PropertyEntry<?>> KERBEROS_PROPERTY_ENTRIES = new ImmutableMap.Builder<String, PropertyEntry<?>>() .put( @@ -135,5 +153,16 @@ public class KerberosConfig extends AuthenticationConfig { false /* immutable */, 60 /* defaultValue */, false /* hidden */)) + .put( + KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_KEY, + PropertyEntry.booleanPropertyEntry( + KEYTAB_FETCH_ALLOW_LOCAL_ADDRESS_KEY, + "Whether to allow Kerberos keytab fetch from local or private addresses from " + + "the Gravitino server side. This is disabled by default to prevent SSRF.", + false /* required */, + false /* immutable */, + false /* defaultValue */, + false /* hidden */, + false /* reserved */)) .build(); }
