This is an automated email from the ASF dual-hosted git repository.
fanng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 73d5ffcfe [#5624] feat(bundles): support ADLS credential provider
(#5737)
73d5ffcfe is described below
commit 73d5ffcfe369f2303277660a27ce1d5b781e9836
Author: JUN <[email protected]>
AuthorDate: Tue Dec 17 14:18:25 2024 +0800
[#5624] feat(bundles): support ADLS credential provider (#5737)
### What changes were proposed in this pull request?
Add ADLS credential provider
### Why are the changes needed?
Fix: #5624
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Added a unit test and verified successful access to the ADLS container.
### Supplementary Information
Chose to use the following versions of Azure libraries:
- `azure-identity` = "1.13.1"
- `azure-storage-file-datalake` = "12.20.0"
- `azure-core-http-okhttp` = "1.12.0"
Instead of the latest versions because, although the official
documentation states support for Java 8 and later, the latest versions
appear to have been compiled with Java 21. This caused compilation
issues in the Gravitino environment. Therefore, downgraded and tested to
find the latest usable versions.
---
.../gravitino/credential/ADLSTokenCredential.java | 116 ++++++++++++
.../gravitino/credential/OSSTokenCredential.java | 16 +-
.../org.apache.gravitino.credential.Credential | 1 +
bundles/azure-bundle/build.gradle.kts | 4 +
.../abs/credential/ADLSLocationUtils.java | 85 +++++++++
.../abs/credential/ADLSTokenProvider.java | 138 ++++++++++++++
.../gravitino/abs/fs/AzureFileSystemProvider.java | 10 +-
....apache.gravitino.credential.CredentialProvider | 7 +-
.../lakehouse/iceberg/IcebergConstants.java | 4 +
.../lakehouse/iceberg/IcebergPropertiesUtils.java | 8 +
.../gravitino/credential/CredentialConstants.java | 3 +
.../apache/gravitino/storage/ABSProperties.java | 29 ---
.../apache/gravitino/storage/AzureProperties.java | 39 ++++
.../integration/test/HadoopABSCatalogIT.java | 10 +-
.../tests/integration/test_gvfs_with_abs.py | 4 +-
.../test/GravitinoVirtualFileSystemABSIT.java | 18 +-
.../credential/CredentialPropertyUtils.java | 12 ++
.../credential/TestCredentialFactory.java | 37 +++-
.../credential/TestCredentialPropertiesUtils.java | 24 ++-
.../credential/config/ADLSCredentialConfig.java | 116 ++++++++++++
.../iceberg-rest-server-dependency.sh | 7 +
dev/docker/iceberg-rest-server/rewrite_config.py | 7 +-
docs/hadoop-catalog.md | 12 +-
docs/how-to-use-gvfs.md | 8 +-
docs/iceberg-rest-service.md | 57 ++++--
gradle/libs.versions.toml | 10 +
iceberg/iceberg-common/build.gradle.kts | 1 +
iceberg/iceberg-rest-server/build.gradle.kts | 8 +
.../integration/test/IcebergRESTADLSIT.java | 205 +++++++++++++++++++++
.../iceberg/integration/test/IcebergRESTGCSIT.java | 11 +-
.../iceberg/integration/test/IcebergRESTOSSIT.java | 21 +--
.../integration/test/IcebergRESTOSSSecretIT.java | 15 +-
.../iceberg/integration/test/IcebergRESTS3IT.java | 19 +-
settings.gradle.kts | 2 +-
34 files changed, 921 insertions(+), 143 deletions(-)
diff --git
a/api/src/main/java/org/apache/gravitino/credential/ADLSTokenCredential.java
b/api/src/main/java/org/apache/gravitino/credential/ADLSTokenCredential.java
new file mode 100644
index 000000000..25c83c2f7
--- /dev/null
+++ b/api/src/main/java/org/apache/gravitino/credential/ADLSTokenCredential.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.credential;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
+
+/** ADLS SAS token credential. */
+public class ADLSTokenCredential implements Credential {
+
+ /** ADLS SAS token credential type. */
+ public static final String ADLS_SAS_TOKEN_CREDENTIAL_TYPE = "adls-sas-token";
+ /** ADLS base domain */
+ public static final String ADLS_DOMAIN = "dfs.core.windows.net";
+ /** ADLS storage account name */
+ public static final String GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME =
"azure-storage-account-name";
+ /** ADLS SAS token used to access ADLS data. */
+ public static final String GRAVITINO_ADLS_SAS_TOKEN = "adls-sas-token";
+
+ private String accountName;
+ private String sasToken;
+ private long expireTimeInMS;
+
+ /**
+ * Constructs an instance of {@link ADLSTokenCredential} with SAS token.
+ *
+ * @param accountName The ADLS account name.
+ * @param sasToken The ADLS SAS token.
+ * @param expireTimeInMS The SAS token expire time in ms.
+ */
+ public ADLSTokenCredential(String accountName, String sasToken, long
expireTimeInMS) {
+ validate(accountName, sasToken, expireTimeInMS);
+ this.accountName = accountName;
+ this.sasToken = sasToken;
+ this.expireTimeInMS = expireTimeInMS;
+ }
+
+ /**
+ * This is the constructor that is used by credential factory to create an
instance of credential
+ * according to the credential information.
+ */
+ public ADLSTokenCredential() {}
+
+ @Override
+ public String credentialType() {
+ return ADLS_SAS_TOKEN_CREDENTIAL_TYPE;
+ }
+
+ @Override
+ public long expireTimeInMs() {
+ return expireTimeInMS;
+ }
+
+ @Override
+ public Map<String, String> credentialInfo() {
+ return (new ImmutableMap.Builder<String, String>())
+ .put(GRAVITINO_ADLS_SAS_TOKEN, sasToken)
+ .build();
+ }
+
+ @Override
+ public void initialize(Map<String, String> credentialInfo, long
expireTimeInMS) {
+ String accountName =
credentialInfo.get(GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME);
+ String sasToken = credentialInfo.get(GRAVITINO_ADLS_SAS_TOKEN);
+ validate(accountName, sasToken, expireTimeInMS);
+ this.accountName = accountName;
+ this.sasToken = sasToken;
+ this.expireTimeInMS = expireTimeInMS;
+ }
+
+ /**
+ * Get ADLS account name
+ *
+ * @return The ADLS account name
+ */
+ public String accountName() {
+ return accountName;
+ }
+
+ /**
+ * Get ADLS SAS token.
+ *
+ * @return The ADLS SAS token.
+ */
+ public String sasToken() {
+ return sasToken;
+ }
+
+ private void validate(String accountName, String sasToken, long
expireTimeInMS) {
+ Preconditions.checkArgument(
+ StringUtils.isNotBlank(accountName), "ADLS account name should not be
empty.");
+ Preconditions.checkArgument(
+ StringUtils.isNotBlank(sasToken), "ADLS SAS token should not be
empty.");
+ Preconditions.checkArgument(
+ expireTimeInMS > 0, "The expire time of ADLSTokenCredential should
great than 0");
+ }
+}
diff --git
a/api/src/main/java/org/apache/gravitino/credential/OSSTokenCredential.java
b/api/src/main/java/org/apache/gravitino/credential/OSSTokenCredential.java
index edf23f207..70e883948 100644
--- a/api/src/main/java/org/apache/gravitino/credential/OSSTokenCredential.java
+++ b/api/src/main/java/org/apache/gravitino/credential/OSSTokenCredential.java
@@ -51,13 +51,7 @@ public class OSSTokenCredential implements Credential {
*/
public OSSTokenCredential(
String accessKeyId, String secretAccessKey, String securityToken, long
expireTimeInMS) {
- Preconditions.checkArgument(
- StringUtils.isNotBlank(accessKeyId), "OSS access key Id should not be
empty");
- Preconditions.checkArgument(
- StringUtils.isNotBlank(secretAccessKey), "OSS access key secret should
not be empty");
- Preconditions.checkArgument(
- StringUtils.isNotBlank(securityToken), "OSS security token should not
be empty");
-
+ validate(accessKeyId, secretAccessKey, securityToken, expireTimeInMS);
this.accessKeyId = accessKeyId;
this.secretAccessKey = secretAccessKey;
this.securityToken = securityToken;
@@ -133,12 +127,12 @@ public class OSSTokenCredential implements Credential {
private void validate(
String accessKeyId, String secretAccessKey, String sessionToken, long
expireTimeInMs) {
Preconditions.checkArgument(
- StringUtils.isNotBlank(accessKeyId), "S3 access key Id should not be
empty");
+ StringUtils.isNotBlank(accessKeyId), "OSS access key Id should not be
empty");
Preconditions.checkArgument(
- StringUtils.isNotBlank(secretAccessKey), "S3 secret access key should
not be empty");
+ StringUtils.isNotBlank(secretAccessKey), "OSS secret access key should
not be empty");
Preconditions.checkArgument(
- StringUtils.isNotBlank(sessionToken), "S3 session token should not be
empty");
+ StringUtils.isNotBlank(sessionToken), "OSS session token should not be
empty");
Preconditions.checkArgument(
- expireTimeInMs > 0, "The expire time of S3TokenCredential should great
than 0");
+ expireTimeInMs > 0, "The expire time of OSSTokenCredential should
great than 0");
}
}
diff --git
a/api/src/main/resources/META-INF/services/org.apache.gravitino.credential.Credential
b/api/src/main/resources/META-INF/services/org.apache.gravitino.credential.Credential
index b6d2dd028..f130b4b64 100644
---
a/api/src/main/resources/META-INF/services/org.apache.gravitino.credential.Credential
+++
b/api/src/main/resources/META-INF/services/org.apache.gravitino.credential.Credential
@@ -22,3 +22,4 @@ org.apache.gravitino.credential.S3SecretKeyCredential
org.apache.gravitino.credential.GCSTokenCredential
org.apache.gravitino.credential.OSSTokenCredential
org.apache.gravitino.credential.OSSSecretKeyCredential
+org.apache.gravitino.credential.ADLSTokenCredential
diff --git a/bundles/azure-bundle/build.gradle.kts
b/bundles/azure-bundle/build.gradle.kts
index 8580c672e..9e4a4add5 100644
--- a/bundles/azure-bundle/build.gradle.kts
+++ b/bundles/azure-bundle/build.gradle.kts
@@ -27,6 +27,7 @@ plugins {
dependencies {
compileOnly(project(":api"))
compileOnly(project(":core"))
+ compileOnly(project(":catalogs:catalog-common"))
compileOnly(project(":catalogs:catalog-hadoop"))
compileOnly(project(":catalogs:hadoop-common")) {
exclude("*")
@@ -34,6 +35,9 @@ dependencies {
compileOnly(libs.hadoop3.common)
+ implementation(libs.azure.identity)
+ implementation(libs.azure.storage.file.datalake)
+
implementation(libs.commons.lang3)
// runtime used
implementation(libs.commons.logging)
diff --git
a/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/ADLSLocationUtils.java
b/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/ADLSLocationUtils.java
new file mode 100644
index 000000000..198b00deb
--- /dev/null
+++
b/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/ADLSLocationUtils.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.abs.credential;
+
+import java.net.URI;
+
+public class ADLSLocationUtils {
+ /** Encapsulates parts of an ADLS URI: container, account name, and path. */
+ public static class ADLSLocationParts {
+ private final String container;
+ private final String accountName;
+ private final String path;
+
+ public ADLSLocationParts(String container, String accountName, String
path) {
+ this.container = container;
+ this.accountName = accountName;
+ this.path = path;
+ }
+
+ public String getContainer() {
+ return container;
+ }
+
+ public String getAccountName() {
+ return accountName;
+ }
+
+ public String getPath() {
+ return path;
+ }
+ }
+
+ /**
+ * Parses an ADLS URI and extracts its components. Example: Input:
+ * "abfss://[email protected]/path/data". Output:
ADLSLocationParts {
+ * container = "container", accountName = "accountName", path = "/path/data"
}
+ *
+ * @param location The ADLS URI (e.g.,
+ * "abfss://[email protected]/path/data").
+ * @return An ADLSLocationParts object containing the container, account
name, and path.
+ * @throws IllegalArgumentException If the URI format is invalid.
+ */
+ public static ADLSLocationParts parseLocation(String location) {
+ URI locationUri = URI.create(location);
+
+ String[] authorityParts = locationUri.getAuthority().split("@");
+
+ if (authorityParts.length <= 1) {
+ throw new IllegalArgumentException("Invalid location: " + location);
+ }
+
+ return new ADLSLocationParts(
+ authorityParts[0], authorityParts[1].split("\\.")[0],
locationUri.getPath());
+ }
+
+ /**
+ * Trims leading and trailing slashes from a given string.
+ *
+ * @param input The string to process.
+ * @return A string without leading or trailing slashes, or null if the
input is null.
+ */
+ public static String trimSlashes(String input) {
+ if (input == null) {
+ return null;
+ }
+ return input.replaceAll("^/+|/*$", "");
+ }
+}
diff --git
a/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/ADLSTokenProvider.java
b/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/ADLSTokenProvider.java
new file mode 100644
index 000000000..e2ee3ed82
--- /dev/null
+++
b/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/ADLSTokenProvider.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.abs.credential;
+
+import com.azure.core.util.Context;
+import com.azure.identity.ClientSecretCredential;
+import com.azure.identity.ClientSecretCredentialBuilder;
+import com.azure.storage.file.datalake.DataLakeServiceClient;
+import com.azure.storage.file.datalake.DataLakeServiceClientBuilder;
+import com.azure.storage.file.datalake.implementation.util.DataLakeSasImplUtil;
+import com.azure.storage.file.datalake.models.UserDelegationKey;
+import com.azure.storage.file.datalake.sas.DataLakeServiceSasSignatureValues;
+import com.azure.storage.file.datalake.sas.PathSasPermission;
+import java.time.OffsetDateTime;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import org.apache.gravitino.credential.ADLSTokenCredential;
+import org.apache.gravitino.credential.Credential;
+import org.apache.gravitino.credential.CredentialConstants;
+import org.apache.gravitino.credential.CredentialContext;
+import org.apache.gravitino.credential.CredentialProvider;
+import org.apache.gravitino.credential.PathBasedCredentialContext;
+import org.apache.gravitino.credential.config.ADLSCredentialConfig;
+
+/** Generates ADLS token to access ADLS data. */
+public class ADLSTokenProvider implements CredentialProvider {
+ private String storageAccountName;
+ private String tenantId;
+ private String clientId;
+ private String clientSecret;
+ private String endpoint;
+ private Integer tokenExpireSecs;
+
+ @Override
+ public void initialize(Map<String, String> properties) {
+ ADLSCredentialConfig adlsCredentialConfig = new
ADLSCredentialConfig(properties);
+ this.storageAccountName = adlsCredentialConfig.storageAccountName();
+ this.tenantId = adlsCredentialConfig.tenantId();
+ this.clientId = adlsCredentialConfig.clientId();
+ this.clientSecret = adlsCredentialConfig.clientSecret();
+ this.endpoint =
+ String.format("https://%s.%s";, storageAccountName,
ADLSTokenCredential.ADLS_DOMAIN);
+ this.tokenExpireSecs = adlsCredentialConfig.tokenExpireInSecs();
+ }
+
+ @Override
+ public void close() {}
+
+ @Override
+ public String credentialType() {
+ return CredentialConstants.ADLS_TOKEN_CREDENTIAL_PROVIDER_TYPE;
+ }
+
+ @Override
+ public Credential getCredential(CredentialContext context) {
+ if (!(context instanceof PathBasedCredentialContext)) {
+ return null;
+ }
+ PathBasedCredentialContext pathBasedCredentialContext =
(PathBasedCredentialContext) context;
+
+ Set<String> writePaths = pathBasedCredentialContext.getWritePaths();
+ Set<String> readPaths = pathBasedCredentialContext.getReadPaths();
+
+ Set<String> combinedPaths = new HashSet<>(writePaths);
+ combinedPaths.addAll(readPaths);
+
+ if (combinedPaths.size() != 1) {
+ throw new IllegalArgumentException(
+ "ADLS should contain exactly one unique path, but found: "
+ + combinedPaths.size()
+ + " paths: "
+ + combinedPaths);
+ }
+ String uniquePath = combinedPaths.iterator().next();
+
+ ClientSecretCredential clientSecretCredential =
+ new ClientSecretCredentialBuilder()
+ .tenantId(tenantId)
+ .clientId(clientId)
+ .clientSecret(clientSecret)
+ .build();
+
+ DataLakeServiceClient dataLakeServiceClient =
+ new DataLakeServiceClientBuilder()
+ .endpoint(endpoint)
+ .credential(clientSecretCredential)
+ .buildClient();
+
+ OffsetDateTime start = OffsetDateTime.now();
+ OffsetDateTime expiry = OffsetDateTime.now().plusSeconds(tokenExpireSecs);
+ UserDelegationKey userDelegationKey =
dataLakeServiceClient.getUserDelegationKey(start, expiry);
+
+ PathSasPermission pathSasPermission =
+ new
PathSasPermission().setReadPermission(true).setListPermission(true);
+
+ if (!writePaths.isEmpty()) {
+ pathSasPermission
+ .setWritePermission(true)
+ .setDeletePermission(true)
+ .setCreatePermission(true)
+ .setAddPermission(true);
+ }
+
+ DataLakeServiceSasSignatureValues signatureValues =
+ new DataLakeServiceSasSignatureValues(expiry, pathSasPermission);
+
+ ADLSLocationUtils.ADLSLocationParts locationParts =
ADLSLocationUtils.parseLocation(uniquePath);
+ String sasToken =
+ new DataLakeSasImplUtil(
+ signatureValues,
+ locationParts.getContainer(),
+ ADLSLocationUtils.trimSlashes(locationParts.getPath()),
+ true)
+ .generateUserDelegationSas(
+ userDelegationKey, locationParts.getAccountName(),
Context.NONE);
+
+ return new ADLSTokenCredential(
+ locationParts.getAccountName(), sasToken,
expiry.toInstant().toEpochMilli());
+ }
+}
diff --git
a/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/fs/AzureFileSystemProvider.java
b/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/fs/AzureFileSystemProvider.java
index cad38e14c..f89240441 100644
---
a/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/fs/AzureFileSystemProvider.java
+++
b/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/fs/AzureFileSystemProvider.java
@@ -26,7 +26,7 @@ import java.util.Map;
import javax.annotation.Nonnull;
import org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider;
import org.apache.gravitino.catalog.hadoop.fs.FileSystemUtils;
-import org.apache.gravitino.storage.ABSProperties;
+import org.apache.gravitino.storage.AzureProperties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -49,13 +49,13 @@ public class AzureFileSystemProvider implements
FileSystemProvider {
Map<String, String> hadoopConfMap =
FileSystemUtils.toHadoopConfigMap(config, ImmutableMap.of());
- if (config.containsKey(ABSProperties.GRAVITINO_ABS_ACCOUNT_NAME)
- && config.containsKey(ABSProperties.GRAVITINO_ABS_ACCOUNT_KEY)) {
+ if
(config.containsKey(AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME)
+ &&
config.containsKey(AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY)) {
hadoopConfMap.put(
String.format(
"fs.azure.account.key.%s.dfs.core.windows.net",
- config.get(ABSProperties.GRAVITINO_ABS_ACCOUNT_NAME)),
- config.get(ABSProperties.GRAVITINO_ABS_ACCOUNT_KEY));
+
config.get(AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME)),
+ config.get(AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY));
}
if (!config.containsKey(ABFS_IMPL_KEY)) {
diff --git
a/api/src/main/resources/META-INF/services/org.apache.gravitino.credential.Credential
b/bundles/azure-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider
similarity index 75%
copy from
api/src/main/resources/META-INF/services/org.apache.gravitino.credential.Credential
copy to
bundles/azure-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider
index b6d2dd028..fb53efffa 100644
---
a/api/src/main/resources/META-INF/services/org.apache.gravitino.credential.Credential
+++
b/bundles/azure-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider
@@ -16,9 +16,4 @@
# specific language governing permissions and limitations
# under the License.
#
-
-org.apache.gravitino.credential.S3TokenCredential
-org.apache.gravitino.credential.S3SecretKeyCredential
-org.apache.gravitino.credential.GCSTokenCredential
-org.apache.gravitino.credential.OSSTokenCredential
-org.apache.gravitino.credential.OSSSecretKeyCredential
+org.apache.gravitino.abs.credential.ADLSTokenProvider
\ No newline at end of file
diff --git
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java
index 4d9e99eba..214f38113 100644
---
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java
+++
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java
@@ -47,6 +47,10 @@ public class IcebergConstants {
public static final String ICEBERG_OSS_ACCESS_KEY_ID =
"client.access-key-id";
public static final String ICEBERG_OSS_ACCESS_KEY_SECRET =
"client.access-key-secret";
+ public static final String ICEBERG_ADLS_STORAGE_ACCOUNT_NAME =
+ "adls.auth.shared-key.account.name";
+ public static final String ICEBERG_ADLS_STORAGE_ACCOUNT_KEY =
"adls.auth.shared-key.account.key";
+
// Iceberg Table properties constants
public static final String COMMENT = "comment";
diff --git
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergPropertiesUtils.java
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergPropertiesUtils.java
index abe08c57d..06f017c59 100644
---
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergPropertiesUtils.java
+++
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergPropertiesUtils.java
@@ -23,6 +23,7 @@ import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
+import org.apache.gravitino.storage.AzureProperties;
import org.apache.gravitino.storage.OSSProperties;
import org.apache.gravitino.storage.S3Properties;
@@ -55,6 +56,13 @@ public class IcebergPropertiesUtils {
map.put(
OSSProperties.GRAVITINO_OSS_ACCESS_KEY_SECRET,
IcebergConstants.ICEBERG_OSS_ACCESS_KEY_SECRET);
+ // ADLS
+ map.put(
+ AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME,
+ IcebergConstants.ICEBERG_ADLS_STORAGE_ACCOUNT_NAME);
+ map.put(
+ AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY,
+ IcebergConstants.ICEBERG_ADLS_STORAGE_ACCOUNT_KEY);
GRAVITINO_CONFIG_TO_ICEBERG = Collections.unmodifiableMap(map);
}
diff --git
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java
index 739cd0139..7dd74d084 100644
---
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java
+++
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java
@@ -29,5 +29,8 @@ public class CredentialConstants {
public static final String OSS_TOKEN_CREDENTIAL_PROVIDER = "oss-token";
public static final String OSS_TOKEN_EXPIRE_IN_SECS =
"oss-token-expire-in-secs";
+ public static final String ADLS_TOKEN_CREDENTIAL_PROVIDER_TYPE =
"adls-token";
+ public static final String ADLS_TOKEN_EXPIRE_IN_SECS =
"adls-token-expire-in-secs";
+
private CredentialConstants() {}
}
diff --git
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/ABSProperties.java
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/ABSProperties.java
deleted file mode 100644
index a76ece32b..000000000
---
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/ABSProperties.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.gravitino.storage;
-
-public class ABSProperties {
-
- // The account name of the Azure Blob Storage.
- public static final String GRAVITINO_ABS_ACCOUNT_NAME = "abs-account-name";
-
- // The account key of the Azure Blob Storage.
- public static final String GRAVITINO_ABS_ACCOUNT_KEY = "abs-account-key";
-}
diff --git
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/AzureProperties.java
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/AzureProperties.java
new file mode 100644
index 000000000..5da0172c5
--- /dev/null
+++
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/AzureProperties.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.storage;
+
+// Defines unified properties for Azure configurations.
+public class AzureProperties {
+
+ // Configuration key for specifying the name of the storage account.
+ public static final String GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME =
"azure-storage-account-name";
+ // Configuration key for specifying the key of the storage account.
+ public static final String GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY =
"azure-storage-account-key";
+
+ // Configuration key for specifying the Azure Active Directory (AAD) tenant
ID.
+ public static final String GRAVITINO_AZURE_TENANT_ID = "azure-tenant-id";
+ // Configuration key for specifying the Azure Active Directory (AAD) client
ID used for
+ // authentication.
+ public static final String GRAVITINO_AZURE_CLIENT_ID = "azure-client-id";
+ // Configuration key for specifying the Azure Active Directory (AAD) client
secret used for
+ // authentication.
+ public static final String GRAVITINO_AZURE_CLIENT_SECRET =
"azure-client-secret";
+
+ private AzureProperties() {}
+}
diff --git
a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopABSCatalogIT.java
b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopABSCatalogIT.java
index 0da915a7d..482daba2e 100644
---
a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopABSCatalogIT.java
+++
b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopABSCatalogIT.java
@@ -31,7 +31,7 @@ import org.apache.gravitino.Schema;
import org.apache.gravitino.abs.fs.AzureFileSystemProvider;
import org.apache.gravitino.file.Fileset;
import org.apache.gravitino.integration.test.util.GravitinoITUtils;
-import org.apache.gravitino.storage.ABSProperties;
+import org.apache.gravitino.storage.AzureProperties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -113,8 +113,8 @@ public class HadoopABSCatalogIT extends HadoopCatalogIT {
protected void createCatalog() {
Map<String, String> map = Maps.newHashMap();
- map.put(ABSProperties.GRAVITINO_ABS_ACCOUNT_NAME, ABS_ACCOUNT_NAME);
- map.put(ABSProperties.GRAVITINO_ABS_ACCOUNT_KEY, ABS_ACCOUNT_KEY);
+ map.put(AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME,
ABS_ACCOUNT_NAME);
+ map.put(AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY,
ABS_ACCOUNT_KEY);
map.put(FILESYSTEM_PROVIDERS, AzureFileSystemProvider.ABS_PROVIDER_NAME);
metalake.createCatalog(catalogName, Catalog.Type.FILESET, provider,
"comment", map);
@@ -138,8 +138,8 @@ public class HadoopABSCatalogIT extends HadoopCatalogIT {
GravitinoITUtils.genRandomName("CatalogCatalogIT"));
Map<String, String> catalogProps = Maps.newHashMap();
catalogProps.put("location", ossLocation);
- catalogProps.put(ABSProperties.GRAVITINO_ABS_ACCOUNT_NAME,
ABS_ACCOUNT_NAME);
- catalogProps.put(ABSProperties.GRAVITINO_ABS_ACCOUNT_KEY, ABS_ACCOUNT_KEY);
+ catalogProps.put(AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME,
ABS_ACCOUNT_NAME);
+ catalogProps.put(AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY,
ABS_ACCOUNT_KEY);
catalogProps.put(FILESYSTEM_PROVIDERS,
AzureFileSystemProvider.ABS_PROVIDER_NAME);
Catalog localCatalog =
diff --git a/clients/client-python/tests/integration/test_gvfs_with_abs.py
b/clients/client-python/tests/integration/test_gvfs_with_abs.py
index 3377c07cd..a218efcfd 100644
--- a/clients/client-python/tests/integration/test_gvfs_with_abs.py
+++ b/clients/client-python/tests/integration/test_gvfs_with_abs.py
@@ -123,8 +123,8 @@ class TestGvfsWithABS(TestGvfsWithHDFS):
comment="",
properties={
"filesystem-providers": "abs",
- "abs-account-name": cls.azure_abs_account_name,
- "abs-account-key": cls.azure_abs_account_key,
+ "azure-storage-account-name": cls.azure_abs_account_name,
+ "azure-storage-account-key": cls.azure_abs_account_key,
},
)
catalog.as_schemas().create_schema(
diff --git
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemABSIT.java
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemABSIT.java
index 11557417f..d69c2d946 100644
---
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemABSIT.java
+++
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemABSIT.java
@@ -30,7 +30,7 @@ import org.apache.gravitino.Catalog;
import org.apache.gravitino.abs.fs.AzureFileSystemProvider;
import org.apache.gravitino.catalog.hadoop.fs.FileSystemUtils;
import org.apache.gravitino.integration.test.util.GravitinoITUtils;
-import org.apache.gravitino.storage.ABSProperties;
+import org.apache.gravitino.storage.AzureProperties;
import org.apache.hadoop.conf.Configuration;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
@@ -77,8 +77,8 @@ public class GravitinoVirtualFileSystemABSIT extends
GravitinoVirtualFileSystemI
Map<String, String> properties = Maps.newHashMap();
- properties.put(ABSProperties.GRAVITINO_ABS_ACCOUNT_NAME, ABS_ACCOUNT_NAME);
- properties.put(ABSProperties.GRAVITINO_ABS_ACCOUNT_KEY, ABS_ACCOUNT_KEY);
+ properties.put(AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME,
ABS_ACCOUNT_NAME);
+ properties.put(AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY,
ABS_ACCOUNT_KEY);
properties.put(FILESYSTEM_PROVIDERS,
AzureFileSystemProvider.ABS_PROVIDER_NAME);
Catalog catalog =
@@ -96,8 +96,8 @@ public class GravitinoVirtualFileSystemABSIT extends
GravitinoVirtualFileSystemI
conf.set("fs.gravitino.client.metalake", metalakeName);
// Pass this configuration to the real file system
- conf.set(ABSProperties.GRAVITINO_ABS_ACCOUNT_NAME, ABS_ACCOUNT_NAME);
- conf.set(ABSProperties.GRAVITINO_ABS_ACCOUNT_KEY, ABS_ACCOUNT_KEY);
+ conf.set(AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME,
ABS_ACCOUNT_NAME);
+ conf.set(AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY,
ABS_ACCOUNT_KEY);
conf.set("fs.abfss.impl",
"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem");
}
@@ -133,13 +133,13 @@ public class GravitinoVirtualFileSystemABSIT extends
GravitinoVirtualFileSystemI
Map<String, String> hadoopConfMap = FileSystemUtils.toHadoopConfigMap(map,
ImmutableMap.of());
- if (gvfsConf.get(ABSProperties.GRAVITINO_ABS_ACCOUNT_NAME) != null
- && gvfsConf.get(ABSProperties.GRAVITINO_ABS_ACCOUNT_KEY) != null) {
+ if (gvfsConf.get(AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME) !=
null
+ && gvfsConf.get(AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY)
!= null) {
hadoopConfMap.put(
String.format(
"fs.azure.account.key.%s.dfs.core.windows.net",
- gvfsConf.get(ABSProperties.GRAVITINO_ABS_ACCOUNT_NAME)),
- gvfsConf.get(ABSProperties.GRAVITINO_ABS_ACCOUNT_KEY));
+
gvfsConf.get(AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME)),
+ gvfsConf.get(AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY));
}
hadoopConfMap.forEach(absConf::set);
diff --git
a/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java
b/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java
index d9fb74903..e1803a6dd 100644
---
a/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java
+++
b/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java
@@ -33,6 +33,7 @@ public class CredentialPropertyUtils {
@VisibleForTesting static final String ICEBERG_S3_SECRET_ACCESS_KEY =
"s3.secret-access-key";
@VisibleForTesting static final String ICEBERG_S3_TOKEN = "s3.session-token";
@VisibleForTesting static final String ICEBERG_GCS_TOKEN =
"gcs.oauth2.token";
+ @VisibleForTesting static final String ICEBERG_ADLS_TOKEN = "adls.sas-token";
@VisibleForTesting static final String ICEBERG_OSS_ACCESS_KEY_ID =
"client.access-key-id";
@VisibleForTesting static final String ICEBERG_OSS_ACCESS_KEY_SECRET =
"client.access-key-secret";
@@ -75,6 +76,17 @@ public class CredentialPropertyUtils {
if (credential instanceof OSSTokenCredential || credential instanceof
OSSSecretKeyCredential) {
return transformProperties(credential.credentialInfo(),
icebergCredentialPropertyMap);
}
+ if (credential instanceof ADLSTokenCredential) {
+ ADLSTokenCredential adlsCredential = (ADLSTokenCredential) credential;
+ String sasTokenKey =
+ String.format(
+ "%s.%s.%s",
+ ICEBERG_ADLS_TOKEN, adlsCredential.accountName(),
ADLSTokenCredential.ADLS_DOMAIN);
+
+ Map<String, String> icebergADLSCredentialProperties = new HashMap<>();
+ icebergADLSCredentialProperties.put(sasTokenKey,
adlsCredential.sasToken());
+ return icebergADLSCredentialProperties;
+ }
return credential.toProperties();
}
diff --git
a/common/src/test/java/org/apache/gravitino/credential/TestCredentialFactory.java
b/common/src/test/java/org/apache/gravitino/credential/TestCredentialFactory.java
index b873c0afa..75a669e38 100644
---
a/common/src/test/java/org/apache/gravitino/credential/TestCredentialFactory.java
+++
b/common/src/test/java/org/apache/gravitino/credential/TestCredentialFactory.java
@@ -42,7 +42,7 @@ public class TestCredentialFactory {
S3TokenCredential.S3_TOKEN_CREDENTIAL_TYPE, s3TokenCredentialInfo,
expireTime);
Assertions.assertEquals(
S3TokenCredential.S3_TOKEN_CREDENTIAL_TYPE,
s3TokenCredential.credentialType());
- Assertions.assertTrue(s3TokenCredential instanceof S3TokenCredential);
+ Assertions.assertInstanceOf(S3TokenCredential.class, s3TokenCredential);
S3TokenCredential s3TokenCredential1 = (S3TokenCredential)
s3TokenCredential;
Assertions.assertEquals("accessKeyId", s3TokenCredential1.accessKeyId());
Assertions.assertEquals("secretAccessKey",
s3TokenCredential1.secretAccessKey());
@@ -67,7 +67,7 @@ public class TestCredentialFactory {
Assertions.assertEquals(
S3SecretKeyCredential.S3_SECRET_KEY_CREDENTIAL_TYPE,
s3SecretKeyCredential.credentialType());
- Assertions.assertTrue(s3SecretKeyCredential instanceof
S3SecretKeyCredential);
+ Assertions.assertInstanceOf(S3SecretKeyCredential.class,
s3SecretKeyCredential);
S3SecretKeyCredential s3SecretKeyCredential1 = (S3SecretKeyCredential)
s3SecretKeyCredential;
Assertions.assertEquals("accessKeyId",
s3SecretKeyCredential1.accessKeyId());
Assertions.assertEquals("secretAccessKey",
s3SecretKeyCredential1.secretAccessKey());
@@ -84,7 +84,7 @@ public class TestCredentialFactory {
GCSTokenCredential.GCS_TOKEN_CREDENTIAL_TYPE,
gcsTokenCredentialInfo, expireTime);
Assertions.assertEquals(
GCSTokenCredential.GCS_TOKEN_CREDENTIAL_TYPE,
gcsTokenCredential.credentialType());
- Assertions.assertTrue(gcsTokenCredential instanceof GCSTokenCredential);
+ Assertions.assertInstanceOf(GCSTokenCredential.class, gcsTokenCredential);
GCSTokenCredential gcsTokenCredential1 = (GCSTokenCredential)
gcsTokenCredential;
Assertions.assertEquals("accessToken", gcsTokenCredential1.token());
Assertions.assertEquals(expireTime, gcsTokenCredential1.expireTimeInMs());
@@ -106,7 +106,7 @@ public class TestCredentialFactory {
OSSTokenCredential.OSS_TOKEN_CREDENTIAL_TYPE,
ossTokenCredentialInfo, expireTime);
Assertions.assertEquals(
OSSTokenCredential.OSS_TOKEN_CREDENTIAL_TYPE,
ossTokenCredential.credentialType());
- Assertions.assertTrue(ossTokenCredential instanceof OSSTokenCredential);
+ Assertions.assertInstanceOf(OSSTokenCredential.class, ossTokenCredential);
OSSTokenCredential ossTokenCredential1 = (OSSTokenCredential)
ossTokenCredential;
Assertions.assertEquals("access-id", ossTokenCredential1.accessKeyId());
Assertions.assertEquals("secret-key",
ossTokenCredential1.secretAccessKey());
@@ -131,11 +131,38 @@ public class TestCredentialFactory {
Assertions.assertEquals(
OSSSecretKeyCredential.OSS_SECRET_KEY_CREDENTIAL_TYPE,
ossSecretKeyCredential.credentialType());
- Assertions.assertTrue(ossSecretKeyCredential instanceof
OSSSecretKeyCredential);
+ Assertions.assertInstanceOf(OSSSecretKeyCredential.class,
ossSecretKeyCredential);
OSSSecretKeyCredential ossSecretKeyCredential1 =
(OSSSecretKeyCredential) ossSecretKeyCredential;
Assertions.assertEquals("accessKeyId",
ossSecretKeyCredential1.accessKeyId());
Assertions.assertEquals("secretAccessKey",
ossSecretKeyCredential1.secretAccessKey());
Assertions.assertEquals(expireTime,
ossSecretKeyCredential1.expireTimeInMs());
}
+
+ @Test
+ void testADLSTokenCredential() {
+ String storageAccountName = "storage-account-name";
+ String sasToken = "sas-token";
+
+ Map<String, String> adlsTokenCredentialInfo =
+ ImmutableMap.of(
+ ADLSTokenCredential.GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME,
+ storageAccountName,
+ ADLSTokenCredential.GRAVITINO_ADLS_SAS_TOKEN,
+ sasToken);
+ long expireTime = 100;
+ Credential credential =
+ CredentialFactory.create(
+ ADLSTokenCredential.ADLS_SAS_TOKEN_CREDENTIAL_TYPE,
+ adlsTokenCredentialInfo,
+ expireTime);
+ Assertions.assertEquals(
+ ADLSTokenCredential.ADLS_SAS_TOKEN_CREDENTIAL_TYPE,
credential.credentialType());
+ Assertions.assertInstanceOf(ADLSTokenCredential.class, credential);
+
+ ADLSTokenCredential adlsTokenCredential = (ADLSTokenCredential) credential;
+ Assertions.assertEquals(storageAccountName,
adlsTokenCredential.accountName());
+ Assertions.assertEquals(sasToken, adlsTokenCredential.sasToken());
+ Assertions.assertEquals(expireTime, adlsTokenCredential.expireTimeInMs());
+ }
}
diff --git
a/common/src/test/java/org/apache/gravitino/credential/TestCredentialPropertiesUtils.java
b/common/src/test/java/org/apache/gravitino/credential/TestCredentialPropertiesUtils.java
index 8e52b1684..cb5eaabe7 100644
---
a/common/src/test/java/org/apache/gravitino/credential/TestCredentialPropertiesUtils.java
+++
b/common/src/test/java/org/apache/gravitino/credential/TestCredentialPropertiesUtils.java
@@ -55,7 +55,7 @@ public class TestCredentialPropertiesUtils {
@Test
void testToIcebergPropertiesForOSS() {
OSSTokenCredential ossTokenCredential =
- new OSSTokenCredential("key", "secret", "security-token", 0);
+ new OSSTokenCredential("key", "secret", "security-token", 100);
Map<String, String> icebergProperties =
CredentialPropertyUtils.toIcebergProperties(ossTokenCredential);
Map<String, String> expectedProperties =
@@ -68,4 +68,26 @@ public class TestCredentialPropertiesUtils {
"security-token");
Assertions.assertEquals(expectedProperties, icebergProperties);
}
+
+ @Test
+ void testToIcebergPropertiesForADLS() {
+ String storageAccountName = "storage-account-name";
+ String sasToken = "sas-token";
+ long expireTimeInMS = 100;
+
+ ADLSTokenCredential adlsTokenCredential =
+ new ADLSTokenCredential(storageAccountName, sasToken, expireTimeInMS);
+ Map<String, String> icebergProperties =
+ CredentialPropertyUtils.toIcebergProperties(adlsTokenCredential);
+
+ String sasTokenKey =
+ String.format(
+ "%s.%s.%s",
+ CredentialPropertyUtils.ICEBERG_ADLS_TOKEN,
+ storageAccountName,
+ ADLSTokenCredential.ADLS_DOMAIN);
+
+ Map<String, String> expectedProperties = ImmutableMap.of(sasTokenKey,
sasToken);
+ Assertions.assertEquals(expectedProperties, icebergProperties);
+ }
}
diff --git
a/core/src/main/java/org/apache/gravitino/credential/config/ADLSCredentialConfig.java
b/core/src/main/java/org/apache/gravitino/credential/config/ADLSCredentialConfig.java
new file mode 100644
index 000000000..e9d368e67
--- /dev/null
+++
b/core/src/main/java/org/apache/gravitino/credential/config/ADLSCredentialConfig.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.credential.config;
+
+import java.util.Map;
+import javax.validation.constraints.NotNull;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.gravitino.Config;
+import org.apache.gravitino.config.ConfigBuilder;
+import org.apache.gravitino.config.ConfigConstants;
+import org.apache.gravitino.config.ConfigEntry;
+import org.apache.gravitino.credential.CredentialConstants;
+import org.apache.gravitino.storage.AzureProperties;
+
+public class ADLSCredentialConfig extends Config {
+
+ public static final ConfigEntry<String> AZURE_STORAGE_ACCOUNT_NAME =
+ new ConfigBuilder(AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME)
+ .doc("The name of the Azure Data Lake Storage account.")
+ .version(ConfigConstants.VERSION_0_7_0)
+ .stringConf()
+ .checkValue(StringUtils::isNotBlank,
ConfigConstants.NOT_BLANK_ERROR_MSG)
+ .create();
+
+ public static final ConfigEntry<String> AZURE_STORAGE_ACCOUNT_KEY =
+ new ConfigBuilder(AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY)
+ .doc("The key of the Azure Data Lake Storage account.")
+ .version(ConfigConstants.VERSION_0_7_0)
+ .stringConf()
+ .checkValue(StringUtils::isNotBlank,
ConfigConstants.NOT_BLANK_ERROR_MSG)
+ .create();
+
+ public static final ConfigEntry<String> AZURE_TENANT_ID =
+ new ConfigBuilder(AzureProperties.GRAVITINO_AZURE_TENANT_ID)
+ .doc("The Azure Active Directory (AAD) tenant ID used for
authentication.")
+ .version(ConfigConstants.VERSION_0_7_0)
+ .stringConf()
+ .checkValue(StringUtils::isNotBlank,
ConfigConstants.NOT_BLANK_ERROR_MSG)
+ .create();
+
+ public static final ConfigEntry<String> AZURE_CLIENT_ID =
+ new ConfigBuilder(AzureProperties.GRAVITINO_AZURE_CLIENT_ID)
+ .doc("The client ID used for authenticating with Azure Active
Directory (AAD).")
+ .version(ConfigConstants.VERSION_0_7_0)
+ .stringConf()
+ .checkValue(StringUtils::isNotBlank,
ConfigConstants.NOT_BLANK_ERROR_MSG)
+ .create();
+
+ public static final ConfigEntry<String> AZURE_CLIENT_SECRET =
+ new ConfigBuilder(AzureProperties.GRAVITINO_AZURE_CLIENT_SECRET)
+ .doc("The client secret used for authenticating with Azure Active
Directory (AAD).")
+ .version(ConfigConstants.VERSION_0_7_0)
+ .stringConf()
+ .checkValue(StringUtils::isNotBlank,
ConfigConstants.NOT_BLANK_ERROR_MSG)
+ .create();
+
+ public static final ConfigEntry<Integer> ADLS_TOKEN_EXPIRE_IN_SECS =
+ new ConfigBuilder(CredentialConstants.ADLS_TOKEN_EXPIRE_IN_SECS)
+ .doc(
+ "The expiration time (in seconds) for the Azure Active Directory
(AAD) authentication token.")
+ .version(ConfigConstants.VERSION_0_7_0)
+ .intConf()
+ .createWithDefault(3600);
+
+ public ADLSCredentialConfig(Map<String, String> properties) {
+ super(false);
+ loadFromMap(properties, k -> true);
+ }
+
+ @NotNull
+ public String storageAccountName() {
+ return this.get(AZURE_STORAGE_ACCOUNT_NAME);
+ }
+
+ @NotNull
+ public String storageAccountKey() {
+ return this.get(AZURE_STORAGE_ACCOUNT_KEY);
+ }
+
+ @NotNull
+ public String tenantId() {
+ return this.get(AZURE_TENANT_ID);
+ }
+
+ @NotNull
+ public String clientId() {
+ return this.get(AZURE_CLIENT_ID);
+ }
+
+ @NotNull
+ public String clientSecret() {
+ return this.get(AZURE_CLIENT_SECRET);
+ }
+
+ @NotNull
+ public Integer tokenExpireInSecs() {
+ return this.get(ADLS_TOKEN_EXPIRE_IN_SECS);
+ }
+}
diff --git a/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh
b/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh
index 5d0015786..2235313dc 100755
--- a/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh
+++ b/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh
@@ -37,12 +37,14 @@ cp -r gravitino-iceberg-rest-server*-bin
${iceberg_rest_server_dir}/packages/gra
cd ${gravitino_home}
./gradlew :bundles:gcp-bundle:jar
./gradlew :bundles:aws-bundle:jar
+./gradlew :bundles:azure-bundle:jar
# prepare bundle jar
cd ${iceberg_rest_server_dir}
mkdir -p bundles
cp ${gravitino_home}/bundles/gcp-bundle/build/libs/gravitino-gcp-bundle-*.jar
bundles/
cp ${gravitino_home}/bundles/aws-bundle/build/libs/gravitino-aws-bundle-*.jar
bundles/
+cp
${gravitino_home}/bundles/azure-bundle/build/libs/gravitino-azure-bundle-*.jar
bundles/
iceberg_gcp_bundle="iceberg-gcp-bundle-1.5.2.jar"
if [ ! -f "bundles/${iceberg_gcp_bundle}" ]; then
@@ -54,6 +56,11 @@ if [ ! -f "bundles/${iceberg_aws_bundle}" ]; then
curl -L -s -o bundles/${iceberg_aws_bundle}
https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/1.5.2/${iceberg_aws_bundle}
fi
+iceberg_azure_bundle="iceberg-azure-bundle-1.5.2.jar"
+if [ ! -f "bundles/${iceberg_azure_bundle}" ]; then
+ curl -L -s -o bundles/${iceberg_azure_bundle}
https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-azure-bundle/1.5.2/${iceberg_azure_bundle}
+fi
+
# download jdbc driver
curl -L -s -o bundles/sqlite-jdbc-3.42.0.0.jar
https://repo1.maven.org/maven2/org/xerial/sqlite-jdbc/3.42.0.0/sqlite-jdbc-3.42.0.0.jar
diff --git a/dev/docker/iceberg-rest-server/rewrite_config.py
b/dev/docker/iceberg-rest-server/rewrite_config.py
index 9e3441d25..624c67750 100755
--- a/dev/docker/iceberg-rest-server/rewrite_config.py
+++ b/dev/docker/iceberg-rest-server/rewrite_config.py
@@ -28,7 +28,12 @@ env_map = {
"GRAVITINO_S3_SECRET_KEY" : "s3-secret-access-key",
"GRAVITINO_S3_REGION" : "s3-region",
"GRAVITINO_S3_ROLE_ARN" : "s3-role-arn",
- "GRAVITINO_S3_EXTERNAL_ID" : "s3-external-id"
+ "GRAVITINO_S3_EXTERNAL_ID" : "s3-external-id",
+ "GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME" : "azure-storage-account-name",
+ "GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY" : "azure-storage-account-key",
+ "GRAVITINO_AZURE_TENANT_ID" : "azure-tenant-id",
+ "GRAVITINO_AZURE_CLIENT_ID" : "azure-client-id",
+ "GRAVITINO_AZURE_CLIENT_SECRET" : "azure-client-secret",
}
init_config = {
diff --git a/docs/hadoop-catalog.md b/docs/hadoop-catalog.md
index 26d27dce8..ce58826cb 100644
--- a/docs/hadoop-catalog.md
+++ b/docs/hadoop-catalog.md
@@ -79,12 +79,12 @@ In the meantime, you need to place the corresponding bundle
jar [`gravitino-aliy
#### Azure Blob Storage fileset
-| Configuration item | Description
| Default value | Required
| Since version |
-|-------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------|-------------------------------------------|------------------|
-| `filesystem-providers` | The file system providers to add. Set it to
`abs` if it's a Azure Blob Storage fileset, or a comma separated string that
contains `abs` like `oss,abs,s3` to support multiple kinds of fileset including
`abs`. | (none) | Yes
| 0.8.0-incubating |
-| `default-filesystem-provider` | The name default filesystem providers of
this Hadoop catalog if users do not specify the scheme in the URI. Default
value is `builtin-local`, for Azure Blob Storage, if we set this value, we can
omit the prefix 'abfss://' in the location. | `builtin-local` | No
| 0.8.0-incubating |
-| `abs-account-name` | The account name of Azure Blob Storage.
| (none) | Yes if it's a Azure Blob
Storage fileset. | 0.8.0-incubating |
-| `abs-account-key` | The account key of Azure Blob Storage.
| (none) | Yes if it's a Azure Blob
Storage fileset. | 0.8.0-incubating |
+| Configuration item | Description
| Default value | Required
| Since version |
+|-----------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------|-------------------------------------------|------------------|
+| `filesystem-providers` | The file system providers to add. Set it
to `abs` if it's a Azure Blob Storage fileset, or a comma separated string that
contains `abs` like `oss,abs,s3` to support multiple kinds of fileset including
`abs`. | (none) | Yes
| 0.8.0-incubating |
+| `default-filesystem-provider` | The name default filesystem providers of
this Hadoop catalog if users do not specify the scheme in the URI. Default
value is `builtin-local`, for Azure Blob Storage, if we set this value, we can
omit the prefix 'abfss://' in the location. | `builtin-local` | No
| 0.8.0-incubating |
+| `azure-storage-account-name ` | The account name of Azure Blob Storage.
| (none) | Yes if it's a Azure
Blob Storage fileset. | 0.8.0-incubating |
+| `azure-storage-account-key` | The account key of Azure Blob Storage.
| (none) | Yes if it's a Azure
Blob Storage fileset. | 0.8.0-incubating |
Similar to the above, you need to place the corresponding bundle jar
[`gravitino-azure-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/azure-bundle/)
in the directory `${GRAVITINO_HOME}/catalogs/hadoop/libs`.
diff --git a/docs/how-to-use-gvfs.md b/docs/how-to-use-gvfs.md
index 34835ec8d..162d535be 100644
--- a/docs/how-to-use-gvfs.md
+++ b/docs/how-to-use-gvfs.md
@@ -101,10 +101,10 @@ In the meantime, you need to place the corresponding
bundle jar [`gravitino-aliy
#### Azure Blob Storage fileset
-| Configuration item | Description
| Default value | Required | Since version
|
-|--------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|-------------------------------------------|------------------|
-| `abs-account-name` | The account name of Azure Blob Storage.
| (none) | Yes if it's a Azure Blob Storage fileset. |
0.8.0-incubating |
-| `abs-account-key` | The account key of Azure Blob Storage.
| (none) | Yes if it's a Azure Blob Storage fileset. |
0.8.0-incubating |
+| Configuration item | Description
| Default value | Required | Since
version |
+|-----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|-------------------------------------------|------------------|
+| `azure-storage-account-name` | The account name of Azure Blob Storage.
| (none) | Yes if it's a Azure Blob Storage fileset. |
0.8.0-incubating |
+| `azure-storage-account-key` | The account key of Azure Blob Storage.
| (none) | Yes if it's a Azure Blob Storage fileset. |
0.8.0-incubating |
Similar to the above, you need to place the corresponding bundle jar
[`gravitino-azure-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/azure-bundle/)
in the Hadoop environment(typically located in
`${HADOOP_HOME}/share/hadoop/common/lib/`).
diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md
index 862bb0486..8d9d49745 100644
--- a/docs/iceberg-rest-service.md
+++ b/docs/iceberg-rest-service.md
@@ -17,8 +17,8 @@ The Apache Gravitino Iceberg REST Server follows the [Apache
Iceberg REST API sp
- multi table transaction
- pagination
- Works as a catalog proxy, supporting `Hive` and `JDBC` as catalog backend.
-- Supports credential vending for `S3`、`GCS` and `OSS`.
-- Supports different storages like `S3`, `HDFS`, `OSS`, `GCS` and provides the
capability to support other storages.
+- Supports credential vending for `S3`、`GCS`、`OSS` and `ADLS`.
+- Supports different storages like `S3`, `HDFS`, `OSS`, `GCS`, `ADLS` and
provides the capability to support other storages.
- Supports event listener.
- Supports Audit log.
- Supports OAuth2 and HTTPS.
@@ -171,6 +171,28 @@ Please make sure the credential file is accessible by
Gravitino, like using `exp
Please set `gravitino.iceberg-rest.warehouse` to
`gs://{bucket_name}/${prefix_name}`, and download [Iceberg gcp
bundle](https://mvnrepository.com/artifact/org.apache.iceberg/iceberg-gcp-bundle)
and place it to the classpath of Gravitino Iceberg REST server,
`iceberg-rest-server/libs` for the auxiliary server, `libs` for the standalone
server.
:::
+#### ADLS
+
+Gravitino Iceberg REST service supports generating SAS token to access ADLS
data.
+
+| Configuration item | Description
| Default value | Required | Since Version |
+|-----------------------------------------------------|-----------------------------------------------------------------------------------------------------------|---------------|----------|------------------|
+| `gravitino.iceberg-rest.io-impl` | The IO implementation
for `FileIO` in Iceberg, use `org.apache.iceberg.azure.adlsv2.ADLSFileIO` for
ADLS. | (none) | Yes | 0.8.0-incubating |
+| `gravitino.iceberg-rest.credential-provider-type` | Supports `adls-token`,
generates a temporary token according to the query data path.
| (none) | Yes | 0.8.0-incubating |
+| `gravitino.iceberg-rest.azure-storage-account-name` | The static storage
account name used to access ADLS data.
| (none) | Yes | 0.8.0-incubating |
+| `gravitino.iceberg-rest.azure-storage-account-key` | The static storage
account key used to access ADLS data.
| (none) | Yes | 0.8.0-incubating |
+| `gravitino.iceberg-rest.azure-tenant-id` | Azure Active Directory
(AAD) tenant ID.
| (none) | Yes | 0.8.0-incubating |
+| `gravitino.iceberg-rest.azure-client-id` | Azure Active Directory
(AAD) client ID used for authentication.
| (none) | Yes | 0.8.0-incubating |
+| `gravitino.iceberg-rest.azure-client-secret` | Azure Active Directory
(AAD) client secret used for authentication.
| (none) | Yes | 0.8.0-incubating |
+
+For other Iceberg ADLS properties not managed by Gravitino like
`adls.read.block-size-bytes`, you could config it directly by
`gravitino.iceberg-rest.adls.read.block-size-bytes`.
+
+If you set `credential-provider-type` explicitly, please downloading
[Gravitino Azure bundle
jar](https://mvnrepository.com/artifact/org.apache.gravitino/azure-bundle), and
place it to the classpath of Iceberg REST server.
+
+:::info
+Please set `gravitino.iceberg-rest.warehouse` to
`abfs[s]://{container-name}@{storage-account-name}.dfs.core.windows.net/{path}`,
and download the [Iceberg Azure
bundle](https://mvnrepository.com/artifact/org.apache.iceberg/iceberg-azure-bundle)
and place it in the classpath of Iceberg REST server.
+:::
+
#### HDFS configuration
You should place HDFS configuration file to the classpath of the Iceberg REST
server, `iceberg-rest-server/conf` for Gravitino server package, `conf` for
standalone Gravitino Iceberg REST server package. When writing to HDFS, the
Gravitino Iceberg REST catalog service can only operate as the specified HDFS
user and doesn't support proxying to other HDFS users. See [How to access
Apache Hadoop](gravitino-server-config.md#how-to-access-apache-hadoop) for more
details.
@@ -418,19 +440,24 @@ docker run -d -p 9001:9001
apache/gravitino-iceberg-rest:0.7.0-incubating
Gravitino Iceberg REST server in docker image could access local storage by
default, you could set the following environment variables if the storage is
cloud/remote storage like S3, please refer to [storage section](#storage) for
more details.
-| Environment variables | Configuration items
| Since version |
-|------------------------------------------------|---------------------------------------------------|-------------------|
-| `GRAVITINO_IO_IMPL` |
`gravitino.iceberg-rest.io-impl` | 0.7.0-incubating |
-| `GRAVITINO_URI` |
`gravitino.iceberg-rest.uri` | 0.7.0-incubating |
-| `GRAVITINO_WAREHOUSE` |
`gravitino.iceberg-rest.warehouse` | 0.7.0-incubating |
-| `GRAVITINO_CREDENTIAL_PROVIDER_TYPE` |
`gravitino.iceberg-rest.credential-provider-type` | 0.7.0-incubating |
-| `GRAVITINO_GCS_CREDENTIAL_FILE_PATH` |
`gravitino.iceberg-rest.gcs-credential-file-path` | 0.7.0-incubating |
-| `GRAVITINO_S3_ACCESS_KEY` |
`gravitino.iceberg-rest.s3-access-key-id` | 0.7.0-incubating |
-| `GRAVITINO_S3_SECRET_KEY` |
`gravitino.iceberg-rest.s3-secret-access-key` | 0.7.0-incubating |
-| `GRAVITINO_S3_REGION` |
`gravitino.iceberg-rest.s3-region` | 0.7.0-incubating |
-| `GRAVITINO_S3_ROLE_ARN` |
`gravitino.iceberg-rest.s3-role-arn` | 0.7.0-incubating |
-| `GRAVITINO_S3_EXTERNAL_ID` |
`gravitino.iceberg-rest.s3-external-id` | 0.7.0-incubating |
-| `GRAVITINO_S3_TOKEN_SERVICE_ENDPOINT` |
`gravitino.iceberg-rest.s3-token-service-endpoint`| 0.8.0-incubating |
+| Environment variables | Configuration items
| Since version |
+|-----------------------------------------|-----------------------------------------------------|-------------------|
+| `GRAVITINO_IO_IMPL` | `gravitino.iceberg-rest.io-impl`
| 0.7.0-incubating |
+| `GRAVITINO_URI` | `gravitino.iceberg-rest.uri`
| 0.7.0-incubating |
+| `GRAVITINO_WAREHOUSE` | `gravitino.iceberg-rest.warehouse`
| 0.7.0-incubating |
+| `GRAVITINO_CREDENTIAL_PROVIDER_TYPE` |
`gravitino.iceberg-rest.credential-provider-type` | 0.7.0-incubating |
+| `GRAVITINO_GCS_CREDENTIAL_FILE_PATH` |
`gravitino.iceberg-rest.gcs-credential-file-path` | 0.7.0-incubating |
+| `GRAVITINO_S3_ACCESS_KEY` |
`gravitino.iceberg-rest.s3-access-key-id` | 0.7.0-incubating |
+| `GRAVITINO_S3_SECRET_KEY` |
`gravitino.iceberg-rest.s3-secret-access-key` | 0.7.0-incubating |
+| `GRAVITINO_S3_REGION` | `gravitino.iceberg-rest.s3-region`
| 0.7.0-incubating |
+| `GRAVITINO_S3_ROLE_ARN` |
`gravitino.iceberg-rest.s3-role-arn` | 0.7.0-incubating |
+| `GRAVITINO_S3_EXTERNAL_ID` |
`gravitino.iceberg-rest.s3-external-id` | 0.7.0-incubating |
+| `GRAVITINO_S3_TOKEN_SERVICE_ENDPOINT` |
`gravitino.iceberg-rest.s3-token-service-endpoint` | 0.8.0-incubating |
+| `GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME` |
`gravitino.iceberg-rest.azure-storage-account-name` | 0.8.0-incubating |
+| `GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY` |
`gravitino.iceberg-rest.azure-storage-account-key` | 0.8.0-incubating |
+| `GRAVITINO_AZURE_TENANT_ID` |
`gravitino.iceberg-rest.azure-tenant-id` | 0.8.0-incubating |
+| `GRAVITINO_AZURE_CLIENT_ID` |
`gravitino.iceberg-rest.azure-client-id` | 0.8.0-incubating |
+| `GRAVITINO_AZURE_CLIENT_SECRET` |
`gravitino.iceberg-rest.azure-client-secret` | 0.8.0-incubating |
Or build it manually to add custom configuration or logics:
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index 630550a7e..a33c300ee 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -18,6 +18,10 @@
#
[versions]
awssdk = "2.28.3"
+azure-identity = "1.13.1"
+azure-storage-file-datalake = "12.20.0"
+reactor-netty-http = "1.2.1"
+reactor-netty-core = "1.2.1"
junit = "5.8.1"
protoc = "3.24.4"
jackson = "2.15.2"
@@ -120,6 +124,10 @@ aws-policy = { group = "software.amazon.awssdk", name =
"iam-policy-builder", ve
aws-s3 = { group = "software.amazon.awssdk", name = "s3", version.ref =
"awssdk" }
aws-sts = { group = "software.amazon.awssdk", name = "sts", version.ref =
"awssdk" }
aws-kms = { group = "software.amazon.awssdk", name = "kms", version.ref =
"awssdk" }
+azure-identity = { group = "com.azure", name = "azure-identity", version.ref =
"azure-identity"}
+azure-storage-file-datalake = { group = "com.azure", name =
"azure-storage-file-datalake", version.ref = "azure-storage-file-datalake"}
+reactor-netty-http = {group = "io.projectreactor.netty", name =
"reactor-netty-http", version.ref = "reactor-netty-http"}
+reactor-netty-core = {group = "io.projectreactor.netty", name =
"reactor-netty-core", version.ref = "reactor-netty-core"}
protobuf-java = { group = "com.google.protobuf", name = "protobuf-java",
version.ref = "protoc" }
protobuf-java-util = { group = "com.google.protobuf", name =
"protobuf-java-util", version.ref = "protoc" }
jackson-databind = { group = "com.fasterxml.jackson.core", name =
"jackson-databind", version.ref = "jackson" }
@@ -191,6 +199,8 @@ commons-configuration1 = { group = "commons-configuration",
name = "commons-conf
iceberg-aliyun = { group = "org.apache.iceberg", name = "iceberg-aliyun",
version.ref = "iceberg" }
iceberg-aws = { group = "org.apache.iceberg", name = "iceberg-aws",
version.ref = "iceberg" }
iceberg-aws-bundle = { group = "org.apache.iceberg", name =
"iceberg-aws-bundle", version.ref = "iceberg" }
+iceberg-azure = { group = "org.apache.iceberg", name = "iceberg-azure",
version.ref = "iceberg" }
+iceberg-azure-bundle = { group = "org.apache.iceberg", name =
"iceberg-azure-bundle", version.ref = "iceberg" }
iceberg-core = { group = "org.apache.iceberg", name = "iceberg-core",
version.ref = "iceberg" }
iceberg-api = { group = "org.apache.iceberg", name = "iceberg-api",
version.ref = "iceberg" }
iceberg-hive-metastore = { group = "org.apache.iceberg", name =
"iceberg-hive-metastore", version.ref = "iceberg" }
diff --git a/iceberg/iceberg-common/build.gradle.kts
b/iceberg/iceberg-common/build.gradle.kts
index abc9a05a5..b67e04238 100644
--- a/iceberg/iceberg-common/build.gradle.kts
+++ b/iceberg/iceberg-common/build.gradle.kts
@@ -44,6 +44,7 @@ dependencies {
implementation(libs.guava)
implementation(libs.iceberg.aliyun)
implementation(libs.iceberg.aws)
+ implementation(libs.iceberg.azure)
implementation(libs.iceberg.hive.metastore)
implementation(libs.iceberg.gcp)
implementation(libs.hadoop2.common) {
diff --git a/iceberg/iceberg-rest-server/build.gradle.kts
b/iceberg/iceberg-rest-server/build.gradle.kts
index e46193c97..03fe32c92 100644
--- a/iceberg/iceberg-rest-server/build.gradle.kts
+++ b/iceberg/iceberg-rest-server/build.gradle.kts
@@ -65,6 +65,7 @@ dependencies {
testImplementation(project(":bundles:aliyun-bundle"))
testImplementation(project(":bundles:aws-bundle"))
testImplementation(project(":bundles:gcp-bundle", configuration = "shadow"))
+ testImplementation(project(":bundles:azure-bundle"))
testImplementation(project(":integration-test-common", "testArtifacts"))
testImplementation("org.scala-lang.modules:scala-collection-compat_$scalaVersion:$scalaCollectionCompatVersion")
@@ -79,6 +80,13 @@ dependencies {
testImplementation(libs.iceberg.aws.bundle)
testImplementation(libs.iceberg.gcp.bundle)
+ // Prevent netty conflict
+ testImplementation(libs.reactor.netty.http)
+ testImplementation(libs.reactor.netty.core)
+ testImplementation(libs.iceberg.azure.bundle) {
+ exclude("io.netty")
+ exclude("com.google.guava", "guava")
+ }
testImplementation(libs.jersey.test.framework.core) {
exclude(group = "org.junit.jupiter")
}
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSIT.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSIT.java
new file mode 100644
index 000000000..570298d05
--- /dev/null
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSIT.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.iceberg.integration.test;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.gravitino.abs.credential.ADLSLocationUtils;
+import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants;
+import org.apache.gravitino.credential.CredentialConstants;
+import org.apache.gravitino.iceberg.common.IcebergConfig;
+import org.apache.gravitino.integration.test.util.BaseIT;
+import org.apache.gravitino.integration.test.util.DownloaderUtils;
+import org.apache.gravitino.integration.test.util.ITUtils;
+import org.apache.gravitino.storage.AzureProperties;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
+
+@SuppressWarnings("FormatStringAnnotation")
+@EnabledIfEnvironmentVariable(named = "GRAVITINO_TEST_CLOUD_IT", matches =
"true")
+public class IcebergRESTADLSIT extends IcebergRESTJdbcCatalogIT {
+
+ private String storageAccountName;
+ private String storageAccountKey;
+ private String tenantId;
+ private String clientId;
+ private String clientSecret;
+ private String warehousePath;
+
+ @Override
+ void initEnv() {
+ this.storageAccountName =
+ System.getenv()
+ .getOrDefault("GRAVITINO_ADLS_STORAGE_ACCOUNT_NAME",
"{STORAGE_ACCOUNT_NAME}");
+ this.storageAccountKey =
+ System.getenv().getOrDefault("GRAVITINO_ADLS_STORAGE_ACCOUNT_KEY",
"{STORAGE_ACCOUNT_KEY}");
+ this.tenantId = System.getenv().getOrDefault("GRAVITINO_ADLS_TENANT_ID",
"{TENANT_ID}");
+ this.clientId = System.getenv().getOrDefault("GRAVITINO_ADLS_CLIENT_ID",
"{CLIENT_ID}");
+ this.clientSecret =
+ System.getenv().getOrDefault("GRAVITINO_ADLS_CLIENT_SECRET",
"{CLIENT_SECRET}");
+ this.warehousePath =
+ String.format(
+ "abfss://%s@%s.dfs.core.windows.net/data/test",
+ System.getenv().getOrDefault("GRAVITINO_ADLS_CONTAINER",
"{ADLS_CONTAINER}"),
+ storageAccountName);
+
+ if (ITUtils.isEmbedded()) {
+ return;
+ }
+ try {
+ downloadIcebergAzureBundleJar();
+ } catch (IOException e) {
+ LOG.warn("Download Iceberg Azure bundle jar failed,", e);
+ throw new RuntimeException(e);
+ }
+ copyAzureBundleJar();
+ }
+
+ @Override
+ public Map<String, String> getCatalogConfig() {
+ HashMap m = new HashMap<String, String>();
+ m.putAll(getCatalogJdbcConfig());
+ m.putAll(getADLSConfig());
+ return m;
+ }
+
+ public boolean supportsCredentialVending() {
+ return true;
+ }
+
+ private Map<String, String> getADLSConfig() {
+ Map configMap = new HashMap<String, String>();
+
+ configMap.put(
+ IcebergConfig.ICEBERG_CONFIG_PREFIX +
CredentialConstants.CREDENTIAL_PROVIDER_TYPE,
+ CredentialConstants.ADLS_TOKEN_CREDENTIAL_PROVIDER_TYPE);
+ configMap.put(
+ IcebergConfig.ICEBERG_CONFIG_PREFIX +
AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME,
+ storageAccountName);
+ configMap.put(
+ IcebergConfig.ICEBERG_CONFIG_PREFIX +
AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY,
+ storageAccountKey);
+ configMap.put(
+ IcebergConfig.ICEBERG_CONFIG_PREFIX +
AzureProperties.GRAVITINO_AZURE_TENANT_ID, tenantId);
+ configMap.put(
+ IcebergConfig.ICEBERG_CONFIG_PREFIX +
AzureProperties.GRAVITINO_AZURE_CLIENT_ID, clientId);
+ configMap.put(
+ IcebergConfig.ICEBERG_CONFIG_PREFIX +
AzureProperties.GRAVITINO_AZURE_CLIENT_SECRET,
+ clientSecret);
+
+ configMap.put(
+ IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConstants.IO_IMPL,
+ "org.apache.iceberg.azure.adlsv2.ADLSFileIO");
+ configMap.put(IcebergConfig.ICEBERG_CONFIG_PREFIX +
IcebergConstants.WAREHOUSE, warehousePath);
+
+ return configMap;
+ }
+
+ private void downloadIcebergAzureBundleJar() throws IOException {
+ String icebergBundleJarName = "iceberg-azure-bundle-1.5.2.jar";
+ String icebergBundleJarUri =
+ "https://repo1.maven.org/maven2/org/apache/iceberg/";
+ + "iceberg-azure-bundle/1.5.2/"
+ + icebergBundleJarName;
+ String gravitinoHome = System.getenv("GRAVITINO_HOME");
+ String targetDir = String.format("%s/iceberg-rest-server/libs/",
gravitinoHome);
+ DownloaderUtils.downloadFile(icebergBundleJarUri, targetDir);
+ }
+
+ private void copyAzureBundleJar() {
+ String gravitinoHome = System.getenv("GRAVITINO_HOME");
+ String targetDir = String.format("%s/iceberg-rest-server/libs/",
gravitinoHome);
+ BaseIT.copyBundleJarsToDirectory("azure-bundle", targetDir);
+ }
+
+ @Test
+ public void testParseLocationValidInput() {
+ String location =
"abfss://[email protected]/data/test/path";
+
+ ADLSLocationUtils.ADLSLocationParts parts =
ADLSLocationUtils.parseLocation(location);
+
+ Assertions.assertEquals("container", parts.getContainer(), "Container name
should match");
+ Assertions.assertEquals("account", parts.getAccountName(), "Account name
should match");
+ Assertions.assertEquals("/data/test/path", parts.getPath(), "Path should
match");
+ }
+
+ @Test
+ public void testParseLocationInvalidInput() {
+ String location = "abfss://container/invalid/location";
+
+ Exception exception =
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () -> {
+ ADLSLocationUtils.parseLocation(location);
+ });
+
+ Assertions.assertTrue(
+ exception.getMessage().contains("Invalid location"),
+ "Exception message should indicate invalid location");
+ }
+
+ @Test
+ public void testTrimSlashesNullInput() {
+ Assertions.assertNull(ADLSLocationUtils.trimSlashes(null), "Null input
should return null");
+ }
+
+ @Test
+ public void testTrimSlashesEmptyInput() {
+ Assertions.assertEquals(
+ "", ADLSLocationUtils.trimSlashes(""), "Empty input should return
empty string");
+ }
+
+ @Test
+ public void testTrimSlashesNoSlashes() {
+ String input = "data/test/path";
+ Assertions.assertEquals(
+ "data/test/path",
+ ADLSLocationUtils.trimSlashes(input),
+ "Input without slashes should remain unchanged");
+ }
+
+ @Test
+ public void testTrimSlashesLeadingAndTrailingSlashes() {
+ String input = "/data/test/path/";
+ Assertions.assertEquals(
+ "data/test/path",
+ ADLSLocationUtils.trimSlashes(input),
+ "Leading and trailing slashes should be trimmed");
+ }
+
+ @Test
+ public void testTrimSlashesMultipleLeadingAndTrailingSlashes() {
+ String input = "///data/test/path///";
+ Assertions.assertEquals(
+ "data/test/path",
+ ADLSLocationUtils.trimSlashes(input),
+ "Multiple leading and trailing slashes should be trimmed");
+ }
+
+ @Test
+ public void testTrimSlashesOnlySlashes() {
+ String input = "////";
+ Assertions.assertEquals(
+ "", ADLSLocationUtils.trimSlashes(input), "Only slashes should result
in an empty string");
+ }
+}
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSIT.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSIT.java
index 89f56c517..8f7821cb4 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSIT.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSIT.java
@@ -22,7 +22,6 @@ package org.apache.gravitino.iceberg.integration.test;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
-import java.util.Optional;
import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants;
import org.apache.gravitino.credential.CredentialConstants;
import org.apache.gravitino.credential.config.GCSCredentialConfig;
@@ -41,9 +40,10 @@ public class IcebergRESTGCSIT extends
IcebergRESTJdbcCatalogIT {
@Override
void initEnv() {
this.gcsWarehouse =
- String.format("gs://%s/test",
getFromEnvOrDefault("GRAVITINO_GCS_BUCKET", "bucketName"));
+ String.format(
+ "gs://%s/test",
System.getenv().getOrDefault("GRAVITINO_GCS_BUCKET", "bucketName"));
this.gcsCredentialPath =
- getFromEnvOrDefault("GOOGLE_APPLICATION_CREDENTIALS",
"credential.json");
+ System.getenv().getOrDefault("GOOGLE_APPLICATION_CREDENTIALS",
"credential.json");
if (ITUtils.isEmbedded()) {
return;
}
@@ -100,9 +100,4 @@ public class IcebergRESTGCSIT extends
IcebergRESTJdbcCatalogIT {
String targetDir = String.format("%s/iceberg-rest-server/libs/",
gravitinoHome);
DownloaderUtils.downloadFile(icebergBundleJarUri, targetDir);
}
-
- private String getFromEnvOrDefault(String envVar, String defaultValue) {
- String envValue = System.getenv(envVar);
- return Optional.ofNullable(envValue).orElse(defaultValue);
- }
}
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java
index fb6bac65b..f3aaafabb 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java
@@ -22,7 +22,6 @@ package org.apache.gravitino.iceberg.integration.test;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
-import java.util.Optional;
import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants;
import org.apache.gravitino.credential.CredentialConstants;
import org.apache.gravitino.iceberg.common.IcebergConfig;
@@ -49,13 +48,14 @@ public class IcebergRESTOSSIT extends
IcebergRESTJdbcCatalogIT {
this.warehouse =
String.format(
"oss://%s/gravitino-test",
- getFromEnvOrDefault("GRAVITINO_OSS_BUCKET", "{BUCKET_NAME}"));
- this.accessKey = getFromEnvOrDefault("GRAVITINO_OSS_ACCESS_KEY",
"{ACCESS_KEY}");
- this.secretKey = getFromEnvOrDefault("GRAVITINO_OSS_SECRET_KEY",
"{SECRET_KEY}");
- this.endpoint = getFromEnvOrDefault("GRAVITINO_OSS_ENDPOINT",
"{GRAVITINO_OSS_ENDPOINT}");
- this.region = getFromEnvOrDefault("GRAVITINO_OSS_REGION",
"oss-cn-hangzhou");
- this.roleArn = getFromEnvOrDefault("GRAVITINO_OSS_ROLE_ARN", "{ROLE_ARN}");
- this.externalId = getFromEnvOrDefault("GRAVITINO_OSS_EXTERNAL_ID", "");
+ System.getenv().getOrDefault("GRAVITINO_OSS_BUCKET",
"{BUCKET_NAME}"));
+ this.accessKey = System.getenv().getOrDefault("GRAVITINO_OSS_ACCESS_KEY",
"{ACCESS_KEY}");
+ this.secretKey = System.getenv().getOrDefault("GRAVITINO_OSS_SECRET_KEY",
"{SECRET_KEY}");
+ this.endpoint =
+ System.getenv().getOrDefault("GRAVITINO_OSS_ENDPOINT",
"{GRAVITINO_OSS_ENDPOINT}");
+ this.region = System.getenv().getOrDefault("GRAVITINO_OSS_REGION",
"oss-cn-hangzhou");
+ this.roleArn = System.getenv().getOrDefault("GRAVITINO_OSS_ROLE_ARN",
"{ROLE_ARN}");
+ this.externalId =
System.getenv().getOrDefault("GRAVITINO_OSS_EXTERNAL_ID", "");
if (ITUtils.isEmbedded()) {
return;
@@ -127,9 +127,4 @@ public class IcebergRESTOSSIT extends
IcebergRESTJdbcCatalogIT {
String targetDir = String.format("%s/iceberg-rest-server/libs/",
gravitinoHome);
BaseIT.copyBundleJarsToDirectory("aliyun-bundle", targetDir);
}
-
- private String getFromEnvOrDefault(String envVar, String defaultValue) {
- String envValue = System.getenv(envVar);
- return Optional.ofNullable(envValue).orElse(defaultValue);
- }
}
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSSecretIT.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSSecretIT.java
index 73f6262d1..cd5c99c46 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSSecretIT.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSSecretIT.java
@@ -22,7 +22,6 @@ package org.apache.gravitino.iceberg.integration.test;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
-import java.util.Optional;
import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants;
import org.apache.gravitino.credential.CredentialConstants;
import org.apache.gravitino.credential.OSSSecretKeyCredential;
@@ -46,10 +45,11 @@ public class IcebergRESTOSSSecretIT extends
IcebergRESTJdbcCatalogIT {
this.warehouse =
String.format(
"oss://%s/gravitino-test",
- getFromEnvOrDefault("GRAVITINO_OSS_BUCKET", "{BUCKET_NAME}"));
- this.accessKey = getFromEnvOrDefault("GRAVITINO_OSS_ACCESS_KEY",
"{ACCESS_KEY}");
- this.secretKey = getFromEnvOrDefault("GRAVITINO_OSS_SECRET_KEY",
"{SECRET_KEY}");
- this.endpoint = getFromEnvOrDefault("GRAVITINO_OSS_ENDPOINT",
"{GRAVITINO_OSS_ENDPOINT}");
+ System.getenv().getOrDefault("GRAVITINO_OSS_BUCKET",
"{BUCKET_NAME}"));
+ this.accessKey = System.getenv().getOrDefault("GRAVITINO_OSS_ACCESS_KEY",
"{ACCESS_KEY}");
+ this.secretKey = System.getenv().getOrDefault("GRAVITINO_OSS_SECRET_KEY",
"{SECRET_KEY}");
+ this.endpoint =
+ System.getenv().getOrDefault("GRAVITINO_OSS_ENDPOINT",
"{GRAVITINO_OSS_ENDPOINT}");
if (ITUtils.isEmbedded()) {
return;
@@ -113,9 +113,4 @@ public class IcebergRESTOSSSecretIT extends
IcebergRESTJdbcCatalogIT {
String targetDir = String.format("%s/iceberg-rest-server/libs/",
gravitinoHome);
BaseIT.copyBundleJarsToDirectory("aliyun-bundle", targetDir);
}
-
- private String getFromEnvOrDefault(String envVar, String defaultValue) {
- String envValue = System.getenv(envVar);
- return Optional.ofNullable(envValue).orElse(defaultValue);
- }
}
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3IT.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3IT.java
index ab372f78c..d31278051 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3IT.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3IT.java
@@ -24,7 +24,6 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants;
import org.apache.gravitino.credential.CredentialConstants;
import org.apache.gravitino.iceberg.common.IcebergConfig;
@@ -52,12 +51,13 @@ public class IcebergRESTS3IT extends
IcebergRESTJdbcCatalogIT {
@Override
void initEnv() {
this.s3Warehouse =
- String.format("s3://%s/test1",
getFromEnvOrDefault("GRAVITINO_S3_BUCKET", "{BUCKET_NAME}"));
- this.accessKey = getFromEnvOrDefault("GRAVITINO_S3_ACCESS_KEY",
"{ACCESS_KEY}");
- this.secretKey = getFromEnvOrDefault("GRAVITINO_S3_SECRET_KEY",
"{SECRET_KEY}");
- this.region = getFromEnvOrDefault("GRAVITINO_S3_REGION", "ap-southeast-2");
- this.roleArn = getFromEnvOrDefault("GRAVITINO_S3_ROLE_ARN", "{ROLE_ARN}");
- this.externalId = getFromEnvOrDefault("GRAVITINO_S3_EXTERNAL_ID", "");
+ String.format(
+ "s3://%s/test1",
System.getenv().getOrDefault("GRAVITINO_S3_BUCKET", "{BUCKET_NAME}"));
+ this.accessKey = System.getenv().getOrDefault("GRAVITINO_S3_ACCESS_KEY",
"{ACCESS_KEY}");
+ this.secretKey = System.getenv().getOrDefault("GRAVITINO_S3_SECRET_KEY",
"{SECRET_KEY}");
+ this.region = System.getenv().getOrDefault("GRAVITINO_S3_REGION",
"ap-southeast-2");
+ this.roleArn = System.getenv().getOrDefault("GRAVITINO_S3_ROLE_ARN",
"{ROLE_ARN}");
+ this.externalId = System.getenv().getOrDefault("GRAVITINO_S3_EXTERNAL_ID",
"");
if (ITUtils.isEmbedded()) {
return;
}
@@ -126,11 +126,6 @@ public class IcebergRESTS3IT extends
IcebergRESTJdbcCatalogIT {
BaseIT.copyBundleJarsToDirectory("aws-bundle", targetDir);
}
- private String getFromEnvOrDefault(String envVar, String defaultValue) {
- String envValue = System.getenv(envVar);
- return Optional.ofNullable(envValue).orElse(defaultValue);
- }
-
/**
* Parses a string representing table properties into a map of key-value
pairs.
*
diff --git a/settings.gradle.kts b/settings.gradle.kts
index 5776d34fa..150acdb00 100644
--- a/settings.gradle.kts
+++ b/settings.gradle.kts
@@ -80,5 +80,5 @@ include("integration-test-common")
include(":bundles:aws-bundle")
include(":bundles:gcp-bundle")
include(":bundles:aliyun-bundle")
-include("bundles:azure-bundle")
+include(":bundles:azure-bundle")
include("catalogs:hadoop-common")
