This is an automated email from the ASF dual-hosted git repository.
diqiu50 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 2c930e8276 [#12450] improvement(catalog): Share AWS Glue credentials
provider (#12575)
2c930e8276 is described below
commit 2c930e8276f77c150e2cc328b37f17c5acca2643
Author: Xu Bai <[email protected]>
AuthorDate: Mon Sep 7 20:09:09 2026 +0800
[#12450] improvement(catalog): Share AWS Glue credentials provider (#12575)
### What changes were proposed in this pull request?
Move the duplicated AWS Glue credentials provider to `catalog-common`
and reuse it in the Glue catalog and Spark connector.
### Why are the changes needed?
The Glue catalog and Spark connector contain equivalent credentials
providers. Sharing the provider removes duplication and allows other
engine connectors to reuse it without depending on server- or
engine-specific modules.
Related to #12450
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
- Catalog common unit tests
- Glue catalog unit tests
- Spark connector common unit tests
- Spark 3.3 runtime JAR content verification
- Spotless checks
---
LICENSE | 4 --
catalogs/catalog-common/build.gradle.kts | 2 +
.../glue/GravitinoGlueCredentialsProvider.java | 37 +++++-----
.../glue/TestGravitinoGlueCredentialsProvider.java | 78 ++++++++++++++++++++++
.../glue/GravitinoGlueCredentialsProvider.java | 71 --------------------
.../connector/glue/GluePropertiesConverter.java | 3 +-
.../glue/TestGluePropertiesConverter.java | 15 ++++-
.../glue/TestGravitinoGlueCredentialsProvider.java | 51 --------------
8 files changed, 114 insertions(+), 147 deletions(-)
diff --git a/LICENSE b/LICENSE
index d983933f22..d45c5cc480 100644
--- a/LICENSE
+++ b/LICENSE
@@ -263,10 +263,6 @@
./lance/lance-common/src/main/java/org/apache/gravitino/lance/common/ops/gravitino/JsonArrowSchemaConverter.java
./lance/lance-common/src/main/java/org/apache/gravitino/lance/common/ops/gravitino/PageUtil.java
- Apache Doris
-
./catalogs/catalog-glue/src/main/java/org/apache/gravitino/catalog/glue/GravitinoGlueCredentialsProvider.java
-
./spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/glue/GravitinoGlueCredentialsProvider.java
-
Apache Hive
./catalogs/hive-metastore-common/src/test/resources/hive-schema-3.1.0.derby.sql
diff --git a/catalogs/catalog-common/build.gradle.kts
b/catalogs/catalog-common/build.gradle.kts
index 41a7a5f030..6975af7627 100644
--- a/catalogs/catalog-common/build.gradle.kts
+++ b/catalogs/catalog-common/build.gradle.kts
@@ -24,6 +24,7 @@ plugins {
// try to avoid adding extra dependencies because it is used by catalogs and
connectors.
dependencies {
compileOnly(project(":common"))
+ compileOnly("software.amazon.awssdk:auth:${libs.versions.awssdk.get()}")
implementation(libs.commons.lang3)
implementation(libs.guava)
@@ -31,6 +32,7 @@ dependencies {
implementation(libs.slf4j.api)
testImplementation(project(":common"))
+
testImplementation("software.amazon.awssdk:auth:${libs.versions.awssdk.get()}")
testImplementation(libs.junit.jupiter.api)
testImplementation(libs.junit.jupiter.params)
diff --git
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/glue/GravitinoGlueCredentialsProvider.java
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/glue/GravitinoGlueCredentialsProvider.java
similarity index 71%
rename from
spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/glue/GravitinoGlueCredentialsProvider.java
rename to
catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/glue/GravitinoGlueCredentialsProvider.java
index 369e09f144..62ccd2808e 100644
---
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/glue/GravitinoGlueCredentialsProvider.java
+++
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/glue/GravitinoGlueCredentialsProvider.java
@@ -16,8 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-
-package org.apache.gravitino.spark.connector.glue;
+package org.apache.gravitino.catalog.glue;
import com.google.common.base.Preconditions;
import java.util.Map;
@@ -30,13 +29,10 @@ import
software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
* AWS credentials provider for Iceberg {@code GlueCatalog} that reads static
credentials from a
* properties map.
*
- * <p>Iceberg 1.10+ loads credentials via {@code client.credentials-provider}.
This class is
- * instantiated dynamically by Iceberg's {@code AwsClientProperties} using the
{@code create(Map)}
- * static factory. The properties map contains the {@code
client.credentials-provider.*} entries
- * with their prefix stripped, i.e. {@code access-key-id} and {@code
secret-access-key}.
- *
- * <p>Adapted from Apache Doris's {@code CustomAwsCredentialsProvider} ({@code
- * org.apache.doris.datasource.iceberg.s3tables.CustomAwsCredentialsProvider}).
+ * <p>Iceberg 1.10+ no longer supports {@code client.access-key-id} directly;
credentials must be
+ * supplied via {@code client.credentials-provider}. This class is configured
in {@code
+ * org.apache.gravitino.catalog.glue.GlueIcebergTableHelper#createGlueCatalog}
when explicit
+ * credentials are provided.
*/
public class GravitinoGlueCredentialsProvider implements
AwsCredentialsProvider {
@@ -46,6 +42,11 @@ public class GravitinoGlueCredentialsProvider implements
AwsCredentialsProvider
private final String accessKeyId;
private final String secretAccessKey;
+ private GravitinoGlueCredentialsProvider(String accessKeyId, String
secretAccessKey) {
+ this.accessKeyId = accessKeyId;
+ this.secretAccessKey = secretAccessKey;
+ }
+
/**
* Creates a credentials provider from the given properties map.
*
@@ -57,21 +58,21 @@ public class GravitinoGlueCredentialsProvider implements
AwsCredentialsProvider
String accessKeyId = properties.get(ACCESS_KEY_ID);
String secretAccessKey = properties.get(SECRET_ACCESS_KEY);
Preconditions.checkArgument(
- StringUtils.isNotBlank(accessKeyId),
- "Glue credentials provider requires 'access-key-id' in
client.credentials-provider.* properties");
+ StringUtils.isNotBlank(accessKeyId), "Glue credentials require
'access-key-id'");
Preconditions.checkArgument(
- StringUtils.isNotBlank(secretAccessKey),
- "Glue credentials provider requires 'secret-access-key' in
client.credentials-provider.* properties");
+ StringUtils.isNotBlank(secretAccessKey), "Glue credentials require
'secret-access-key'");
return new GravitinoGlueCredentialsProvider(accessKeyId, secretAccessKey);
}
- GravitinoGlueCredentialsProvider(String accessKeyId, String secretAccessKey)
{
- this.accessKeyId = accessKeyId;
- this.secretAccessKey = secretAccessKey;
- }
-
@Override
public AwsCredentials resolveCredentials() {
+ if (StringUtils.isBlank(accessKeyId)) {
+ throw new IllegalStateException("Access key ID is not set");
+ }
+ if (StringUtils.isBlank(secretAccessKey)) {
+ throw new IllegalStateException("Secret access key is not set");
+ }
+
return AwsBasicCredentials.create(accessKeyId, secretAccessKey);
}
}
diff --git
a/catalogs/catalog-common/src/test/java/org/apache/gravitino/catalog/glue/TestGravitinoGlueCredentialsProvider.java
b/catalogs/catalog-common/src/test/java/org/apache/gravitino/catalog/glue/TestGravitinoGlueCredentialsProvider.java
new file mode 100644
index 0000000000..886188ad45
--- /dev/null
+++
b/catalogs/catalog-common/src/test/java/org/apache/gravitino/catalog/glue/TestGravitinoGlueCredentialsProvider.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.catalog.glue;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+
+class TestGravitinoGlueCredentialsProvider {
+
+ @Test
+ void testCreateAndResolveValidCredentials() {
+ AwsCredentials credentials =
+ GravitinoGlueCredentialsProvider.create(
+ Map.of("access-key-id", "AKID", "secret-access-key", "SECRET"))
+ .resolveCredentials();
+
+ Assertions.assertInstanceOf(AwsBasicCredentials.class, credentials);
+ Assertions.assertEquals("AKID", credentials.accessKeyId());
+ Assertions.assertEquals("SECRET", credentials.secretAccessKey());
+ }
+
+ @Test
+ void testExtraPropertiesAreIgnored() {
+ AwsCredentials credentials =
+ GravitinoGlueCredentialsProvider.create(
+ Map.of("access-key-id", "AKID", "secret-access-key", "SECRET",
"unused", "value"))
+ .resolveCredentials();
+
+ Assertions.assertEquals("AKID", credentials.accessKeyId());
+ Assertions.assertEquals("SECRET", credentials.secretAccessKey());
+ }
+
+ @Test
+ void testInvalidPropertiesAreRejected() {
+ Assertions.assertThrows(
+ IllegalArgumentException.class, () ->
GravitinoGlueCredentialsProvider.create(null));
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () ->
GravitinoGlueCredentialsProvider.create(Map.of("secret-access-key", "SECRET")));
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () -> GravitinoGlueCredentialsProvider.create(Map.of("access-key-id",
"AKID")));
+
+ Map<String, String> blankAccessKey = new HashMap<>();
+ blankAccessKey.put("access-key-id", " ");
+ blankAccessKey.put("secret-access-key", "SECRET");
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () -> GravitinoGlueCredentialsProvider.create(blankAccessKey));
+
+ Map<String, String> blankSecretKey = new HashMap<>();
+ blankSecretKey.put("access-key-id", "AKID");
+ blankSecretKey.put("secret-access-key", " ");
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () -> GravitinoGlueCredentialsProvider.create(blankSecretKey));
+ }
+}
diff --git
a/catalogs/catalog-glue/src/main/java/org/apache/gravitino/catalog/glue/GravitinoGlueCredentialsProvider.java
b/catalogs/catalog-glue/src/main/java/org/apache/gravitino/catalog/glue/GravitinoGlueCredentialsProvider.java
deleted file mode 100644
index 4ed99b2acf..0000000000
---
a/catalogs/catalog-glue/src/main/java/org/apache/gravitino/catalog/glue/GravitinoGlueCredentialsProvider.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.gravitino.catalog.glue;
-
-import java.util.Map;
-import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
-import software.amazon.awssdk.auth.credentials.AwsCredentials;
-import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
-
-/**
- * AWS credentials provider for Iceberg {@code GlueCatalog} that reads static
credentials from a
- * properties map.
- *
- * <p>Iceberg 1.10+ no longer supports {@code client.access-key-id} directly;
credentials must be
- * supplied via {@code client.credentials-provider}. This class is configured
in {@link
- * GlueIcebergTableHelper#createGlueCatalog} when explicit credentials are
provided.
- *
- * <p>Adapted from Apache Doris's {@code CustomAwsCredentialsProvider} ({@code
- * org.apache.doris.datasource.iceberg.s3tables.CustomAwsCredentialsProvider}).
- */
-class GravitinoGlueCredentialsProvider implements AwsCredentialsProvider {
-
- private static final String ACCESS_KEY_ID = "access-key-id";
- private static final String SECRET_ACCESS_KEY = "secret-access-key";
-
- private final String accessKeyId;
- private final String secretAccessKey;
-
- /**
- * Creates a credentials provider from the given properties map.
- *
- * @param properties map containing {@code access-key-id} and {@code
secret-access-key}
- * @return a new {@link AwsCredentialsProvider} instance
- */
- public static AwsCredentialsProvider create(Map<String, String> properties) {
- return new GravitinoGlueCredentialsProvider(
- properties.get(ACCESS_KEY_ID), properties.get(SECRET_ACCESS_KEY));
- }
-
- GravitinoGlueCredentialsProvider(String accessKeyId, String secretAccessKey)
{
- this.accessKeyId = accessKeyId;
- this.secretAccessKey = secretAccessKey;
- }
-
- @Override
- public AwsCredentials resolveCredentials() {
- if (accessKeyId == null || accessKeyId.isEmpty()) {
- throw new IllegalStateException("Access key ID is not set");
- }
- if (secretAccessKey == null || secretAccessKey.isEmpty()) {
- throw new IllegalStateException("Secret access key is not set");
- }
- return AwsBasicCredentials.create(accessKeyId, secretAccessKey);
- }
-}
diff --git
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/glue/GluePropertiesConverter.java
b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/glue/GluePropertiesConverter.java
index a24e5f315f..127e398921 100644
---
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/glue/GluePropertiesConverter.java
+++
b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/glue/GluePropertiesConverter.java
@@ -26,6 +26,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.apache.gravitino.catalog.glue.GlueConstants;
+import org.apache.gravitino.catalog.glue.GravitinoGlueCredentialsProvider;
import org.apache.gravitino.spark.connector.PropertiesConverter;
import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.aws.glue.GlueCatalog;
@@ -71,7 +72,7 @@ public class GluePropertiesConverter implements
PropertiesConverter {
* AwsClientProperties} can instantiate it dynamically via {@code
client.credentials-provider}.
*/
public static final String GRAVITINO_GLUE_CREDENTIALS_PROVIDER =
-
"org.apache.gravitino.spark.connector.glue.GravitinoGlueCredentialsProvider";
+ GravitinoGlueCredentialsProvider.class.getName();
/** Gravitino catalog property key for the AWS access key ID. */
public static final String AWS_ACCESS_KEY_ID = "aws-access-key-id";
diff --git
a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/glue/TestGluePropertiesConverter.java
b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/glue/TestGluePropertiesConverter.java
index 1340a85935..7199ef95ef 100644
---
a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/glue/TestGluePropertiesConverter.java
+++
b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/glue/TestGluePropertiesConverter.java
@@ -21,10 +21,12 @@ package org.apache.gravitino.spark.connector.glue;
import com.google.common.collect.ImmutableMap;
import java.util.Map;
+import org.apache.gravitino.catalog.glue.GravitinoGlueCredentialsProvider;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInstance;
import org.junit.jupiter.api.TestInstance.Lifecycle;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
@TestInstance(Lifecycle.PER_CLASS)
public class TestGluePropertiesConverter {
@@ -61,7 +63,7 @@ public class TestGluePropertiesConverter {
Assertions.assertEquals(
"http://localhost:4566";,
icebergProps.get(GluePropertiesConverter.GLUE_ENDPOINT));
Assertions.assertEquals(
-
"org.apache.gravitino.spark.connector.glue.GravitinoGlueCredentialsProvider",
+ GravitinoGlueCredentialsProvider.class.getName(),
icebergProps.get(GluePropertiesConverter.CLIENT_CREDENTIALS_PROVIDER));
Assertions.assertEquals(
"AKIAIOSFODNN7EXAMPLE",
icebergProps.get("client.credentials-provider.access-key-id"));
@@ -103,7 +105,7 @@ public class TestGluePropertiesConverter {
GluePropertiesConverter.AWS_ACCESS_KEY_ID, "access-key",
GluePropertiesConverter.AWS_SECRET_ACCESS_KEY, "secret-key"));
Assertions.assertEquals(
-
"org.apache.gravitino.spark.connector.glue.GravitinoGlueCredentialsProvider",
+ GravitinoGlueCredentialsProvider.class.getName(),
icebergProps.get(GluePropertiesConverter.CLIENT_CREDENTIALS_PROVIDER));
Assertions.assertEquals(
"access-key",
icebergProps.get("client.credentials-provider.access-key-id"));
@@ -123,6 +125,15 @@ public class TestGluePropertiesConverter {
Assertions.assertNull(blankRegion.get(GluePropertiesConverter.CLIENT_REGION));
}
+ @Test
+ void testSharedCredentialsProviderIsReflectivelyLoadable() throws
ClassNotFoundException {
+ Class<?> providerClass =
+
Class.forName(GluePropertiesConverter.GRAVITINO_GLUE_CREDENTIALS_PROVIDER);
+
+ Assertions.assertEquals(GravitinoGlueCredentialsProvider.class,
providerClass);
+
Assertions.assertTrue(AwsCredentialsProvider.class.isAssignableFrom(providerClass));
+ }
+
@Test
void testToIcebergCatalogPropertiesNullInputThrows() {
Assertions.assertThrows(
diff --git
a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/glue/TestGravitinoGlueCredentialsProvider.java
b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/glue/TestGravitinoGlueCredentialsProvider.java
deleted file mode 100644
index 5a5113f07b..0000000000
---
a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/glue/TestGravitinoGlueCredentialsProvider.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.gravitino.spark.connector.glue;
-
-import java.util.Map;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-import software.amazon.awssdk.auth.credentials.AwsCredentials;
-import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
-
-public class TestGravitinoGlueCredentialsProvider {
-
- @Test
- void testCreateAndResolveValidCredentials() {
- AwsCredentialsProvider provider =
- GravitinoGlueCredentialsProvider.create(
- Map.of("access-key-id", "AKID", "secret-access-key", "SECRET"));
- AwsCredentials creds = provider.resolveCredentials();
- Assertions.assertEquals("AKID", creds.accessKeyId());
- Assertions.assertEquals("SECRET", creds.secretAccessKey());
- }
-
- @Test
- void testCreateWithMissingKeysThrows() {
- Assertions.assertThrows(
- IllegalArgumentException.class, () ->
GravitinoGlueCredentialsProvider.create(Map.of()));
- }
-
- @Test
- void testCreateWithNullPropertiesThrows() {
- Assertions.assertThrows(
- IllegalArgumentException.class, () ->
GravitinoGlueCredentialsProvider.create(null));
- }
-}
