This is an automated email from the ASF dual-hosted git repository.
jshao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 53bd22713 [#4992] support credential vending framework (#4995)
53bd22713 is described below
commit 53bd22713f879fdf4c7150c9ad3f099f06704209
Author: FANNG <[email protected]>
AuthorDate: Tue Oct 15 11:06:07 2024 +0800
[#4992] support credential vending framework (#4995)
### What changes were proposed in this pull request?
support credential vending framework
### Why are the changes needed?
Fix: #4992
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
1. add UT
2. propose a draft PR in #4966 , and could run pass S3 token with
Gravitino IcebergRESTServer
---
.../apache/gravitino/credential/Credential.java | 67 +++++++++++++++++
.../credential/CatalogCredentialContext.java | 38 ++++++++++
.../gravitino/credential/CredentialContext.java | 30 ++++++++
.../gravitino/credential/CredentialProvider.java | 56 +++++++++++++++
.../credential/CredentialProviderFactory.java | 69 ++++++++++++++++++
.../credential/PathBasedCredentialContext.java | 58 +++++++++++++++
.../credential/DummyCredentialProvider.java | 83 ++++++++++++++++++++++
.../credential/TestCredentialProvider.java | 54 ++++++++++++++
....apache.gravitino.credential.CredentialProvider | 19 +++++
9 files changed, 474 insertions(+)
diff --git a/api/src/main/java/org/apache/gravitino/credential/Credential.java
b/api/src/main/java/org/apache/gravitino/credential/Credential.java
new file mode 100644
index 000000000..b2fdb1971
--- /dev/null
+++ b/api/src/main/java/org/apache/gravitino/credential/Credential.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.credential;
+
+import com.google.common.collect.ImmutableMap;
+import java.util.Map;
+
+/** Interface representing a credential with type, expiration time, and
additional information. */
+public interface Credential {
+ /** Credential type in the credential. */
+ String CREDENTIAL_TYPE = "credential-type";
+ /** Credential expire time in ms since the epoch. */
+ String EXPIRE_TIME_IN_MS = "expire-time-in-ms";
+
+ /**
+ * Returns the type of the credential. It should be the same as the
credential type of the
+ * credential provider.
+ *
+ * @return the credential type as a String.
+ */
+ String credentialType();
+
+ /**
+ * Returns the expiration time of the credential in milliseconds since the
epoch, 0 means not
+ * expire.
+ *
+ * @return the expiration time as a long.
+ */
+ long expireTimeInMs();
+
+ /**
+ * Returns credential information.
+ *
+ * @return a map of credential information.
+ */
+ Map<String, String> credentialInfo();
+
+ /**
+ * Converts the credential to properties to transfer the credential though
API.
+ *
+ * @return a map containing credential properties.
+ */
+ default Map<String, String> toProperties() {
+ return new ImmutableMap.Builder<String, String>()
+ .putAll(credentialInfo())
+ .put(CREDENTIAL_TYPE, credentialType())
+ .put(EXPIRE_TIME_IN_MS, String.valueOf(expireTimeInMs()))
+ .build();
+ }
+}
diff --git
a/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialContext.java
b/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialContext.java
new file mode 100644
index 000000000..a39dbba01
--- /dev/null
+++
b/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialContext.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.credential;
+
+import com.google.common.base.Preconditions;
+import javax.validation.constraints.NotNull;
+
+/** CatalogCredentialContext is generated when user requesting catalog
credentials. */
+public class CatalogCredentialContext implements CredentialContext {
+ @NotNull private final String userName;
+
+ public CatalogCredentialContext(String userName) {
+ Preconditions.checkNotNull(userName, "User name should not be null");
+ this.userName = userName;
+ }
+
+ @Override
+ public String getUserName() {
+ return userName;
+ }
+}
diff --git
a/core/src/main/java/org/apache/gravitino/credential/CredentialContext.java
b/core/src/main/java/org/apache/gravitino/credential/CredentialContext.java
new file mode 100644
index 000000000..6e82efea0
--- /dev/null
+++ b/core/src/main/java/org/apache/gravitino/credential/CredentialContext.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.credential;
+
+/** Contains credential context information to get credential from a
credential provider. */
+public interface CredentialContext {
+ /**
+ * Providing the username.
+ *
+ * @return A string identifying user name.
+ */
+ String getUserName();
+}
diff --git
a/core/src/main/java/org/apache/gravitino/credential/CredentialProvider.java
b/core/src/main/java/org/apache/gravitino/credential/CredentialProvider.java
new file mode 100644
index 000000000..4056cd00b
--- /dev/null
+++ b/core/src/main/java/org/apache/gravitino/credential/CredentialProvider.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.credential;
+
+import java.io.Closeable;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+/**
+ * Interface for credential providers.
+ *
+ * <p>A credential provider is responsible for managing and retrieving
credentials.
+ */
+public interface CredentialProvider extends Closeable {
+ /**
+ * Initializes the credential provider with catalog properties.
+ *
+ * @param properties catalog properties that can be used to configure the
provider. The specific
+ * properties required vary by implementation.
+ */
+ void initialize(Map<String, String> properties);
+
+ /**
+ * Returns the type of credential, it should be identical in Gravitino.
+ *
+ * @return A string identifying the type of credentials.
+ */
+ String credentialType();
+
+ /**
+ * Obtains a credential based on the provided context information.
+ *
+ * @param context A context object providing necessary information for
retrieving credentials.
+ * @return A Credential object containing the authentication information
needed to access a system
+ * or resource. Null will be returned if no credential is available.
+ */
+ @Nullable
+ Credential getCredential(CredentialContext context);
+}
diff --git
a/core/src/main/java/org/apache/gravitino/credential/CredentialProviderFactory.java
b/core/src/main/java/org/apache/gravitino/credential/CredentialProviderFactory.java
new file mode 100644
index 000000000..3833eeda9
--- /dev/null
+++
b/core/src/main/java/org/apache/gravitino/credential/CredentialProviderFactory.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.credential;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Streams;
+import java.util.List;
+import java.util.Map;
+import java.util.ServiceLoader;
+import java.util.stream.Collectors;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CredentialProviderFactory {
+ private static final Logger LOG =
LoggerFactory.getLogger(CredentialProviderFactory.class);
+
+ public static CredentialProvider create(
+ String credentialType, Map<String, String> catalogProperties) {
+ Class<? extends CredentialProvider> providerClz =
lookupCredentialProvider(credentialType);
+ try {
+ CredentialProvider provider =
providerClz.getDeclaredConstructor().newInstance();
+ provider.initialize(catalogProperties);
+ return provider;
+ } catch (Exception e) {
+ LOG.warn("Create credential provider failed, {}", credentialType, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static Class<? extends CredentialProvider> lookupCredentialProvider(
+ String credentialType) {
+ ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+ ServiceLoader<CredentialProvider> serviceLoader =
+ ServiceLoader.load(CredentialProvider.class, classLoader);
+ List<Class<? extends CredentialProvider>> providers =
+ Streams.stream(serviceLoader.iterator())
+ .filter(
+ credentialProvider ->
+
credentialType.equalsIgnoreCase(credentialProvider.credentialType()))
+ .map(CredentialProvider::getClass)
+ .collect(Collectors.toList());
+
+ if (providers.isEmpty()) {
+ throw new IllegalArgumentException("No credential provider found for: "
+ credentialType);
+ } else if (providers.size() > 1) {
+ throw new IllegalArgumentException(
+ "Multiple credential providers found for: " + credentialType);
+ } else {
+ return Iterables.getOnlyElement(providers);
+ }
+ }
+}
diff --git
a/core/src/main/java/org/apache/gravitino/credential/PathBasedCredentialContext.java
b/core/src/main/java/org/apache/gravitino/credential/PathBasedCredentialContext.java
new file mode 100644
index 000000000..03e7bbe0e
--- /dev/null
+++
b/core/src/main/java/org/apache/gravitino/credential/PathBasedCredentialContext.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.credential;
+
+import com.google.common.base.Preconditions;
+import java.util.Set;
+import javax.validation.constraints.NotNull;
+
+/**
+ * LocationContext is generated when user requesting resources associated with
storage location like
+ * table, fileset, etc.
+ */
+public class PathBasedCredentialContext implements CredentialContext {
+
+ @NotNull private final Set<String> writePaths;
+ @NotNull private final Set<String> readPaths;
+ @NotNull private final String userName;
+
+ public PathBasedCredentialContext(
+ String userName, Set<String> writePaths, Set<String> readPaths) {
+ Preconditions.checkNotNull(userName, "User name should not be null");
+ Preconditions.checkNotNull(writePaths, "Write paths should not be null");
+ Preconditions.checkNotNull(readPaths, "Read paths should not be null");
+ this.userName = userName;
+ this.writePaths = writePaths;
+ this.readPaths = readPaths;
+ }
+
+ @Override
+ public String getUserName() {
+ return userName;
+ }
+
+ public Set<String> getWritePaths() {
+ return writePaths;
+ }
+
+ public Set<String> getReadPaths() {
+ return readPaths;
+ }
+}
diff --git
a/core/src/test/java/org/apache/gravitino/credential/DummyCredentialProvider.java
b/core/src/test/java/org/apache/gravitino/credential/DummyCredentialProvider.java
new file mode 100644
index 000000000..864635e96
--- /dev/null
+++
b/core/src/test/java/org/apache/gravitino/credential/DummyCredentialProvider.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.credential;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import java.util.Map;
+import java.util.Set;
+import lombok.Getter;
+
+public class DummyCredentialProvider implements CredentialProvider {
+ Map<String, String> properties;
+ static final String CREDENTIAL_TYPE = "dummy";
+
+ @Override
+ public void initialize(Map<String, String> properties) {
+ this.properties = properties;
+ }
+
+ @Override
+ public void close() {}
+
+ @Override
+ public String credentialType() {
+ return CREDENTIAL_TYPE;
+ }
+
+ @Override
+ public Credential getCredential(CredentialContext context) {
+ Preconditions.checkArgument(
+ context instanceof PathBasedCredentialContext
+ || context instanceof CatalogCredentialContext,
+ "Doesn't support context: " + context.getClass().getSimpleName());
+ if (context instanceof PathBasedCredentialContext) {
+ return new DummyCredential((PathBasedCredentialContext) context);
+ }
+ return null;
+ }
+
+ public static class DummyCredential implements Credential {
+
+ @Getter private Set<String> writeLocations;
+ @Getter private Set<String> readLocations;
+
+ public DummyCredential(PathBasedCredentialContext locationContext) {
+ this.writeLocations = locationContext.getWritePaths();
+ this.readLocations = locationContext.getReadPaths();
+ }
+
+ @Override
+ public String credentialType() {
+ return DummyCredentialProvider.CREDENTIAL_TYPE;
+ }
+
+ @Override
+ public long expireTimeInMs() {
+ return 0;
+ }
+
+ @Override
+ public Map<String, String> credentialInfo() {
+ return ImmutableMap.of(
+ "writeLocation", writeLocations.toString(), "readLocation",
readLocations.toString());
+ }
+ }
+}
diff --git
a/core/src/test/java/org/apache/gravitino/credential/TestCredentialProvider.java
b/core/src/test/java/org/apache/gravitino/credential/TestCredentialProvider.java
new file mode 100644
index 000000000..b419375b1
--- /dev/null
+++
b/core/src/test/java/org/apache/gravitino/credential/TestCredentialProvider.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.credential;
+
+import java.util.Map;
+import org.apache.gravitino.credential.DummyCredentialProvider.DummyCredential;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.testcontainers.shaded.com.google.common.collect.ImmutableMap;
+import org.testcontainers.shaded.com.google.common.collect.ImmutableSet;
+
+public class TestCredentialProvider {
+ @Test
+ void testCredentialProvider() {
+ Map<String, String> catalogProperties = ImmutableMap.of("a", "b");
+ CredentialProvider credentialProvider =
+ CredentialProviderFactory.create(
+ DummyCredentialProvider.CREDENTIAL_TYPE, catalogProperties);
+ Assertions.assertEquals(
+ DummyCredentialProvider.CREDENTIAL_TYPE,
credentialProvider.credentialType());
+ Assertions.assertTrue(credentialProvider instanceof
DummyCredentialProvider);
+ DummyCredentialProvider dummyCredentialProvider =
(DummyCredentialProvider) credentialProvider;
+ Assertions.assertEquals(catalogProperties,
dummyCredentialProvider.properties);
+
+ ImmutableSet<String> writeLocations = ImmutableSet.of("location1");
+ ImmutableSet<String> readLocations = ImmutableSet.of("location2");
+
+ PathBasedCredentialContext locationContext =
+ new PathBasedCredentialContext("user", writeLocations, readLocations);
+ Credential credential =
dummyCredentialProvider.getCredential(locationContext);
+ Assertions.assertTrue(credential instanceof DummyCredential);
+ DummyCredential dummyCredential = (DummyCredential) credential;
+
+ Assertions.assertEquals(writeLocations,
dummyCredential.getWriteLocations());
+ Assertions.assertEquals(readLocations, dummyCredential.getReadLocations());
+ }
+}
diff --git
a/core/src/test/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider
b/core/src/test/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider
new file mode 100644
index 000000000..cbdbff0be
--- /dev/null
+++
b/core/src/test/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+org.apache.gravitino.credential.DummyCredentialProvider
\ No newline at end of file
