This is an automated email from the ASF dual-hosted git repository.
jshao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 94c6a724d [#5842] feat(core): supports credential REST endpoint in
Gravitino server (#5841)
94c6a724d is described below
commit 94c6a724dd94bce383c00cf238f87f506e9a160f
Author: FANNG <[email protected]>
AuthorDate: Tue Dec 17 19:11:01 2024 +0800
[#5842] feat(core): supports credential REST endpoint in Gravitino server
(#5841)
### What changes were proposed in this pull request?
add credential REST endpoint in Gravitino server
### Why are the changes needed?
Fix: #5842
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
add UT and run test in POC
---------
Co-authored-by: Jerry Shao <[email protected]>
---
.../java/org/apache/gravitino/GravitinoEnv.java | 14 ++
.../apache/gravitino/catalog/CatalogManager.java | 4 +
.../gravitino/catalog/CredentialManager.java | 53 +++++++
.../apache/gravitino/server/GravitinoServer.java | 2 +
.../server/web/rest/ExceptionHandlers.java | 38 +++++
.../rest/MetadataObjectCredentialOperations.java | 100 +++++++++++++
.../TestMetadataObjectCredentialOperations.java | 164 +++++++++++++++++++++
7 files changed, 375 insertions(+)
diff --git a/core/src/main/java/org/apache/gravitino/GravitinoEnv.java
b/core/src/main/java/org/apache/gravitino/GravitinoEnv.java
index 1cad967a9..db6ddc235 100644
--- a/core/src/main/java/org/apache/gravitino/GravitinoEnv.java
+++ b/core/src/main/java/org/apache/gravitino/GravitinoEnv.java
@@ -28,6 +28,7 @@ import org.apache.gravitino.auxiliary.AuxiliaryServiceManager;
import org.apache.gravitino.catalog.CatalogDispatcher;
import org.apache.gravitino.catalog.CatalogManager;
import org.apache.gravitino.catalog.CatalogNormalizeDispatcher;
+import org.apache.gravitino.catalog.CredentialManager;
import org.apache.gravitino.catalog.FilesetDispatcher;
import org.apache.gravitino.catalog.FilesetNormalizeDispatcher;
import org.apache.gravitino.catalog.FilesetOperationDispatcher;
@@ -105,6 +106,8 @@ public class GravitinoEnv {
private MetalakeDispatcher metalakeDispatcher;
+ private CredentialManager credentialManager;
+
private AccessControlDispatcher accessControlDispatcher;
private IdGenerator idGenerator;
@@ -257,6 +260,15 @@ public class GravitinoEnv {
return metalakeDispatcher;
}
+ /**
+ * Get the {@link CredentialManager} associated with the Gravitino
environment.
+ *
+ * @return The {@link CredentialManager} instance.
+ */
+ public CredentialManager credentialManager() {
+ return credentialManager;
+ }
+
/**
* Get the IdGenerator associated with the Gravitino environment.
*
@@ -417,6 +429,8 @@ public class GravitinoEnv {
new CatalogNormalizeDispatcher(catalogHookDispatcher);
this.catalogDispatcher = new CatalogEventDispatcher(eventBus,
catalogNormalizeDispatcher);
+ this.credentialManager = new CredentialManager(catalogManager,
entityStore, idGenerator);
+
SchemaOperationDispatcher schemaOperationDispatcher =
new SchemaOperationDispatcher(catalogManager, entityStore,
idGenerator);
SchemaHookDispatcher schemaHookDispatcher = new
SchemaHookDispatcher(schemaOperationDispatcher);
diff --git
a/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java
b/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java
index da79ff702..2e77b8e16 100644
--- a/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java
+++ b/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java
@@ -133,6 +133,10 @@ public class CatalogManager implements CatalogDispatcher,
Closeable {
this.classLoader = classLoader;
}
+ public BaseCatalog catalog() {
+ return catalog;
+ }
+
public <R> R doWithSchemaOps(ThrowableFunction<SupportsSchemas, R> fn)
throws Exception {
return classLoader.withClassLoader(
cl -> {
diff --git
a/core/src/main/java/org/apache/gravitino/catalog/CredentialManager.java
b/core/src/main/java/org/apache/gravitino/catalog/CredentialManager.java
new file mode 100644
index 000000000..808fc96fb
--- /dev/null
+++ b/core/src/main/java/org/apache/gravitino/catalog/CredentialManager.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.catalog;
+
+import java.util.List;
+import org.apache.commons.lang3.NotImplementedException;
+import org.apache.gravitino.EntityStore;
+import org.apache.gravitino.NameIdentifier;
+import org.apache.gravitino.connector.BaseCatalog;
+import org.apache.gravitino.credential.Credential;
+import org.apache.gravitino.exceptions.NoSuchCatalogException;
+import org.apache.gravitino.storage.IdGenerator;
+import org.apache.gravitino.utils.NameIdentifierUtil;
+
+/** Get credentials with the specific catalog classloader. */
+public class CredentialManager extends OperationDispatcher {
+
+ public CredentialManager(
+ CatalogManager catalogManager, EntityStore store, IdGenerator
idGenerator) {
+ super(catalogManager, store, idGenerator);
+ }
+
+ public List<Credential> getCredentials(NameIdentifier identifier) {
+ return doWithCatalog(
+ NameIdentifierUtil.getCatalogIdentifier(identifier),
+ c -> getCredentials(c.catalog(), identifier),
+ NoSuchCatalogException.class);
+ }
+
+ private List<Credential> getCredentials(BaseCatalog catalog, NameIdentifier
identifier) {
+ throw new NotImplementedException(
+ String.format(
+ "Load credentials is not implemented for catalog: %s, identifier:
%s",
+ catalog.name(), identifier));
+ }
+}
diff --git
a/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java
b/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java
index 36c112f00..554791fff 100644
--- a/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java
+++ b/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java
@@ -26,6 +26,7 @@ import javax.servlet.Servlet;
import org.apache.gravitino.Configs;
import org.apache.gravitino.GravitinoEnv;
import org.apache.gravitino.catalog.CatalogDispatcher;
+import org.apache.gravitino.catalog.CredentialManager;
import org.apache.gravitino.catalog.FilesetDispatcher;
import org.apache.gravitino.catalog.PartitionDispatcher;
import org.apache.gravitino.catalog.SchemaDispatcher;
@@ -114,6 +115,7 @@ public class GravitinoServer extends ResourceConfig {
bind(gravitinoEnv.filesetDispatcher()).to(FilesetDispatcher.class).ranked(1);
bind(gravitinoEnv.topicDispatcher()).to(TopicDispatcher.class).ranked(1);
bind(gravitinoEnv.tagManager()).to(TagManager.class).ranked(1);
+
bind(gravitinoEnv.credentialManager()).to(CredentialManager.class).ranked(1);
}
});
register(JsonProcessingExceptionMapper.class);
diff --git
a/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java
b/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java
index 8d1ba8565..faf94f506 100644
---
a/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java
+++
b/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java
@@ -121,6 +121,11 @@ public class ExceptionHandlers {
return TagExceptionHandler.INSTANCE.handle(op, tag, parent, e);
}
+ public static Response handleCredentialException(
+ OperationType op, String metadataObjectName, Exception e) {
+ return CredentialExceptionHandler.INSTANCE.handle(op, metadataObjectName,
"", e);
+ }
+
public static Response handleTestConnectionException(Exception e) {
ErrorResponse response;
if (e instanceof IllegalArgumentException) {
@@ -369,6 +374,7 @@ public class ExceptionHandlers {
}
private static class FilesetExceptionHandler extends BaseExceptionHandler {
+
private static final ExceptionHandler INSTANCE = new
FilesetExceptionHandler();
private static String getFilesetErrorMsg(
@@ -520,6 +526,7 @@ public class ExceptionHandlers {
}
private static class TopicExceptionHandler extends BaseExceptionHandler {
+
private static final ExceptionHandler INSTANCE = new
TopicExceptionHandler();
private static String getTopicErrorMsg(
@@ -558,6 +565,7 @@ public class ExceptionHandlers {
private static class UserPermissionOperationExceptionHandler
extends BasePermissionExceptionHandler {
+
private static final ExceptionHandler INSTANCE = new
UserPermissionOperationExceptionHandler();
@Override
@@ -622,6 +630,35 @@ public class ExceptionHandlers {
}
}
+ private static class CredentialExceptionHandler extends BaseExceptionHandler
{
+
+ private static final ExceptionHandler INSTANCE = new
CredentialExceptionHandler();
+
+ private static String getCredentialErrorMsg(String parent, String reason) {
+ return String.format(
+ "Failed to get credentials under object [%s], reason [%s]", parent,
reason);
+ }
+
+ @Override
+ public Response handle(OperationType op, String credential, String parent,
Exception e) {
+ String errorMsg = getCredentialErrorMsg(parent, getErrorMsg(e));
+ LOG.warn(errorMsg, e);
+
+ if (e instanceof IllegalArgumentException) {
+ return Utils.illegalArguments(errorMsg, e);
+
+ } else if (e instanceof NotFoundException) {
+ return Utils.notFound(errorMsg, e);
+
+ } else if (e instanceof NotInUseException) {
+ return Utils.notInUse(errorMsg, e);
+
+ } else {
+ return super.handle(op, credential, parent, e);
+ }
+ }
+ }
+
private static class TagExceptionHandler extends BaseExceptionHandler {
private static final ExceptionHandler INSTANCE = new TagExceptionHandler();
@@ -661,6 +698,7 @@ public class ExceptionHandlers {
}
private static class OwnerExceptionHandler extends BaseExceptionHandler {
+
private static final ExceptionHandler INSTANCE = new
OwnerExceptionHandler();
private static String getOwnerErrorMsg(
diff --git
a/server/src/main/java/org/apache/gravitino/server/web/rest/MetadataObjectCredentialOperations.java
b/server/src/main/java/org/apache/gravitino/server/web/rest/MetadataObjectCredentialOperations.java
new file mode 100644
index 000000000..7c6ea4a8e
--- /dev/null
+++
b/server/src/main/java/org/apache/gravitino/server/web/rest/MetadataObjectCredentialOperations.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.server.web.rest;
+
+import com.codahale.metrics.annotation.ResponseMetered;
+import com.codahale.metrics.annotation.Timed;
+import java.util.List;
+import java.util.Locale;
+import javax.inject.Inject;
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.Response;
+import org.apache.gravitino.MetadataObject;
+import org.apache.gravitino.MetadataObjects;
+import org.apache.gravitino.NameIdentifier;
+import org.apache.gravitino.catalog.CredentialManager;
+import org.apache.gravitino.credential.Credential;
+import org.apache.gravitino.dto.credential.CredentialDTO;
+import org.apache.gravitino.dto.responses.CredentialResponse;
+import org.apache.gravitino.dto.util.DTOConverters;
+import org.apache.gravitino.metrics.MetricNames;
+import org.apache.gravitino.server.web.Utils;
+import org.apache.gravitino.utils.MetadataObjectUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Path("/metalakes/{metalake}/objects/{type}/{fullName}/credentials")
+public class MetadataObjectCredentialOperations {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(MetadataObjectCredentialOperations.class);
+
+ private CredentialManager credentialManager;
+
+ @SuppressWarnings("unused")
+ @Context
+ private HttpServletRequest httpRequest;
+
+ @Inject
+ public MetadataObjectCredentialOperations(CredentialManager dispatcher) {
+ this.credentialManager = dispatcher;
+ }
+
+ @GET
+ @Produces("application/vnd.gravitino.v1+json")
+ @Timed(name = "get-credentials." + MetricNames.HTTP_PROCESS_DURATION,
absolute = true)
+ @ResponseMetered(name = "get-credentials", absolute = true)
+ public Response getCredentials(
+ @PathParam("metalake") String metalake,
+ @PathParam("type") String type,
+ @PathParam("fullName") String fullName) {
+ LOG.info(
+ "Received get credentials request for object type: {}, full name: {}
under metalake: {}",
+ type,
+ fullName,
+ metalake);
+
+ try {
+ return Utils.doAs(
+ httpRequest,
+ () -> {
+ MetadataObject object =
+ MetadataObjects.parse(
+ fullName,
MetadataObject.Type.valueOf(type.toUpperCase(Locale.ROOT)));
+
+ NameIdentifier identifier =
MetadataObjectUtil.toEntityIdent(metalake, object);
+ List<Credential> credentials =
credentialManager.getCredentials(identifier);
+ if (credentials == null) {
+ return Utils.ok(new CredentialResponse(new CredentialDTO[0]));
+ }
+ return Utils.ok(
+ new CredentialResponse(
+ DTOConverters.toDTO(credentials.toArray(new
Credential[credentials.size()]))));
+ });
+ } catch (Exception e) {
+ return ExceptionHandlers.handleCredentialException(OperationType.GET,
fullName, e);
+ }
+ }
+}
diff --git
a/server/src/test/java/org/apache/gravitino/server/web/rest/TestMetadataObjectCredentialOperations.java
b/server/src/test/java/org/apache/gravitino/server/web/rest/TestMetadataObjectCredentialOperations.java
new file mode 100644
index 000000000..1ac5d3813
--- /dev/null
+++
b/server/src/test/java/org/apache/gravitino/server/web/rest/TestMetadataObjectCredentialOperations.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.server.web.rest;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.Arrays;
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.core.Application;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import org.apache.gravitino.MetadataObject;
+import org.apache.gravitino.MetadataObjects;
+import org.apache.gravitino.catalog.CredentialManager;
+import org.apache.gravitino.credential.Credential;
+import org.apache.gravitino.credential.S3SecretKeyCredential;
+import org.apache.gravitino.dto.responses.CredentialResponse;
+import org.apache.gravitino.dto.responses.ErrorConstants;
+import org.apache.gravitino.dto.responses.ErrorResponse;
+import org.apache.gravitino.dto.util.DTOConverters;
+import org.apache.gravitino.exceptions.NoSuchCredentialException;
+import org.apache.gravitino.rest.RESTUtils;
+import org.glassfish.jersey.internal.inject.AbstractBinder;
+import org.glassfish.jersey.server.ResourceConfig;
+import org.glassfish.jersey.test.JerseyTest;
+import org.glassfish.jersey.test.TestProperties;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class TestMetadataObjectCredentialOperations extends JerseyTest {
+
+ private static class MockServletRequestFactory extends
ServletRequestFactoryBase {
+
+ @Override
+ public HttpServletRequest get() {
+ HttpServletRequest request = mock(HttpServletRequest.class);
+ when(request.getRemoteUser()).thenReturn(null);
+ return request;
+ }
+ }
+
+ private CredentialManager credentialManager = mock(CredentialManager.class);
+
+ private String metalake = "test_metalake";
+
+ @Override
+ protected Application configure() {
+ try {
+ forceSet(
+ TestProperties.CONTAINER_PORT,
String.valueOf(RESTUtils.findAvailablePort(2000, 3000)));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ ResourceConfig resourceConfig = new ResourceConfig();
+ resourceConfig.register(MetadataObjectCredentialOperations.class);
+ resourceConfig.register(
+ new AbstractBinder() {
+ @Override
+ protected void configure() {
+ bind(credentialManager).to(CredentialManager.class).ranked(2);
+
bindFactory(MockServletRequestFactory.class).to(HttpServletRequest.class);
+ }
+ });
+
+ return resourceConfig;
+ }
+
+ @Test
+ public void testGetCredentialsForCatalog() {
+ testGetCredentialsForObject(MetadataObjects.parse("catalog",
MetadataObject.Type.CATALOG));
+ }
+
+ @Test
+ public void testGetCredentialsForFileset() {
+ testGetCredentialsForObject(
+ MetadataObjects.parse("catalog.schema.fileset",
MetadataObject.Type.FILESET));
+ }
+
+ private void testGetCredentialsForObject(MetadataObject metadataObject) {
+
+ S3SecretKeyCredential credential = new S3SecretKeyCredential("access-id",
"secret-key");
+ // Test return one credential
+
when(credentialManager.getCredentials(any())).thenReturn(Arrays.asList(credential));
+ Response response =
+ target(basePath(metalake))
+ .path(metadataObject.type().toString())
+ .path(metadataObject.fullName())
+ .path("/credentials")
+ .request(MediaType.APPLICATION_JSON_TYPE)
+ .accept("application/vnd.gravitino.v1+json")
+ .get();
+
+ Assertions.assertEquals(Response.Status.OK.getStatusCode(),
response.getStatus());
+ CredentialResponse credentialResponse =
response.readEntity(CredentialResponse.class);
+ Assertions.assertEquals(0, credentialResponse.getCode());
+ Assertions.assertEquals(1, credentialResponse.getCredentials().length);
+ Credential credentialToTest =
DTOConverters.fromDTO(credentialResponse.getCredentials()[0]);
+ Assertions.assertTrue(credentialToTest instanceof S3SecretKeyCredential);
+ Assertions.assertEquals("access-id", ((S3SecretKeyCredential)
credentialToTest).accessKeyId());
+ Assertions.assertEquals(
+ "secret-key", ((S3SecretKeyCredential)
credentialToTest).secretAccessKey());
+ Assertions.assertEquals(0, credentialToTest.expireTimeInMs());
+
+ // Test doesn't return credential
+ when(credentialManager.getCredentials(any())).thenReturn(null);
+ response =
+ target(basePath(metalake))
+ .path(metadataObject.type().toString())
+ .path(metadataObject.fullName())
+ .path("/credentials")
+ .request(MediaType.APPLICATION_JSON_TYPE)
+ .accept("application/vnd.gravitino.v1+json")
+ .get();
+
+ Assertions.assertEquals(Response.Status.OK.getStatusCode(),
response.getStatus());
+ credentialResponse = response.readEntity(CredentialResponse.class);
+ Assertions.assertEquals(0, credentialResponse.getCode());
+ Assertions.assertEquals(0, credentialResponse.getCredentials().length);
+
+ // Test throws NoSuchCredentialException
+ doThrow(new NoSuchCredentialException("mock error"))
+ .when(credentialManager)
+ .getCredentials(any());
+ response =
+ target(basePath(metalake))
+ .path(metadataObject.type().toString())
+ .path(metadataObject.fullName())
+ .path("/credentials")
+ .request(MediaType.APPLICATION_JSON_TYPE)
+ .accept("application/vnd.gravitino.v1+json")
+ .get();
+
+ Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(),
response.getStatus());
+ ErrorResponse errorResponse = response.readEntity(ErrorResponse.class);
+ Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE,
errorResponse.getCode());
+ Assertions.assertEquals(
+ NoSuchCredentialException.class.getSimpleName(),
errorResponse.getType());
+ }
+
+ private String basePath(String metalake) {
+ return "/metalakes/" + metalake + "/objects";
+ }
+}
