Repository: knox Updated Branches: refs/heads/KNOX-1204 [created] 27d381dcc
KNOX-1204 - KIP-11 - S3 Access through Knox API Project: http://git-wip-us.apache.org/repos/asf/knox/repo Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/43567b0a Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/43567b0a Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/43567b0a Branch: refs/heads/KNOX-1204 Commit: 43567b0a094f149d244f68109d76d90a36f8924a Parents: c7f2855 Author: Larry McCay <[email protected]> Authored: Sat Mar 10 17:39:35 2018 -0500 Committer: Larry McCay <[email protected]> Committed: Fri Mar 30 11:06:09 2018 -0400 ---------------------------------------------------------------------- gateway-release/pom.xml | 4 + gateway-service-knoxs3/pom.xml | 76 +++++++ .../service/knoxs3/KnoxS3ServiceMessages.java | 32 +++ .../service/knoxs3/S3BucketsResource.java | 199 +++++++++++++++++ .../KnoxS3ServiceDeploymentContributor.java | 55 +++++ ....gateway.deploy.ServiceDeploymentContributor | 19 ++ .../service/knoxs3/S3BucketsResourceTest.java | 79 +++++++ .../services/s3client/KnoxS3ClientBuilder.java | 219 +++++++++++++++++++ .../services/s3client/S3PolicyModel.java | 60 +++++ pom.xml | 6 + 10 files changed, 749 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/knox/blob/43567b0a/gateway-release/pom.xml ---------------------------------------------------------------------- diff --git a/gateway-release/pom.xml b/gateway-release/pom.xml index c729535..b65c2ef 100644 --- a/gateway-release/pom.xml +++ b/gateway-release/pom.xml @@ -217,6 +217,10 @@ </dependency> <dependency> <groupId>${gateway-group}</groupId> + <artifactId>gateway-service-knoxs3</artifactId> + </dependency> + <dependency> + <groupId>${gateway-group}</groupId> <artifactId>gateway-service-nifi</artifactId> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/knox/blob/43567b0a/gateway-service-knoxs3/pom.xml ---------------------------------------------------------------------- diff --git a/gateway-service-knoxs3/pom.xml b/gateway-service-knoxs3/pom.xml new file mode 100644 index 0000000..37fcc52 --- /dev/null +++ b/gateway-service-knoxs3/pom.xml @@ -0,0 +1,76 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.knox</groupId> + <artifactId>gateway</artifactId> + <version>1.0.0</version> + </parent> + <groupId>org.apache.knox</groupId> + <artifactId>gateway-service-knoxs3</artifactId> + <version>1.0.0</version> + <name>gateway-service-health</name> + <url>http://maven.apache.org</url> + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + </properties> + <dependencies> + <dependency> + <groupId>${gateway-group}</groupId> + <artifactId>gateway-util-common</artifactId> + </dependency> + <dependency> + <groupId>${gateway-group}</groupId> + <artifactId>gateway-spi</artifactId> + </dependency> + <dependency> + <groupId>${gateway-group}</groupId> + <artifactId>gateway-provider-rewrite</artifactId> + </dependency> + <dependency> + <groupId>${gateway-group}</groupId> + <artifactId>gateway-provider-jersey</artifactId> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.knox</groupId> + <artifactId>gateway-test-utils</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.easymock</groupId> + <artifactId>easymock</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>io.dropwizard.metrics</groupId> + <artifactId>metrics-servlets</artifactId> + </dependency> + <dependency> + <groupId>org.apache.knox</groupId> + <artifactId>gateway-server</artifactId> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/knox/blob/43567b0a/gateway-service-knoxs3/src/main/java/org/apache/knox/gateway/service/knoxs3/KnoxS3ServiceMessages.java ---------------------------------------------------------------------- diff --git a/gateway-service-knoxs3/src/main/java/org/apache/knox/gateway/service/knoxs3/KnoxS3ServiceMessages.java b/gateway-service-knoxs3/src/main/java/org/apache/knox/gateway/service/knoxs3/KnoxS3ServiceMessages.java new file mode 100644 index 0000000..36e1f21 --- /dev/null +++ b/gateway-service-knoxs3/src/main/java/org/apache/knox/gateway/service/knoxs3/KnoxS3ServiceMessages.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.service.knoxs3; + +import org.apache.knox.gateway.i18n.messages.Message; +import org.apache.knox.gateway.i18n.messages.MessageLevel; +import org.apache.knox.gateway.i18n.messages.Messages; +import org.apache.knox.gateway.i18n.messages.StackTrace; + +@Messages(logger = "org.apache.knox.gateway.service.health") +public interface KnoxS3ServiceMessages { + @Message(level = MessageLevel.INFO, text = "{0}") + void basicInfo(String original); + + @Message(level = MessageLevel.ERROR, text = "Unable to get health stats for {0}, due to {1}") + void logException(String name, @StackTrace(level = MessageLevel.DEBUG) Exception e); +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/knox/blob/43567b0a/gateway-service-knoxs3/src/main/java/org/apache/knox/gateway/service/knoxs3/S3BucketsResource.java ---------------------------------------------------------------------- diff --git a/gateway-service-knoxs3/src/main/java/org/apache/knox/gateway/service/knoxs3/S3BucketsResource.java b/gateway-service-knoxs3/src/main/java/org/apache/knox/gateway/service/knoxs3/S3BucketsResource.java new file mode 100644 index 0000000..d0b25d7 --- /dev/null +++ b/gateway-service-knoxs3/src/main/java/org/apache/knox/gateway/service/knoxs3/S3BucketsResource.java @@ -0,0 +1,199 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.service.knoxs3; + +import org.apache.knox.gateway.i18n.messages.MessagesFactory; +import org.apache.knox.gateway.services.GatewayServices; +import org.apache.knox.gateway.util.JsonUtils; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.Bucket; +import javax.annotation.PostConstruct; +import javax.servlet.ServletContext; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; +import java.io.PrintWriter; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import static javax.ws.rs.core.MediaType.APPLICATION_JSON; + + +@Path(S3BucketsResource.RESOURCE_PATH) +public class S3BucketsResource { + private static final String BUCKETS_API_PATH = "buckets"; + private static final String BUCKET_API_PATH = "buckets/{id}"; + private static KnoxS3ServiceMessages log = MessagesFactory.get(KnoxS3ServiceMessages.class); + private static final String VERSION_TAG = "v1"; + static final String RESOURCE_PATH = "/knoxs3/" + VERSION_TAG; + + private static final String CONTENT_TYPE = "application/json"; + private static final String CACHE_CONTROL = "Cache-Control"; + private static final String NO_CACHE = "must-revalidate,no-cache,no-store"; + + private KnoxS3ClientBuilder s3b = new KnoxS3ClientBuilder(); + + @Context + HttpServletRequest request; + + @Context + private HttpServletResponse response; + + @Context + ServletContext context; + + @PostConstruct + public void init() { + s3b.init(getProperties()); + } + + private Properties getProperties() { + Properties props = new Properties(); + String paramName = null; + Enumeration<String> e = context.getInitParameterNames(); + while (e.hasMoreElements()) { + paramName = (String)e.nextElement(); + if (paramName.startsWith("s3.")) { + props.setProperty(paramName, context.getInitParameter(paramName)); + } + } + + return props; + } + + @PUT + @Produces({APPLICATION_JSON}) + @Path(BUCKET_API_PATH) + public Response createBucket(@PathParam("id") String id) { + // use this to get to idbroker service once moved there + GatewayServices services = (GatewayServices) request.getServletContext() + .getAttribute(GatewayServices.GATEWAY_SERVICES_ATTRIBUTE); + return getCreateBucketResponse(id); + } + + private Response getCreateBucketResponse(String id) { + response.setStatus(HttpServletResponse.SC_OK); + response.setHeader(CACHE_CONTROL, NO_CACHE); + response.setContentType(CONTENT_TYPE); + PrintWriter writer = null; + try { + writer = response.getWriter(); + doCreateBucket(id); + } catch (Exception ioe) { + log.logException("create", ioe); + return Response.serverError().entity(String.format("Failed to reply correctly due to : %s ", ioe)).build(); + } finally { + if (writer != null) { + writer.close(); + } + } + return Response.ok().build(); + } + + private void doCreateBucket(String id) { + AmazonS3 s3 = s3b.getS3Client(); + s3.createBucket(id); + } + + @DELETE + @Produces({APPLICATION_JSON}) + @Path(BUCKET_API_PATH) + public Response deleteBucket(@PathParam("id") String id) { + // use this to get to idbroker service once moved there + GatewayServices services = (GatewayServices) request.getServletContext() + .getAttribute(GatewayServices.GATEWAY_SERVICES_ATTRIBUTE); + return getDeleteBucketResponse(id); + + } + + private Response getDeleteBucketResponse(String id) { + response.setStatus(HttpServletResponse.SC_OK); + response.setHeader(CACHE_CONTROL, NO_CACHE); + response.setContentType(CONTENT_TYPE); + PrintWriter writer = null; + try { + writer = response.getWriter(); + doDeleteBucket(id); + } catch (Exception ioe) { + log.logException("delete", ioe); + return Response.serverError().entity(String.format("Failed to reply correctly due to : %s ", ioe)).build(); + } finally { + if (writer != null) { + writer.close(); + } + } + return Response.ok().build(); + } + + private void doDeleteBucket(String id) { + AmazonS3 s3 = s3b.getS3Client(); + s3.deleteBucket(id); + } + + @GET + @Produces({APPLICATION_JSON}) + @Path(BUCKETS_API_PATH) + public Response listBuckets() { + // use this to get to idbroker service once moved there + GatewayServices services = (GatewayServices) request.getServletContext() + .getAttribute(GatewayServices.GATEWAY_SERVICES_ATTRIBUTE); + return getListBucketsResponse(); + } + + private Response getListBucketsResponse() { + response.setStatus(HttpServletResponse.SC_OK); + response.setHeader(CACHE_CONTROL, NO_CACHE); + response.setContentType(CONTENT_TYPE); + PrintWriter writer = null; + try { + writer = response.getWriter(); + writer.println(getBuckets()); + } catch (Exception e) { + log.logException("list", e); + return Response.serverError().entity(String.format("Failed to reply correctly due to : %s ", e)).build(); + } finally { + if (writer != null) { + writer.close(); + } + } + return Response.ok().build(); + } + + private String getBuckets() { + AmazonS3 s3 = s3b.getS3Client(); + Map<String, Object> bucketMap = new HashMap<String, Object>(); + List<Bucket> buckets = s3.listBuckets(); + for (Bucket bucket : buckets) { + System.out.println(bucket.getName()); + bucketMap.put(bucket.getName(), bucket); + } + return JsonUtils.renderAsJsonString(bucketMap); + } + +} http://git-wip-us.apache.org/repos/asf/knox/blob/43567b0a/gateway-service-knoxs3/src/main/java/org/apache/knox/gateway/service/knoxs3/deploy/KnoxS3ServiceDeploymentContributor.java ---------------------------------------------------------------------- diff --git a/gateway-service-knoxs3/src/main/java/org/apache/knox/gateway/service/knoxs3/deploy/KnoxS3ServiceDeploymentContributor.java b/gateway-service-knoxs3/src/main/java/org/apache/knox/gateway/service/knoxs3/deploy/KnoxS3ServiceDeploymentContributor.java new file mode 100644 index 0000000..2a7fdb6 --- /dev/null +++ b/gateway-service-knoxs3/src/main/java/org/apache/knox/gateway/service/knoxs3/deploy/KnoxS3ServiceDeploymentContributor.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.service.knoxs3.deploy; + +import org.apache.knox.gateway.jersey.JerseyServiceDeploymentContributorBase; + +public class KnoxS3ServiceDeploymentContributor extends JerseyServiceDeploymentContributorBase { + + /* (non-Javadoc) + * @see ServiceDeploymentContributor#getRole() + */ + @Override + public String getRole() { + return "KNOXS3"; + } + + /* (non-Javadoc) + * @see ServiceDeploymentContributor#getName() + */ + @Override + public String getName() { + return "KnoxS3Service"; + } + + /* (non-Javadoc) + * @see JerseyServiceDeploymentContributorBase#getPackages() + */ + @Override + protected String[] getPackages() { + return new String[]{"org.apache.knox.gateway.service.knoxs3"}; + } + + /* (non-Javadoc) + * @see JerseyServiceDeploymentContributorBase#getPatterns() + */ + @Override + protected String[] getPatterns() { + return new String[]{"*/**?**"}; + } +} http://git-wip-us.apache.org/repos/asf/knox/blob/43567b0a/gateway-service-knoxs3/src/main/resources/META-INF/services/org.apache.knox.gateway.deploy.ServiceDeploymentContributor ---------------------------------------------------------------------- diff --git a/gateway-service-knoxs3/src/main/resources/META-INF/services/org.apache.knox.gateway.deploy.ServiceDeploymentContributor b/gateway-service-knoxs3/src/main/resources/META-INF/services/org.apache.knox.gateway.deploy.ServiceDeploymentContributor new file mode 100644 index 0000000..fcfbdd7 --- /dev/null +++ b/gateway-service-knoxs3/src/main/resources/META-INF/services/org.apache.knox.gateway.deploy.ServiceDeploymentContributor @@ -0,0 +1,19 @@ +########################################################################## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +########################################################################## + +org.apache.knox.gateway.service.health.deploy.HealthServiceDeploymentContributor \ No newline at end of file http://git-wip-us.apache.org/repos/asf/knox/blob/43567b0a/gateway-service-knoxs3/src/test/java/org/apache/knox/gateway/service/knoxs3/S3BucketsResourceTest.java ---------------------------------------------------------------------- diff --git a/gateway-service-knoxs3/src/test/java/org/apache/knox/gateway/service/knoxs3/S3BucketsResourceTest.java b/gateway-service-knoxs3/src/test/java/org/apache/knox/gateway/service/knoxs3/S3BucketsResourceTest.java new file mode 100644 index 0000000..d75c7ce --- /dev/null +++ b/gateway-service-knoxs3/src/test/java/org/apache/knox/gateway/service/knoxs3/S3BucketsResourceTest.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.service.knoxs3; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +import org.apache.knox.gateway.util.JsonUtils; +import org.junit.Test; + +public class S3BucketsResourceTest { + @Test + public void testPolicyCreation() { + + String policy = "{\n" + + " \"Version\": \"2012-10-17\",\n" + + " \"Statement\": [\n" + + " {\n" + + " \"Effect\": \"Allow\",\n" + + " \"Action\": [\n" + + " \"s3:Get*\",\n" + + " \"s3:List*\"\n" + + // " \"s3:Delete*\"\n" + + " ],\n" + + " \"Resource\": \"*\"\n" + + " }\n" + + " ]\n" + + "}"; + System.out.println(policy); + + HashMap<String, Object> policyModel = new HashMap<String, Object>(); + policyModel.put("Version", "2012-10-17"); + ArrayList<Map<String, Object>> statement = new ArrayList<Map<String, Object>>(); + + policyModel.put("Version", "2012-10-17"); + policyModel.put("Statement", statement ); + HashMap<String, Object> statementMap = new HashMap<String, Object>(); + statementMap.put("Effect", "Allow"); + ArrayList<String> actionArray = new ArrayList<String>(); + actionArray.add("s3:Get*"); + actionArray.add("s3:List*"); + statementMap.put("Action", actionArray ); + statement.add(statementMap); + policyModel.put("Resource", "*"); + + System.out.println(JsonUtils.renderAsJsonString(policyModel)); + + S3PolicyModel model = new S3PolicyModel(); + model.setEffect("Allow"); + model.addAction("s3:Get*"); + model.addAction("s3:List*"); + model.setResource("*"); + System.out.println(model); + + model = new S3PolicyModel(); + model.setEffect("Allow"); + model.addAction("s3:Get*"); + model.addAction("s3:List*"); + model.addResource("this"); + model.addResource("that"); + System.out.println(model); +} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/knox/blob/43567b0a/gateway-spi/src/main/java/org/apache/knox/gateway/services/s3client/KnoxS3ClientBuilder.java ---------------------------------------------------------------------- diff --git a/gateway-spi/src/main/java/org/apache/knox/gateway/services/s3client/KnoxS3ClientBuilder.java b/gateway-spi/src/main/java/org/apache/knox/gateway/services/s3client/KnoxS3ClientBuilder.java new file mode 100644 index 0000000..cebfba0 --- /dev/null +++ b/gateway-spi/src/main/java/org/apache/knox/gateway/services/s3client/KnoxS3ClientBuilder.java @@ -0,0 +1,219 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.service.knoxs3; + +import java.security.AccessController; +import java.security.Principal; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import javax.security.auth.Subject; + +import org.apache.knox.gateway.security.GroupPrincipal; +import org.apache.knox.gateway.security.ImpersonatedPrincipal; +import org.apache.knox.gateway.security.PrimaryPrincipal; + +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicSessionCredentials; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.securitytoken.AWSSecurityTokenService; +import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder; +import com.amazonaws.services.securitytoken.model.Credentials; +import com.amazonaws.services.securitytoken.model.GetFederationTokenRequest; +import com.amazonaws.services.securitytoken.model.GetFederationTokenResult; + +public class KnoxS3ClientBuilder { + private Map<String, PolicyConfig> userPolicyConfig = new HashMap<String, PolicyConfig>(); + private Map<String, PolicyConfig> groupPolicyConfig = new HashMap<String, PolicyConfig>(); + + public KnoxS3ClientBuilder() { + } + + public AmazonS3 getS3Client() { + BasicSessionCredentials sessionCredentials = getSessionCredentials(); + if (sessionCredentials == null) { + throw new RuntimeException("No S3 credentials available."); + } + + AmazonS3 s3 = AmazonS3ClientBuilder.standard().withRegion(Regions.US_EAST_1) + .withCredentials(new AWSStaticCredentialsProvider(sessionCredentials)).build(); + return s3; + } + + public void init(Properties context) { + buildPolicyMaps(context); + } + + private void buildPolicyMaps(Properties context) { + /* + <service> + <role>KNOXS3</role> + <param> + <name>s3.user.policy.action.guest</name> + <value>s3:Get*,s3:List*</value> + </param> + <param> + <name>s3.user.policy.resource.guest</name> + <value>*</value> + </param> + <param> + <name>s3.group.policy.action.admin</name> + <value>*</value> + </param> + <param> + <name>s3.group.policy.resource.admin</name> + <value>*</value> + </param> + </service> + */ + + String paramName = null; + Enumeration<Object> e = context.keys(); + while (e.hasMoreElements()) { + paramName = (String)e.nextElement(); + if (paramName.startsWith("s3.")) { + String[] elements = paramName.split("\\."); + if (elements[1].equals("user")) { + PolicyConfig policy = userPolicyConfig.get(elements[4]); + if (policy == null) { + policy = new PolicyConfig(); + userPolicyConfig.put(elements[4], policy); + } + if (elements[3].equals("action")) { + policy.actions=context.getProperty(paramName); + } else { + policy.resources=context.getProperty(paramName); + } + if (policy.actions != null && policy.resources != null) { + buildS3PolicyModel(policy); + } + }else if (elements[1].equals("group")) { + PolicyConfig policy = groupPolicyConfig.get(elements[4]); + if (policy == null) { + policy = new PolicyConfig(); + groupPolicyConfig.put(elements[4], policy); + } + if (elements[3].equals("action")) { + policy.actions=context.getProperty(paramName); + } else { + policy.resources=context.getProperty(paramName); + } + if (policy.actions != null && policy.resources != null) { + buildS3PolicyModel(policy); + } + } + } + } + } + + private void buildS3PolicyModel(PolicyConfig policy) { + S3PolicyModel model = new S3PolicyModel(); + model.setEffect("Allow"); + String[] actions = policy.actions.split(","); + for (int i = 0; i < actions.length; i++) { + model.addAction(actions[i]); + } + String[] resources = policy.resources.split(","); + if (resources.length > 1) { + for (int i = 0; i < resources.length; i++) { + model.addResource(resources[i]); + } + } else { + model.setResource(resources[0]); + } + policy.policy = model.toString(); + } + + private BasicSessionCredentials getSessionCredentials() { + AWSSecurityTokenService sts_client = AWSSecurityTokenServiceClientBuilder.standard().withRegion(Regions.US_EAST_1).build(); + String policy = ""; + BasicSessionCredentials sessionCredentials = null; + try { + String username = null; + Subject subject = Subject.getSubject(AccessController.getContext()); + username = getEffectiveUserName(subject); + policy = buildPolicy(username, subject); + + if (policy != null) { + GetFederationTokenRequest request = new GetFederationTokenRequest(username).withPolicy(policy); + GetFederationTokenResult result = sts_client.getFederationToken(request); + System.out.println(result.getCredentials()); + + Credentials session_creds = result.getCredentials(); + sessionCredentials = new BasicSessionCredentials( + session_creds.getAccessKeyId(), + session_creds.getSecretAccessKey(), + session_creds.getSessionToken()); + } + } catch (Exception e) { + e.printStackTrace(); + } + return sessionCredentials; + } + + private String getEffectiveUserName(Subject subject) { + String username; + Principal primaryPrincipal = (Principal)subject.getPrincipals(PrimaryPrincipal.class).toArray()[0]; + Object[] impersonations = subject.getPrincipals(ImpersonatedPrincipal.class).toArray(); + if (impersonations.length > 0) { + username = ((Principal)impersonations[0]).getName(); + } + else { + username = primaryPrincipal.getName(); + } + return username; + } + + private String buildPolicy(String username, Subject subject) { + String policy = null; + List<String> groupNames = new ArrayList<String>(); + Object[] groups = subject.getPrincipals(GroupPrincipal.class).toArray(); + for (int i = 0; i < groups.length; i++) { + groupNames.add( + ((Principal)groups[0]).getName()); + } + + PolicyConfig config = userPolicyConfig.get(username); + if (config == null) { + // check for a group policy match + for (String groupName : groupNames) { + config = groupPolicyConfig.get(groupName); + if (config != null) { + // just accept first match for now + break; + } + } + } + if (config != null) { + policy = config.policy; + } + return policy; + } + + private class PolicyConfig { + public String actions = null; + public String resources = null; + public String policy = null; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/knox/blob/43567b0a/gateway-spi/src/main/java/org/apache/knox/gateway/services/s3client/S3PolicyModel.java ---------------------------------------------------------------------- diff --git a/gateway-spi/src/main/java/org/apache/knox/gateway/services/s3client/S3PolicyModel.java b/gateway-spi/src/main/java/org/apache/knox/gateway/services/s3client/S3PolicyModel.java new file mode 100644 index 0000000..f7f3084 --- /dev/null +++ b/gateway-spi/src/main/java/org/apache/knox/gateway/services/s3client/S3PolicyModel.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.service.knoxs3; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +import org.apache.knox.gateway.util.JsonUtils; + +public class S3PolicyModel { + HashMap<String, Object> policyModel = new HashMap<String, Object>(); + ArrayList<String> actionArray = new ArrayList<String>(); + HashMap<String, Object> statementMap = new HashMap<String, Object>(); + ArrayList<String> resourcesArray = new ArrayList<String>(); + + public S3PolicyModel() { + policyModel.put("Version", "2012-10-17"); + ArrayList<Map<String, Object>> statement = new ArrayList<Map<String, Object>>(); + policyModel.put("Statement", statement ); + statement.add(statementMap); + statementMap.put("Action", actionArray ); + statementMap.put("Resource", resourcesArray); + } + + public void setEffect(String effect) { + statementMap.put("Effect", effect); + } + + public void addAction(String action) { + actionArray.add(action); + } + + public void addResource(String resource) { + resourcesArray.add(resource); + } + + public void setResource(String resource) { + statementMap.put("Resource", resource); + } + + public String toString() { + return JsonUtils.renderAsJsonString(policyModel); + } +} http://git-wip-us.apache.org/repos/asf/knox/blob/43567b0a/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 3297aa8..1c49602 100644 --- a/pom.xml +++ b/pom.xml @@ -78,6 +78,7 @@ <module>gateway-service-knoxsso</module> <module>gateway-service-knoxssout</module> <module>gateway-service-knoxtoken</module> + <module>gateway-service-knoxs3</module> <module>gateway-service-health</module> <module>gateway-service-webhdfs</module> <module>gateway-service-tgs</module> @@ -633,6 +634,11 @@ </dependency> <dependency> <groupId>${gateway-group}</groupId> + <artifactId>gateway-service-knoxs3</artifactId> + <version>${gateway-version}</version> + </dependency> + <dependency> + <groupId>${gateway-group}</groupId> <artifactId>gateway-service-admin</artifactId> <version>${gateway-version}</version> </dependency>
