This is an automated email from the ASF dual-hosted git repository.
swagle pushed a commit to branch HDDS-4440-s3-performance
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-4440-s3-performance by
this push:
new 1b891fb HDDS-5612. Create Ozone s3 authentication token and client
authentication transport (#2655)
1b891fb is described below
commit 1b891fb64f9343b1a43ffc2671607a143c9bd92a
Author: Neil Joshi <[email protected]>
AuthorDate: Mon Oct 4 12:39:14 2021 -0600
HDDS-5612. Create Ozone s3 authentication token and client authentication
transport (#2655)
* Modifications made as per reviewers comments. Clean up code, remove
unnecessary code (not used or required). Changed OzoneManagerProtocol
OmRequest from using multiple optional single variable entries for s3 auth to
using an optional S3Authentication structure containing s3 auth info. Added
default enabling of s3 Grpc on om through configuration setting in compose
ozone cluster and re-enabled s3 smoke tests for that cluster.
---
.../ozone/om/protocolPB/GrpcOmTransport.java | 2 +
.../dist/src/main/compose/ozone/docker-config | 1 +
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 16 +-
.../src/main/proto/OmClientProtocol.proto | 11 ++
.../apache/hadoop/ozone/s3/OzoneClientCache.java | 73 +++++++++
.../hadoop/ozone/s3/OzoneClientProducer.java | 104 +------------
.../java/org/apache/hadoop/ozone/s3/UgiFilter.java | 169 +++++++++++++++++++++
.../ozone/s3/signature/AWSSignatureProcessor.java | 46 +++---
.../ozone/s3/signature/StringToSignProducer.java | 21 +--
.../resources/webapps/s3gateway/WEB-INF/web.xml | 8 +
.../hadoop/ozone/s3/TestOzoneClientProducer.java | 59 -------
11 files changed, 294 insertions(+), 216 deletions(-)
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/GrpcOmTransport.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/GrpcOmTransport.java
index 68c1d2c..752f29e 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/GrpcOmTransport.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/GrpcOmTransport.java
@@ -76,6 +76,8 @@ public class GrpcOmTransport implements OmTransport {
maxSize = conf.getInt(OZONE_OM_GRPC_MAXIMUM_RESPONSE_LENGTH,
OZONE_OM_GRPC_MAXIMUM_RESPONSE_LENGTH_DEFAULT);
+
+ start();
}
public void start() {
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-config
b/hadoop-ozone/dist/src/main/compose/ozone/docker-config
index d71b3ac..4023bb1 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-config
@@ -35,6 +35,7 @@ OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay=1m
OZONE-SITE.XML_ozone.datanode.pipeline.limit=1
OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
OZONE-SITE.XML_hdds.container.report.interval=60s
+OZONE-SITE.XML_ozone.om.s3.grpc.server_enabled=true
OZONE_CONF_DIR=/etc/hadoop
OZONE_LOG_DIR=/var/log/hadoop
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test.sh
b/hadoop-ozone/dist/src/main/compose/ozone/test.sh
index dfde1c1..7695acc 100755
--- a/hadoop-ozone/dist/src/main/compose/ozone/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone/test.sh
@@ -37,9 +37,9 @@ execute_robot_test scm gdpr
execute_robot_test scm security/ozone-secure-token.robot
-#for bucket in link generated; do
-# execute_robot_test scm -v BUCKET:${bucket} -N s3-${bucket} s3
-#done
+for bucket in link generated; do
+ execute_robot_test scm -v BUCKET:${bucket} -N s3-${bucket} s3
+done
execute_robot_test scm recon
@@ -66,11 +66,11 @@ start_docker_env
execute_robot_test scm -v SCHEME:ofs -v BUCKET_TYPE:link -N
ozonefs-prefix-ofs-link ozonefs/ozonefs.robot
execute_robot_test scm -v SCHEME:o3fs -v BUCKET_TYPE:bucket -N
ozonefs-prefix-o3fs-bucket ozonefs/ozonefs.robot
-#execute_robot_test scm -v BUCKET:${bucket} -N
s3-${bucket}-prefix-layout-objectputget s3/objectputget.robot
-#execute_robot_test scm -v BUCKET:${bucket} -N
s3-${bucket}-prefix-layout-objectdelete s3/objectdelete.robot
-#execute_robot_test scm -v BUCKET:${bucket} -N
s3-${bucket}-prefix-layout-objectcopy s3/objectcopy.robot
-#execute_robot_test scm -v BUCKET:${bucket} -N
s3-${bucket}-prefix-layout-objectmultidelete s3/objectmultidelete.robot
-#execute_robot_test scm -v BUCKET:${bucket} -N
s3-${bucket}-prefix-layout-MultipartUpload s3/MultipartUpload.robot
+execute_robot_test scm -v BUCKET:${bucket} -N
s3-${bucket}-prefix-layout-objectputget s3/objectputget.robot
+execute_robot_test scm -v BUCKET:${bucket} -N
s3-${bucket}-prefix-layout-objectdelete s3/objectdelete.robot
+execute_robot_test scm -v BUCKET:${bucket} -N
s3-${bucket}-prefix-layout-objectcopy s3/objectcopy.robot
+execute_robot_test scm -v BUCKET:${bucket} -N
s3-${bucket}-prefix-layout-objectmultidelete s3/objectmultidelete.robot
+execute_robot_test scm -v BUCKET:${bucket} -N
s3-${bucket}-prefix-layout-MultipartUpload s3/MultipartUpload.robot
stop_docker_env
diff --git
a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index 63a46a0..c5eb4a9 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -184,6 +184,8 @@ message OMRequest {
optional RevokeS3SecretRequest RevokeS3SecretRequest =
93;
optional PurgePathsRequest purgePathsRequest =
94;
+
+ optional S3Authentication s3Authentication =
95;
}
message OMResponse {
@@ -1344,6 +1346,15 @@ message UpdateGetS3SecretRequest {
}
/**
+ This will be used by OM to authenicate S3 gateway requests on a per request
basis.
+*/
+message S3Authentication {
+ required string stringToSign = 1;
+ required string signature = 2;
+ required string accessId = 3;
+}
+
+/**
The OM service that takes care of Ozone namespace.
*/
service OzoneManagerService {
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientCache.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientCache.java
new file mode 100644
index 0000000..c00b1cb
--- /dev/null
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientCache.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.s3;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.PreDestroy;
+import javax.enterprise.context.ApplicationScoped;
+import java.io.IOException;
+
+/**
+ * Cached ozone client for s3 requests.
+ */
+@ApplicationScoped
+public final class OzoneClientCache {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(OzoneClientCache.class);
+ // single, cached OzoneClient established on first connection
+ // for s3g gRPC OmTransport, OmRequest - OmResponse channel
+ private static OzoneClientCache instance;
+ private OzoneClient client;
+
+ private OzoneClientCache(String omServiceID,
+ OzoneConfiguration ozoneConfiguration)
+ throws IOException {
+ try {
+ if (omServiceID == null) {
+ client = OzoneClientFactory.getRpcClient(ozoneConfiguration);
+ } else {
+ // As in HA case, we need to pass om service ID.
+ client = OzoneClientFactory.getRpcClient(omServiceID,
+ ozoneConfiguration);
+ }
+ } catch (IOException e) {
+ LOG.warn("cannot create OzoneClient", e);
+ throw e;
+ }
+ }
+
+ public static OzoneClient getOzoneClientInstance(String omServiceID,
+ OzoneConfiguration
+ ozoneConfiguration)
+ throws IOException {
+ if (instance == null) {
+ instance = new OzoneClientCache(omServiceID, ozoneConfiguration);
+ }
+ return instance.client;
+ }
+
+ @PreDestroy
+ public void destroy() throws IOException {
+ client.close();
+ }
+}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java
index 83809c3..e675223 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java
@@ -17,36 +17,17 @@
*/
package org.apache.hadoop.ozone.s3;
-import javax.annotation.PreDestroy;
import javax.enterprise.context.RequestScoped;
import javax.enterprise.inject.Produces;
import javax.inject.Inject;
import javax.ws.rs.WebApplicationException;
-import javax.ws.rs.container.ContainerRequestContext;
-import javax.ws.rs.core.Context;
import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.security.PrivilegedExceptionAction;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.ozone.OzoneSecurityUtil;
import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
-import org.apache.hadoop.ozone.s3.signature.SignatureInfo;
-import org.apache.hadoop.ozone.s3.signature.SignatureInfo.Version;
-import org.apache.hadoop.ozone.s3.signature.SignatureProcessor;
-import org.apache.hadoop.ozone.s3.signature.StringToSignProducer;
-import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import com.google.common.annotations.VisibleForTesting;
-import static
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMTokenProto.Type.S3AUTHINFO;
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INTERNAL_ERROR;
-import static
org.apache.hadoop.ozone.s3.exception.S3ErrorTable.MALFORMED_HEADER;
-import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -62,20 +43,11 @@ public class OzoneClientProducer {
private OzoneClient client;
@Inject
- private SignatureProcessor signatureProcessor;
-
- @Inject
private OzoneConfiguration ozoneConfiguration;
@Inject
- private Text omService;
-
- @Inject
private String omServiceID;
- @Context
- private ContainerRequestContext context;
-
@Produces
public OzoneClient createClient() throws WebApplicationException,
IOException {
@@ -83,60 +55,13 @@ public class OzoneClientProducer {
return client;
}
- @PreDestroy
- public void destroy() throws IOException {
- client.close();
- }
-
private OzoneClient getClient(OzoneConfiguration config)
throws WebApplicationException {
OzoneClient ozoneClient = null;
try {
- SignatureInfo signatureInfo = signatureProcessor.parseSignature();
-
- String stringToSign = "";
- if (signatureInfo.getVersion() == Version.V4) {
- stringToSign =
- StringToSignProducer.createSignatureBase(signatureInfo, context);
- }
-
- String awsAccessId = signatureInfo.getAwsAccessId();
- validateAccessId(awsAccessId);
-
- UserGroupInformation remoteUser =
- UserGroupInformation.createRemoteUser(awsAccessId);
- if (OzoneSecurityUtil.isSecurityEnabled(config)) {
- LOG.debug("Creating s3 auth info for client.");
-
- if (signatureInfo.getVersion() == Version.NONE) {
- throw MALFORMED_HEADER;
- }
-
- OzoneTokenIdentifier identifier = new OzoneTokenIdentifier();
- identifier.setTokenType(S3AUTHINFO);
- identifier.setStrToSign(stringToSign);
- identifier.setSignature(signatureInfo.getSignature());
- identifier.setAwsAccessId(awsAccessId);
- identifier.setOwner(new Text(awsAccessId));
- if (LOG.isTraceEnabled()) {
- LOG.trace("Adding token for service:{}", omService);
- }
- Token<OzoneTokenIdentifier> token = new Token(identifier.getBytes(),
- identifier.getSignature().getBytes(StandardCharsets.UTF_8),
- identifier.getKind(),
- omService);
- remoteUser.addToken(token);
-
- }
ozoneClient =
- remoteUser.doAs((PrivilegedExceptionAction<OzoneClient>) () -> {
- return createOzoneClient();
- });
- } catch (OS3Exception ex) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Error during Client Creation: ", ex);
- }
- throw wrapOS3Exception(ex);
+ OzoneClientCache.getOzoneClientInstance(omServiceID,
+ ozoneConfiguration);
} catch (Exception e) {
// For any other critical errors during object creation throw Internal
// error.
@@ -148,35 +73,10 @@ public class OzoneClientProducer {
return ozoneClient;
}
- @NotNull
- @VisibleForTesting
- OzoneClient createOzoneClient() throws IOException {
- if (omServiceID == null) {
- return OzoneClientFactory.getRpcClient(ozoneConfiguration);
- } else {
- // As in HA case, we need to pass om service ID.
- return OzoneClientFactory.getRpcClient(omServiceID,
- ozoneConfiguration);
- }
- }
-
- // ONLY validate aws access id when needed.
- private void validateAccessId(String awsAccessId) throws Exception {
- if (awsAccessId == null || awsAccessId.equals("")) {
- LOG.error("Malformed s3 header. awsAccessID: ", awsAccessId);
- throw wrapOS3Exception(MALFORMED_HEADER);
- }
- }
-
public void setOzoneConfiguration(OzoneConfiguration config) {
this.ozoneConfiguration = config;
}
- @VisibleForTesting
- public void setSignatureParser(SignatureProcessor awsSignatureProcessor) {
- this.signatureProcessor = awsSignatureProcessor;
- }
-
private WebApplicationException wrapOS3Exception(OS3Exception os3Exception) {
return new WebApplicationException(os3Exception,
os3Exception.getHttpCode());
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/UgiFilter.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/UgiFilter.java
new file mode 100644
index 0000000..24dd186
--- /dev/null
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/UgiFilter.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.s3;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OzoneSecurityUtil;
+import org.apache.hadoop.ozone.s3.exception.OS3Exception;
+import org.apache.hadoop.ozone.s3.signature.SignatureInfo;
+import org.apache.hadoop.ozone.s3.signature.StringToSignProducer;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.WebApplicationException;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.PrivilegedExceptionAction;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Enumeration;
+import org.apache.hadoop.ozone.s3.signature.AWSSignatureProcessor;
+
+import com.google.common.annotations.VisibleForTesting;
+import static
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMTokenProto.Type.S3AUTHINFO;
+import static
org.apache.hadoop.ozone.s3.exception.S3ErrorTable.MALFORMED_HEADER;
+
+/**
+ * Preprocessing filter for every request.
+ * - creates OzoneToken containing aws signature
+ * aws id and stringToSign for aws authenication; stores OzoneToken in
+ * thread local variable (UserGroupInformation object) avail to all
+ * s3 rest command endpoints
+ */
+public class UgiFilter implements Filter {
+ public static final Logger LOG = LoggerFactory.getLogger(UgiFilter.class);
+
+ @Inject
+ private OzoneConfiguration ozoneConfiguration;
+ @Inject
+ private Text omService;
+
+ @Override
+ public void init(FilterConfig filterConfig) throws ServletException {
+
+ }
+
+ @Override
+ public void doFilter(ServletRequest servletRequest,
+ ServletResponse servletResponse, FilterChain
filterChain)
+ throws IOException, ServletException {
+ Map<String, String> headerMap = new HashMap<>();
+ HttpServletRequest httpRequest = (HttpServletRequest) servletRequest;
+ Enumeration<String> headerNames = httpRequest.getHeaderNames();
+ while (headerNames.hasMoreElements()) {
+ String headerKey = headerNames.nextElement();
+ headerMap.put(headerKey, httpRequest.getHeader(headerKey));
+ LOG.info("request {} : {}", headerKey, httpRequest.getHeader(headerKey));
+ }
+ AWSSignatureProcessor signature = new AWSSignatureProcessor(headerMap,
+ httpRequest.getParameterMap());
+ SignatureInfo signatureInfo;
+ String stringToSign = "";
+ String awsAccessId = "";
+ try {
+ signatureInfo = signature.parseSignature();
+ if (signatureInfo.getVersion() == SignatureInfo.Version.V4) {
+ stringToSign =
+ StringToSignProducer.createSignatureBase(signatureInfo,
+ httpRequest.getScheme(),
+ httpRequest.getMethod(),
+ httpRequest.getPathInfo(),
+ AWSSignatureProcessor.LowerCaseKeyStringMap
+ .fromHeaderMap(headerMap),
+ StringToSignProducer.fromMultiValueToSingleValueMap(
+ httpRequest.getParameterMap()));
+ }
+ awsAccessId = signatureInfo.getAwsAccessId();
+ validateAccessId(awsAccessId);
+ } catch (Throwable t) {
+ throw new IOException(t);
+ }
+
+ UserGroupInformation remoteUser =
+ UserGroupInformation.createRemoteUser(awsAccessId);
+
+ if (OzoneSecurityUtil.isSecurityEnabled(ozoneConfiguration)) {
+ LOG.debug("Creating s3 auth info for client.");
+
+ if (signatureInfo.getVersion() == SignatureInfo.Version.NONE) {
+ //throw MALFORMED_HEADER;
+ throw new IOException("MALFORMED_HEADER");
+ }
+
+ OzoneTokenIdentifier identifier = new OzoneTokenIdentifier();
+ identifier.setTokenType(S3AUTHINFO);
+ identifier.setStrToSign(stringToSign);
+ identifier.setSignature(signatureInfo.getSignature());
+ identifier.setAwsAccessId(awsAccessId);
+ identifier.setOwner(new Text(awsAccessId));
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Adding token for service:{}", omService);
+ }
+ Token<OzoneTokenIdentifier> token = new Token(identifier.getBytes(),
+ identifier.getSignature().getBytes(StandardCharsets.UTF_8),
+ identifier.getKind(),
+ omService);
+ remoteUser.addToken(token);
+ }
+ try {
+ remoteUser.doAs((PrivilegedExceptionAction<Void>) () -> {
+ filterChain.doFilter(httpRequest, servletResponse);
+ return null;
+ });
+ } catch (InterruptedException e) {
+ throw new IOException("Interrupted thread call doAs", e);
+ }
+ }
+
+ @Override
+ public void destroy() { }
+
+ private WebApplicationException wrapOS3Exception(OS3Exception os3Exception) {
+ return new WebApplicationException(os3Exception,
+ os3Exception.getHttpCode());
+ }
+
+ private void validateAccessId(String awsAccessId) throws Exception {
+ if (awsAccessId == null || awsAccessId.equals("")) {
+ LOG.error("Malformed s3 header. awsAccessID: {}", awsAccessId);
+ throw wrapOS3Exception(MALFORMED_HEADER);
+ }
+ }
+ @VisibleForTesting
+ public void setOzoneConfiguration(OzoneConfiguration config) {
+ this.ozoneConfiguration = config;
+ }
+
+ @VisibleForTesting
+ public void setOmService(Text omService) {
+ this.omService = omService;
+ }
+
+}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java
index 807b473..4757d86 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java
@@ -17,10 +17,6 @@
*/
package org.apache.hadoop.ozone.s3.signature;
-import javax.enterprise.context.RequestScoped;
-import javax.ws.rs.container.ContainerRequestContext;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MultivaluedMap;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -41,19 +37,24 @@ import org.slf4j.LoggerFactory;
* header. For more details refer to AWS documentation https://docs.aws
* .amazon.com/general/latest/gr/sigv4-create-canonical-request.html.
**/
-@RequestScoped
+
public class AWSSignatureProcessor implements SignatureProcessor {
private static final Logger LOG =
LoggerFactory.getLogger(AWSSignatureProcessor.class);
- @Context
- private ContainerRequestContext context;
+ private Map<String, String> headerMap;
+ private Map<String, String[]> parameterMap;
+ public AWSSignatureProcessor(Map<String, String> headerMap,
+ Map<String, String[]> parameterMap) {
+ this.headerMap = headerMap;
+ this.parameterMap = parameterMap;
+ }
public SignatureInfo parseSignature() throws OS3Exception {
LowerCaseKeyStringMap headers =
- LowerCaseKeyStringMap.fromHeaderMap(context.getHeaders());
+ LowerCaseKeyStringMap.fromHeaderMap(headerMap);
String authHeader = headers.get("Authorization");
@@ -62,7 +63,7 @@ public class AWSSignatureProcessor implements
SignatureProcessor {
headers.get(StringToSignProducer.X_AMAZ_DATE)));
signatureParsers.add(new AuthorizationV4QueryParser(
StringToSignProducer.fromMultiValueToSingleValueMap(
- context.getUriInfo().getQueryParameters())));
+ parameterMap)));
signatureParsers.add(new AuthorizationV2HeaderParser(authHeader));
SignatureInfo signatureInfo = null;
@@ -81,11 +82,6 @@ public class AWSSignatureProcessor implements
SignatureProcessor {
return signatureInfo;
}
- @VisibleForTesting
- public void setContext(ContainerRequestContext context) {
- this.context = context;
- }
-
/**
* A simple map which forces lower case key usage.
*/
@@ -98,28 +94,20 @@ public class AWSSignatureProcessor implements
SignatureProcessor {
}
public static LowerCaseKeyStringMap fromHeaderMap(
- MultivaluedMap<String,
+ Map<String,
String> rawHeaders
) {
-
//header map is MUTABLE. It's better to save it here. (with lower case
// keys!!!)
final LowerCaseKeyStringMap headers =
new LowerCaseKeyStringMap();
-
- for (Entry<String, List<String>> headerEntry : rawHeaders.entrySet()) {
- if (0 < headerEntry.getValue().size()) {
- String headerKey = headerEntry.getKey();
- if (headers.containsKey(headerKey)) {
- //multiple headers from the same type are combined
- headers.put(headerKey,
- headers.get(headerKey) + "," + headerEntry.getValue().get(0));
- } else {
- headers.put(headerKey, headerEntry.getValue().get(0));
- }
+ rawHeaders.forEach((k, v) -> {
+ if (headers.containsKey(k)) {
+ headers.put(k, headers.get(k) + "," + v);
+ } else {
+ headers.put(k, v);
}
- }
-
+ });
headers.fixContentType();
if (LOG.isTraceEnabled()) {
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java
index 3202a96..045ad9a 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java
@@ -17,8 +17,6 @@
*/
package org.apache.hadoop.ozone.s3.signature;
-import javax.ws.rs.container.ContainerRequestContext;
-import javax.ws.rs.core.MultivaluedMap;
import java.io.UnsupportedEncodingException;
import java.net.InetAddress;
import java.net.URI;
@@ -76,19 +74,6 @@ public final class StringToSignProducer {
private StringToSignProducer() {
}
- public static String createSignatureBase(
- SignatureInfo signatureInfo,
- ContainerRequestContext context
- ) throws Exception {
- return createSignatureBase(signatureInfo,
- context.getUriInfo().getRequestUri().getScheme(),
- context.getMethod(),
- context.getUriInfo().getRequestUri().getPath(),
- LowerCaseKeyStringMap.fromHeaderMap(context.getHeaders()),
- fromMultiValueToSingleValueMap(
- context.getUriInfo().getQueryParameters()));
- }
-
@VisibleForTesting
public static String createSignatureBase(
SignatureInfo signatureInfo,
@@ -137,11 +122,11 @@ public final class StringToSignProducer {
}
public static Map<String, String> fromMultiValueToSingleValueMap(
- MultivaluedMap<String, String> queryParameters
+ Map<String, String[]> queryParameters
) {
Map<String, String> result = new HashMap<>();
- for (String key : queryParameters.keySet()) {
- result.put(key, queryParameters.getFirst(key));
+ for (Map.Entry<String, String[]> entry : queryParameters.entrySet()) {
+ result.put(entry.getKey(), entry.getValue()[0]);
}
return result;
}
diff --git
a/hadoop-ozone/s3gateway/src/main/resources/webapps/s3gateway/WEB-INF/web.xml
b/hadoop-ozone/s3gateway/src/main/resources/webapps/s3gateway/WEB-INF/web.xml
index 79bf7b9..1b3c028 100644
---
a/hadoop-ozone/s3gateway/src/main/resources/webapps/s3gateway/WEB-INF/web.xml
+++
b/hadoop-ozone/s3gateway/src/main/resources/webapps/s3gateway/WEB-INF/web.xml
@@ -33,6 +33,10 @@
</filter-class>
</filter>
<filter>
+ <filter-name>perrequest-preprocess-signaturelocalthread</filter-name>
+ <filter-class>org.apache.hadoop.ozone.s3.UgiFilter</filter-class>
+ </filter>
+ <filter>
<filter-name>info-page-redirect</filter-name>
<filter-class>org.apache.hadoop.ozone.s3.RootPageDisplayFilter
</filter-class>
@@ -45,6 +49,10 @@
<filter-name>info-page-redirect</filter-name>
<url-pattern>/*</url-pattern>
</filter-mapping>
+ <filter-mapping>
+ <filter-name>perrequest-preprocess-signaturelocalthread</filter-name>
+ <url-pattern>/*</url-pattern>
+ </filter-mapping>
<listener>
<listener-class>org.jboss.weld.environment.servlet.Listener</listener-class>
</listener>
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestOzoneClientProducer.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestOzoneClientProducer.java
index 38776f1..e77c5a5 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestOzoneClientProducer.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestOzoneClientProducer.java
@@ -18,32 +18,18 @@
package org.apache.hadoop.ozone.s3;
import javax.ws.rs.WebApplicationException;
-import javax.ws.rs.container.ContainerRequestContext;
-import javax.ws.rs.core.MultivaluedHashMap;
-import javax.ws.rs.core.MultivaluedMap;
-import javax.ws.rs.core.UriInfo;
-import java.net.URI;
import java.util.Arrays;
import java.util.Collection;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.om.OMConfigKeys;
-import org.apache.hadoop.ozone.s3.signature.AWSSignatureProcessor;
-import static
org.apache.hadoop.ozone.s3.signature.SignatureParser.AUTHORIZATION_HEADER;
-import static
org.apache.hadoop.ozone.s3.signature.SignatureProcessor.CONTENT_MD5;
-import static
org.apache.hadoop.ozone.s3.signature.SignatureProcessor.CONTENT_TYPE;
-import static
org.apache.hadoop.ozone.s3.signature.SignatureProcessor.HOST_HEADER;
-import static
org.apache.hadoop.ozone.s3.signature.StringToSignProducer.X_AMAZ_DATE;
-import static
org.apache.hadoop.ozone.s3.signature.StringToSignProducer.X_AMZ_CONTENT_SHA256;
import static org.junit.Assert.fail;
-
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import org.mockito.Mockito;
/**
* Test class for @{@link OzoneClientProducer}.
@@ -52,37 +38,16 @@ import org.mockito.Mockito;
public class TestOzoneClientProducer {
private OzoneClientProducer producer;
- private MultivaluedMap<String, String> headerMap;
- private MultivaluedMap<String, String> queryMap;
- private String authHeader;
- private String contentMd5;
- private String host;
- private String amzContentSha256;
- private String date;
- private String contentType;
- private ContainerRequestContext context;
- private UriInfo uriInfo;
public TestOzoneClientProducer(
String authHeader, String contentMd5,
String host, String amzContentSha256, String date, String contentType
)
throws Exception {
- this.authHeader = authHeader;
- this.contentMd5 = contentMd5;
- this.host = host;
- this.amzContentSha256 = amzContentSha256;
- this.date = date;
- this.contentType = contentType;
producer = new OzoneClientProducer();
- headerMap = new MultivaluedHashMap<>();
- queryMap = new MultivaluedHashMap<>();
- uriInfo = Mockito.mock(UriInfo.class);
- context = Mockito.mock(ContainerRequestContext.class);
OzoneConfiguration config = new OzoneConfiguration();
config.setBoolean(OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY, true);
config.set(OMConfigKeys.OZONE_OM_ADDRESS_KEY, "");
- setupContext();
producer.setOzoneConfiguration(config);
}
@@ -132,28 +97,4 @@ public class TestOzoneClientProducer {
}
}
- private void setupContext() throws Exception {
- headerMap.putSingle(AUTHORIZATION_HEADER, authHeader);
- headerMap.putSingle(CONTENT_MD5, contentMd5);
- headerMap.putSingle(HOST_HEADER, host);
- headerMap.putSingle(X_AMZ_CONTENT_SHA256, amzContentSha256);
- headerMap.putSingle(X_AMAZ_DATE, date);
- headerMap.putSingle(CONTENT_TYPE, contentType);
-
- Mockito.when(uriInfo.getQueryParameters()).thenReturn(queryMap);
- Mockito.when(uriInfo.getRequestUri()).thenReturn(new URI(""));
-
- Mockito.when(context.getUriInfo()).thenReturn(uriInfo);
- Mockito.when(context.getHeaders()).thenReturn(headerMap);
- Mockito.when(context.getHeaderString(AUTHORIZATION_HEADER))
- .thenReturn(authHeader);
- Mockito.when(context.getUriInfo().getQueryParameters())
- .thenReturn(queryMap);
-
- AWSSignatureProcessor awsSignatureProcessor = new AWSSignatureProcessor();
- awsSignatureProcessor.setContext(context);
-
- producer.setSignatureParser(awsSignatureProcessor);
- }
-
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
