This is an automated email from the ASF dual-hosted git repository.
ritesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 1c48935dd9 HDDS-7035. Generate strToSign before applying virtual host
style filter. (#5123)
1c48935dd9 is described below
commit 1c48935dd9da72b45e32d3cccee4a2286174b0a8
Author: SaketaChalamchala <[email protected]>
AuthorDate: Thu Sep 21 09:53:46 2023 -0700
HDDS-7035. Generate strToSign before applying virtual host style filter.
(#5123)
---
.../dist/src/main/compose/common/ec-test.sh | 3 +-
.../dist/src/main/compose/ozone-ha/test.sh | 5 +-
.../main/compose/ozone/disabled-test-s3-haproxy.sh | 3 +-
.../s3g-virtual-host.yaml} | 42 ++-
.../test-s3g-virtual-host.sh} | 13 +-
.../dist/src/main/compose/ozonesecure-ha/test.sh | 6 +-
.../src/main/compose/ozonesecure/test-vault.sh | 3 +-
.../src/main/smoketest/s3/awss3virtualhost.robot | 56 ++++
.../dist/src/main/smoketest/s3/commonawslib.robot | 3 +
...lientProducer.java => AuthorizationFilter.java} | 107 +++----
.../org/apache/hadoop/ozone/s3/ClientIpFilter.java | 3 +-
.../apache/hadoop/ozone/s3/HeaderPreprocessor.java | 6 +-
.../hadoop/ozone/s3/OzoneClientProducer.java | 64 ----
.../hadoop/ozone/s3/VirtualHostStyleFilter.java | 15 +-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 8 +
.../ozone/s3/signature/AWSSignatureProcessor.java | 2 +
.../hadoop/ozone/s3/signature/SignatureInfo.java | 57 ++++
.../ozone/s3/signature/StringToSignProducer.java | 3 +-
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 1 +
.../hadoop/ozone/s3/TestAuthorizationFilter.java | 333 +++++++++++++++++++++
.../hadoop/ozone/s3/TestOzoneClientProducer.java | 160 +---------
.../ozone/s3/TestVirtualHostStyleFilter.java | 42 ++-
.../signature/TestAuthorizationV4QueryParser.java | 4 +-
.../s3/signature/TestStringToSignProducer.java | 4 +-
24 files changed, 615 insertions(+), 328 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/compose/common/ec-test.sh
b/hadoop-ozone/dist/src/main/compose/common/ec-test.sh
index cfc46f058a..65a659563e 100755
--- a/hadoop-ozone/dist/src/main/compose/common/ec-test.sh
+++ b/hadoop-ozone/dist/src/main/compose/common/ec-test.sh
@@ -17,7 +17,8 @@
start_docker_env 5
-execute_robot_test scm -v BUCKET:erasure s3
+## Exclude virtual-host tests. This is tested separately as it requires
additional config.
+execute_robot_test scm -v BUCKET:erasure --exclude virtual-host s3
prefix=${RANDOM}
execute_robot_test scm -v PREFIX:${prefix} ec/basic.robot
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh
b/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh
index 2d73133fdd..08c8522696 100755
--- a/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh
@@ -35,11 +35,12 @@ execute_robot_test ${SCM} basic/links.robot
execute_robot_test ${SCM} -v SCHEME:ofs -v BUCKET_TYPE:link -N
ozonefs-ofs-link ozonefs/ozonefs.robot
-exclude=""
+## Exclude virtual-host tests. This is tested separately as it requires
additional config.
+exclude="--exclude virtual-host"
for bucket in generated; do
execute_robot_test ${SCM} -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3
# some tests are independent of the bucket type, only need to be run once
- exclude="--exclude no-bucket-type"
+ exclude="--exclude virtual-host --exclude no-bucket-type"
done
execute_robot_test ${SCM} freon
diff --git
a/hadoop-ozone/dist/src/main/compose/ozone/disabled-test-s3-haproxy.sh
b/hadoop-ozone/dist/src/main/compose/ozone/disabled-test-s3-haproxy.sh
index 9612b92da4..6cf3901b9d 100755
--- a/hadoop-ozone/dist/src/main/compose/ozone/disabled-test-s3-haproxy.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone/disabled-test-s3-haproxy.sh
@@ -26,4 +26,5 @@ source "$COMPOSE_DIR/../testlib.sh"
start_docker_env
-execute_robot_test scm s3
+## Exclude virtual-host tests. This is tested separately as it requires
additional config.
+execute_robot_test scm --exclude virtual-host s3
diff --git a/hadoop-ozone/dist/src/main/compose/common/ec-test.sh
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/s3g-virtual-host.yaml
old mode 100755
new mode 100644
similarity index 53%
copy from hadoop-ozone/dist/src/main/compose/common/ec-test.sh
copy to hadoop-ozone/dist/src/main/compose/ozonesecure-ha/s3g-virtual-host.yaml
index cfc46f058a..708231ad20
--- a/hadoop-ozone/dist/src/main/compose/common/ec-test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/s3g-virtual-host.yaml
@@ -1,4 +1,3 @@
-#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,15 +14,34 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-start_docker_env 5
+version: "3.8"
-execute_robot_test scm -v BUCKET:erasure s3
-
-prefix=${RANDOM}
-execute_robot_test scm -v PREFIX:${prefix} ec/basic.robot
-docker-compose up -d --no-recreate --scale datanode=4
-execute_robot_test scm -v PREFIX:${prefix} -N read-4-datanodes ec/read.robot
-docker-compose up -d --no-recreate --scale datanode=3
-execute_robot_test scm -v PREFIX:${prefix} -N read-3-datanodes ec/read.robot
-docker-compose up -d --no-recreate --scale datanode=5
-execute_robot_test scm -v container:1 -v count:5 -N EC-recovery
replication/wait.robot
+x-s3g-virtual-host-config:
+ &s3g-virtual-host-config
+ environment:
+ - OZONE-SITE.XML_ozone.s3g.domain.name=s3g.internal
+services:
+ datanode1:
+ <<: *s3g-virtual-host-config
+ datanode2:
+ <<: *s3g-virtual-host-config
+ datanode3:
+ <<: *s3g-virtual-host-config
+ om1:
+ <<: *s3g-virtual-host-config
+ om2:
+ <<: *s3g-virtual-host-config
+ om3:
+ <<: *s3g-virtual-host-config
+ scm1.org:
+ <<: *s3g-virtual-host-config
+ scm2.org:
+ <<: *s3g-virtual-host-config
+ scm3.org:
+ <<: *s3g-virtual-host-config
+ s3g:
+ <<: *s3g-virtual-host-config
+ extra_hosts:
+ - "bucket1.s3g.internal: 172.25.0.114"
+ recon:
+ <<: *s3g-virtual-host-config
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/test-vault.sh
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-s3g-virtual-host.sh
similarity index 81%
copy from hadoop-ozone/dist/src/main/compose/ozonesecure/test-vault.sh
copy to
hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-s3g-virtual-host.sh
index f0af40792d..2bc3125801 100755
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/test-vault.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-s3g-virtual-host.sh
@@ -15,17 +15,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-#suite:secure
+#suite:HA-secure
COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
export COMPOSE_DIR
+export SECURITY_ENABLED=true
+export OM_SERVICE_ID="omservice"
+export SCM=scm1.org
+export COMPOSE_FILE=docker-compose.yaml:s3g-virtual-host.yaml
+
# shellcheck source=/dev/null
source "$COMPOSE_DIR/../testlib.sh"
-export SECURITY_ENABLED=true
-export COMPOSE_FILE=docker-compose.yaml:vault.yaml
-
start_docker_env
-execute_robot_test scm s3
+## Run virtual host test cases
+execute_robot_test s3g -N s3-virtual-host s3/awss3virtualhost.robot
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh
index 70d45b33c6..f6ab69746c 100755
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh
@@ -41,11 +41,13 @@ execute_robot_test s3g -v SCHEME:o3fs -v BUCKET_TYPE:link
-N ozonefs-o3fs-link o
execute_robot_test s3g basic/links.robot
-exclude=""
+## Exclude virtual-host tests. This is tested separately as it requires
additional config.
+exclude="--exclude virtual-host"
for bucket in encrypted link; do
execute_robot_test s3g -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3
# some tests are independent of the bucket type, only need to be run once
- exclude="--exclude no-bucket-type"
+ ## Exclude virtual-host.robot
+ exclude="--exclude virtual-host --exclude no-bucket-type"
done
execute_robot_test s3g admincli
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/test-vault.sh
b/hadoop-ozone/dist/src/main/compose/ozonesecure/test-vault.sh
index f0af40792d..f2bc0948fe 100755
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/test-vault.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/test-vault.sh
@@ -28,4 +28,5 @@ export COMPOSE_FILE=docker-compose.yaml:vault.yaml
start_docker_env
-execute_robot_test scm s3
+## Exclude virtual-host tests. This is tested separately as it requires
additional config.
+execute_robot_test scm --exclude virtual-host s3
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/awss3virtualhost.robot
b/hadoop-ozone/dist/src/main/smoketest/s3/awss3virtualhost.robot
new file mode 100644
index 0000000000..8f77e2c3e8
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/awss3virtualhost.robot
@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation S3 gateway test with aws cli using virtual host style
address
+Library OperatingSystem
+Library String
+Resource ../commonlib.robot
+Resource ./commonawslib.robot
+Test Timeout 5 minutes
+Suite Setup Setup s3 tests
+Default Tags virtual-host
+
+*** Variables ***
+${ENDPOINT_URL} http://s3g.internal:9878
+${BUCKET} bucket1
+${OZONE_S3_ADDRESS_STYLE} virtual
+
+*** Test Cases ***
+
+File upload and directory list with virtual style addressing
+ Create bucket with name ${BUCKET}
+ Execute date > /tmp/testfile
+ ${result} = Execute AWSS3Cli cp /tmp/testfile
s3://${BUCKET} --debug
+ Should contain ${result}
url=http://bucket1.s3g.internal:9878/
+ Should contain ${result} upload
+ ${result} = Execute AWSS3Cli cp /tmp/testfile
s3://${BUCKET}/dir1/dir2/file --debug
+ Should contain ${result}
url=http://bucket1.s3g.internal:9878/dir1/dir2/file
+ Should contain ${result} upload
+ ${result} = Execute AWSS3Cli ls s3://${BUCKET} --debug
+ Should contain ${result}
url=http://bucket1.s3g.internal:9878/
+ Should contain ${result} testfile
+ Should contain ${result} dir1
+ Should not contain ${result} dir2
+ ${result} = Execute AWSS3Cli ls s3://${BUCKET}/dir1/
--debug
+ Should contain ${result}
url=http://bucket1.s3g.internal:9878/
+ Should contain ${result} prefix=dir1
+ Should not contain ${result} testfile
+ Should contain ${result} dir2/
+ ${result} = Execute AWSS3Cli ls
s3://${BUCKET}/dir1/dir2/file
+ Should not contain ${result} testfile
+ Should not contain ${result} dir1
+ Should not contain ${result} dir2
+ Should contain ${result} file
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
index 0a6a42f005..98691876e1 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
@@ -26,6 +26,7 @@ ${BUCKET} generated
${KEY_NAME} key1
${OZONE_S3_TESTS_SET_UP} ${FALSE}
${OZONE_AWS_ACCESS_KEY_ID} ${EMPTY}
+${OZONE_S3_ADDRESS_STYLE} path
*** Keywords ***
Execute AWSS3APICli
@@ -85,6 +86,8 @@ Setup secure v4 headers
Execute aws configure set
aws_access_key_id ${accessKey}
Execute aws configure set
aws_secret_access_key ${secret}
Execute aws configure set region
us-west-1
+ Execute aws configure set
default.s3.addressing_style ${OZONE_S3_ADDRESS_STYLE}
+
Setup dummy credentials for S3
Execute aws configure set
default.s3.signature_version s3v4
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/AuthorizationFilter.java
similarity index 58%
copy from
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java
copy to
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/AuthorizationFilter.java
index bfab4559a3..d49ff17f3b 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/AuthorizationFilter.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -17,20 +17,16 @@
*/
package org.apache.hadoop.ozone.s3;
-import javax.annotation.PreDestroy;
-import javax.enterprise.context.RequestScoped;
-import javax.enterprise.inject.Produces;
+import javax.annotation.Priority;
import javax.inject.Inject;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.container.ContainerRequestContext;
-import javax.ws.rs.core.Context;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.container.PreMatching;
import javax.ws.rs.core.Response;
-import java.io.IOException;
+import javax.ws.rs.ext.Provider;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.om.protocol.S3Auth;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
import org.apache.hadoop.ozone.s3.signature.SignatureInfo;
@@ -40,53 +36,50 @@ import
org.apache.hadoop.ozone.s3.signature.StringToSignProducer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.ACCESS_DENIED;
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INTERNAL_ERROR;
import static
org.apache.hadoop.ozone.s3.exception.S3ErrorTable.S3_AUTHINFO_CREATION_ERROR;
/**
- * This class creates the OzoneClient for the Rest endpoints.
+ * Filter used to construct string to sign from unfiltered request.
+ * It should be executed before all other filters as the original
+ * could be enriched later.
*/
-@RequestScoped
-public class OzoneClientProducer {
- private static final Logger LOG =
- LoggerFactory.getLogger(OzoneClientProducer.class);
+@Provider
+@PreMatching
+@Priority(AuthorizationFilter.PRIORITY)
+public class AuthorizationFilter implements ContainerRequestFilter {
+ public static final int PRIORITY = 50;
- private OzoneClient client;
+ private static final Logger LOG = LoggerFactory.getLogger(
+ AuthorizationFilter.class);
@Inject
private SignatureProcessor signatureProcessor;
@Inject
- private OzoneConfiguration ozoneConfiguration;
-
- @Context
- private ContainerRequestContext context;
+ private SignatureInfo signatureInfo;
- @Produces
- public synchronized OzoneClient createClient() throws
WebApplicationException,
+ @Override
+ public void filter(ContainerRequestContext context) throws
IOException {
- ozoneConfiguration.set("ozone.om.group.rights", "NONE");
- client = getClient(ozoneConfiguration);
- return client;
- }
+ // Skip authentication if the uri is hitting S3Secret generation or
+ // revocation endpoint.
+ if (context.getUriInfo().getRequestUri().getPath().startsWith("/secret")) {
+ return;
+ }
- @PreDestroy
- public void destroy() throws IOException {
- client.getObjectStore().getClientProxy().clearThreadLocalS3Auth();
- }
- @Produces
- public S3Auth getSignature() {
try {
- SignatureInfo signatureInfo = signatureProcessor.parseSignature();
- String stringToSign = "";
+ signatureInfo.initialize(signatureProcessor.parseSignature());
if (signatureInfo.getVersion() == Version.V4) {
- stringToSign =
- StringToSignProducer.createSignatureBase(signatureInfo, context);
+ signatureInfo.setStrToSign(
+ StringToSignProducer.createSignatureBase(signatureInfo, context));
} else {
LOG.debug("Unsupported AWS signature version: {}",
- signatureInfo.getVersion());
+ signatureInfo.getVersion());
throw S3_AUTHINFO_CREATION_ERROR;
}
@@ -96,12 +89,6 @@ public class OzoneClientProducer {
LOG.debug("Malformed s3 header. awsAccessID: {}", awsAccessId);
throw ACCESS_DENIED;
}
-
- // Note: userPrincipal is initialized to be the same value as accessId,
- // could be updated later in RpcClient#getS3Volume
- return new S3Auth(stringToSign,
- signatureInfo.getSignature(),
- awsAccessId, awsAccessId);
} catch (OS3Exception ex) {
LOG.debug("Error during Client Creation: ", ex);
throw wrapOS3Exception(ex);
@@ -109,34 +96,24 @@ public class OzoneClientProducer {
// For any other critical errors during object creation throw Internal
// error.
LOG.debug("Error during Client Creation: ", e);
- throw wrapOS3Exception(S3ErrorTable.newError(INTERNAL_ERROR, null, e));
+ throw wrapOS3Exception(
+ S3ErrorTable.newError(INTERNAL_ERROR, null, e));
}
}
- private OzoneClient getClient(OzoneConfiguration config)
- throws IOException {
- OzoneClient ozoneClient = null;
- try {
- ozoneClient =
- OzoneClientCache.getOzoneClientInstance(ozoneConfiguration);
- } catch (Exception e) {
- // For any other critical errors during object creation throw Internal
- // error.
- if (LOG.isDebugEnabled()) {
- LOG.debug("Error during Client Creation: ", e);
- }
- throw e;
- }
- return ozoneClient;
+ @VisibleForTesting
+ public void setSignatureParser(SignatureProcessor awsSignatureProcessor) {
+ this.signatureProcessor = awsSignatureProcessor;
}
- public synchronized void setOzoneConfiguration(OzoneConfiguration config) {
- this.ozoneConfiguration = config;
+ @VisibleForTesting
+ public void setSignatureInfo(SignatureInfo signatureInfo) {
+ this.signatureInfo = signatureInfo;
}
@VisibleForTesting
- public void setSignatureParser(SignatureProcessor awsSignatureProcessor) {
- this.signatureProcessor = awsSignatureProcessor;
+ public SignatureInfo getSignatureInfo() {
+ return signatureInfo;
}
private WebApplicationException wrapOS3Exception(OS3Exception os3Exception) {
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/ClientIpFilter.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/ClientIpFilter.java
index 921b18d9b5..858ebbcd0b 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/ClientIpFilter.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/ClientIpFilter.java
@@ -38,7 +38,8 @@ import java.io.IOException;
@Priority(ClientIpFilter.PRIORITY)
public class ClientIpFilter implements ContainerRequestFilter {
- public static final int PRIORITY = 200;
+ public static final int PRIORITY = HeaderPreprocessor.PRIORITY +
+ S3GatewayHttpServer.FILTER_PRIORITY_DO_AFTER;
public static final String CLIENT_IP_HEADER = "client_ip";
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/HeaderPreprocessor.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/HeaderPreprocessor.java
index 344fd90971..c9deb22649 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/HeaderPreprocessor.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/HeaderPreprocessor.java
@@ -34,10 +34,12 @@ import java.io.IOException;
*/
@Provider
@PreMatching
-@Priority(VirtualHostStyleFilter.PRIORITY
- + S3GatewayHttpServer.FILTER_PRIORITY_DO_AFTER)
+@Priority(HeaderPreprocessor.PRIORITY)
public class HeaderPreprocessor implements ContainerRequestFilter {
+ public static final int PRIORITY = VirtualHostStyleFilter.PRIORITY +
+ S3GatewayHttpServer.FILTER_PRIORITY_DO_AFTER;
+
public static final String MULTIPART_UPLOAD_MARKER = "ozone/mpu";
public static final String CONTENT_TYPE = "Content-Type";
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java
index bfab4559a3..50e1f80e9c 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java
@@ -24,26 +24,13 @@ import javax.inject.Inject;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.container.ContainerRequestContext;
import javax.ws.rs.core.Context;
-import javax.ws.rs.core.Response;
import java.io.IOException;
-import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.om.protocol.S3Auth;
-import org.apache.hadoop.ozone.s3.exception.OS3Exception;
-import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
-import org.apache.hadoop.ozone.s3.signature.SignatureInfo;
-import org.apache.hadoop.ozone.s3.signature.SignatureInfo.Version;
-import org.apache.hadoop.ozone.s3.signature.SignatureProcessor;
-import org.apache.hadoop.ozone.s3.signature.StringToSignProducer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.ACCESS_DENIED;
-import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INTERNAL_ERROR;
-import static
org.apache.hadoop.ozone.s3.exception.S3ErrorTable.S3_AUTHINFO_CREATION_ERROR;
-
/**
* This class creates the OzoneClient for the Rest endpoints.
*/
@@ -55,9 +42,6 @@ public class OzoneClientProducer {
private OzoneClient client;
- @Inject
- private SignatureProcessor signatureProcessor;
-
@Inject
private OzoneConfiguration ozoneConfiguration;
@@ -76,42 +60,6 @@ public class OzoneClientProducer {
public void destroy() throws IOException {
client.getObjectStore().getClientProxy().clearThreadLocalS3Auth();
}
- @Produces
- public S3Auth getSignature() {
- try {
- SignatureInfo signatureInfo = signatureProcessor.parseSignature();
- String stringToSign = "";
- if (signatureInfo.getVersion() == Version.V4) {
- stringToSign =
- StringToSignProducer.createSignatureBase(signatureInfo, context);
- } else {
- LOG.debug("Unsupported AWS signature version: {}",
- signatureInfo.getVersion());
- throw S3_AUTHINFO_CREATION_ERROR;
- }
-
- String awsAccessId = signatureInfo.getAwsAccessId();
- // ONLY validate aws access id when needed.
- if (awsAccessId == null || awsAccessId.equals("")) {
- LOG.debug("Malformed s3 header. awsAccessID: {}", awsAccessId);
- throw ACCESS_DENIED;
- }
-
- // Note: userPrincipal is initialized to be the same value as accessId,
- // could be updated later in RpcClient#getS3Volume
- return new S3Auth(stringToSign,
- signatureInfo.getSignature(),
- awsAccessId, awsAccessId);
- } catch (OS3Exception ex) {
- LOG.debug("Error during Client Creation: ", ex);
- throw wrapOS3Exception(ex);
- } catch (Exception e) {
- // For any other critical errors during object creation throw Internal
- // error.
- LOG.debug("Error during Client Creation: ", e);
- throw wrapOS3Exception(S3ErrorTable.newError(INTERNAL_ERROR, null, e));
- }
- }
private OzoneClient getClient(OzoneConfiguration config)
throws IOException {
@@ -133,16 +81,4 @@ public class OzoneClientProducer {
public synchronized void setOzoneConfiguration(OzoneConfiguration config) {
this.ozoneConfiguration = config;
}
-
- @VisibleForTesting
- public void setSignatureParser(SignatureProcessor awsSignatureProcessor) {
- this.signatureProcessor = awsSignatureProcessor;
- }
-
- private WebApplicationException wrapOS3Exception(OS3Exception os3Exception) {
- return new WebApplicationException(os3Exception.getErrorMessage(),
- os3Exception,
- Response.status(os3Exception.getHttpCode())
- .entity(os3Exception.toXml()).build());
- }
}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/VirtualHostStyleFilter.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/VirtualHostStyleFilter.java
index a257155e76..32c1eb9eb2 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/VirtualHostStyleFilter.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/VirtualHostStyleFilter.java
@@ -49,7 +49,8 @@ import static
org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_DOMAIN_NA
@Priority(VirtualHostStyleFilter.PRIORITY)
public class VirtualHostStyleFilter implements ContainerRequestFilter {
- public static final int PRIORITY = 100;
+ public static final int PRIORITY = AuthorizationFilter.PRIORITY +
+ S3GatewayHttpServer.FILTER_PRIORITY_DO_AFTER;
private static final Logger LOG = LoggerFactory.getLogger(
VirtualHostStyleFilter.class);
@@ -62,6 +63,12 @@ public class VirtualHostStyleFilter implements
ContainerRequestFilter {
@Override
public void filter(ContainerRequestContext requestContext) throws
IOException {
+ // Skip this filter if the uri is hitting S3Secret generation or
+ // revocation endpoint.
+ if (requestContext.getUriInfo().getRequestUri().getPath()
+ .startsWith("/secret")) {
+ return;
+ }
domains = conf.getTrimmedStrings(OZONE_S3G_DOMAIN_NAME);
@@ -103,12 +110,12 @@ public class VirtualHostStyleFilter implements
ContainerRequestFilter {
URI baseURI = requestContext.getUriInfo().getBaseUri();
String currentPath = requestContext.getUriInfo().getPath();
String newPath = bucketName;
- if (currentPath != null) {
- newPath += String.format("%s", currentPath);
- }
MultivaluedMap<String, String> queryParams = requestContext.getUriInfo()
.getQueryParameters();
UriBuilder requestAddrBuilder =
UriBuilder.fromUri(baseURI).path(newPath);
+ if (currentPath != null) {
+ requestAddrBuilder.path(currentPath);
+ }
queryParams.forEach((k, v) -> requestAddrBuilder.queryParam(k,
v.toArray()));
URI requestAddr = requestAddrBuilder.build();
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
index 66fb7df190..97027abd7c 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
@@ -57,6 +57,7 @@ import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.ozone.s3.metrics.S3GatewayMetrics;
+import org.apache.hadoop.ozone.s3.signature.SignatureInfo;
import org.apache.hadoop.ozone.s3.util.AuditUtils;
import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
@@ -75,6 +76,8 @@ public abstract class EndpointBase implements Auditor {
@Inject
private OzoneClient client;
@Inject
+ private SignatureInfo signatureInfo;
+
private S3Auth s3Auth;
@Context
private ContainerRequestContext context;
@@ -114,6 +117,11 @@ public abstract class EndpointBase implements Auditor {
*/
@PostConstruct
public void initialization() {
+ // Note: userPrincipal is initialized to be the same value as accessId,
+ // could be updated later in RpcClient#getS3Volume
+ s3Auth = new S3Auth(signatureInfo.getStringToSign(),
+ signatureInfo.getSignature(),
+ signatureInfo.getAwsAccessId(), signatureInfo.getAwsAccessId());
LOG.debug("S3 access id: {}", s3Auth.getAccessID());
getClient().getObjectStore()
.getClientProxy()
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java
index 1ae8682186..43a1e6b713 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java
@@ -95,6 +95,8 @@ public class AWSSignatureProcessor implements
SignatureProcessor {
"", "", "", "", "", "", "", false
);
}
+ signatureInfo.setUnfilteredURI(
+ context.getUriInfo().getRequestUri().getPath());
return signatureInfo;
}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureInfo.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureInfo.java
index 0da8895921..d1db38a50a 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureInfo.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureInfo.java
@@ -17,11 +17,14 @@
*/
package org.apache.hadoop.ozone.s3.signature;
+import javax.enterprise.context.RequestScoped;
+
/**
* Signature and related information.
* <p>
* Required to create stringToSign and token.
*/
+@RequestScoped
public class SignatureInfo {
private Version version;
@@ -48,6 +51,13 @@ public class SignatureInfo {
private boolean signPayload = true;
+ private String unfilteredURI = null;
+
+
+ private String stringToSign = null;
+
+ public SignatureInfo() { }
+
@SuppressWarnings("checkstyle:ParameterNumber")
public SignatureInfo(
Version version,
@@ -59,6 +69,35 @@ public class SignatureInfo {
String credentialScope,
String algorithm,
boolean signPayload
+ ) {
+ initialize(version, date, dateTime, awsAccessId, signature, signedHeaders,
+ credentialScope, algorithm, signPayload, null, null);
+ }
+
+ public void initialize(
+ SignatureInfo signatureInfo
+ ) {
+ initialize(signatureInfo.getVersion(), signatureInfo.getDate(),
+ signatureInfo.getDateTime(), signatureInfo.getAwsAccessId(),
+ signatureInfo.getSignature(), signatureInfo.getSignedHeaders(),
+ signatureInfo.getCredentialScope(), signatureInfo.getAlgorithm(),
+ signatureInfo.isSignPayload(), signatureInfo.getUnfilteredURI(),
+ signatureInfo.getStringToSign());
+ }
+
+ @SuppressWarnings({"checkstyle:ParameterNumber", "checkstyle:HiddenField"})
+ public void initialize(
+ Version version,
+ String date,
+ String dateTime,
+ String awsAccessId,
+ String signature,
+ String signedHeaders,
+ String credentialScope,
+ String algorithm,
+ boolean signPayload,
+ String uri,
+ String stringToSign
) {
this.version = version;
this.date = date;
@@ -69,6 +108,8 @@ public class SignatureInfo {
this.credentialScope = credentialScope;
this.algorithm = algorithm;
this.signPayload = signPayload;
+ this.unfilteredURI = uri;
+ this.stringToSign = stringToSign;
}
public String getAwsAccessId() {
@@ -107,6 +148,22 @@ public class SignatureInfo {
return dateTime;
}
+ public String getUnfilteredURI() {
+ return unfilteredURI;
+ }
+
+ public String getStringToSign() {
+ return stringToSign;
+ }
+
+ public void setUnfilteredURI(String uri) {
+ this.unfilteredURI = uri;
+ }
+
+ public void setStrToSign(String strToSign) {
+ this.stringToSign = strToSign;
+ }
+
/**
* Signature version.
*/
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java
index b3fec69ba1..a8a4f6072d 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java
@@ -81,7 +81,6 @@ public final class StringToSignProducer {
return createSignatureBase(signatureInfo,
context.getUriInfo().getRequestUri().getScheme(),
context.getMethod(),
- context.getUriInfo().getRequestUri().getPath(),
LowerCaseKeyStringMap.fromHeaderMap(context.getHeaders()),
fromMultiValueToSingleValueMap(
context.getUriInfo().getQueryParameters()));
@@ -92,7 +91,6 @@ public final class StringToSignProducer {
SignatureInfo signatureInfo,
String scheme,
String method,
- String uri,
LowerCaseKeyStringMap headers,
Map<String, String> queryParams
) throws Exception {
@@ -111,6 +109,7 @@ public final class StringToSignProducer {
String credentialScope = signatureInfo.getCredentialScope();
// If the absolute path is empty, use a forward slash (/)
+ String uri = signatureInfo.getUnfilteredURI();
uri = (uri.trim().length() > 0) ? uri : "/";
// Encode URI and preserve forward slashes
strToSign.append(signatureInfo.getAlgorithm() + NEWLINE);
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
index 05e9503225..df3d01936b 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
@@ -64,6 +64,7 @@ public final class S3Consts {
public static final String CUSTOM_METADATA_HEADER_PREFIX = "x-amz-meta-";
+
public static final String DECODED_CONTENT_LENGTH_HEADER =
"x-amz-decoded-content-length";
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestAuthorizationFilter.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestAuthorizationFilter.java
new file mode 100644
index 0000000000..3bf3dcdd7b
--- /dev/null
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestAuthorizationFilter.java
@@ -0,0 +1,333 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.s3;
+
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.core.MultivaluedHashMap;
+import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.core.UriInfo;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.util.stream.Stream;
+
+import org.apache.hadoop.ozone.s3.signature.AWSSignatureProcessor;
+import org.apache.hadoop.ozone.s3.signature.SignatureInfo;
+import org.apache.hadoop.ozone.s3.signature.StringToSignProducer;
+import org.apache.kerby.util.Hex;
+
+import static java.net.HttpURLConnection.HTTP_BAD_REQUEST;
+import static java.net.HttpURLConnection.HTTP_FORBIDDEN;
+
+import static
org.apache.hadoop.ozone.s3.signature.AWSSignatureProcessor.DATE_FORMATTER;
+import static
org.apache.hadoop.ozone.s3.exception.S3ErrorTable.MALFORMED_HEADER;
+import static
org.apache.hadoop.ozone.s3.exception.S3ErrorTable.S3_AUTHINFO_CREATION_ERROR;
+import static
org.apache.hadoop.ozone.s3.signature.SignatureParser.AUTHORIZATION_HEADER;
+import static
org.apache.hadoop.ozone.s3.signature.SignatureProcessor.CONTENT_MD5;
+import static
org.apache.hadoop.ozone.s3.signature.SignatureProcessor.CONTENT_TYPE;
+import static
org.apache.hadoop.ozone.s3.signature.SignatureProcessor.HOST_HEADER;
+import static
org.apache.hadoop.ozone.s3.signature.StringToSignProducer.X_AMAZ_DATE;
+import static
org.apache.hadoop.ozone.s3.signature.StringToSignProducer.X_AMZ_CONTENT_SHA256;
+import static org.junit.Assert.fail;
+import static org.junit.jupiter.params.provider.Arguments.arguments;
+
+import org.junit.Assert;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.mockito.Mockito;
+
+/**
+ * This class test string to sign generation.
+ */
+public class TestAuthorizationFilter {
+
+ private AuthorizationFilter authorizationFilter = new AuthorizationFilter();
+
+ private MultivaluedMap<String, String> headerMap;
+ private MultivaluedMap<String, String> queryMap;
+ private MultivaluedMap<String, String> pathParamsMap;
+
+ private static final String DATETIME = StringToSignProducer.TIME_FORMATTER.
+ format(LocalDateTime.now());
+
+ private static final String CURDATE = DATE_FORMATTER.format(LocalDate.now());
+
+ private static Stream<Arguments>testAuthFilterFailuresInput() {
+ return Stream.of(
+ arguments(
+ "GET",
+ "AWS4-HMAC-SHA256 Credential=testuser1/20190221/us-west-1/s3" +
+ "/aws4_request, SignedHeaders=content-md5;host;" +
+ "x-amz-content-sha256;x-amz-date, " +
+ "Signature" +
+ "=56ec73ba1974f8feda8365c3caef89c5d4a688d5f9baccf47" +
+ "65f46a14cd745ad",
+ "Zi68x2nPDDXv5qfDC+ZWTg==",
+ "s3g:9878",
+ "e2bd43f11c97cde3465e0e8d1aad77af7ec7aa2ed8e213cd0e24" +
+ "1e28375860c6",
+ "20190221T002037Z",
+ "",
+ "/",
+ MALFORMED_HEADER.getErrorMessage()
+ ),
+ arguments(
+ "GET",
+ "AWS4-HMAC-SHA256 " +
+ "Credential=AKIDEXAMPLE/20150830/us-east-1/iam/aws4_request," +
+ " SignedHeaders=content-type;host;x-amz-date, " +
+ "Signature=" +
+ "5d672d79c15b13162d9279b0855cfba6789a8edb4c82c400" +
+ "e06b5924a6f2b5d7",
+ "",
+ "iam.amazonaws.com",
+ "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "20150830T123600Z",
+ "application/x-www-form-urlencoded; charset=utf-8",
+ "",
+ MALFORMED_HEADER.getErrorMessage()
+ ),
+ arguments(null, null, null, null, null, null, null, null,
+ S3_AUTHINFO_CREATION_ERROR.getErrorMessage()),
+ arguments(null, "", null, null, null, null, null, null,
+ S3_AUTHINFO_CREATION_ERROR.getErrorMessage()),
+ // AWS V2 signature
+ arguments(
+ "GET",
+ "AWS AKIDEXAMPLE:St7bHPOdkmsX/GITGe98rOQiUCg=",
+ "",
+ "s3g:9878",
+ "",
+ "Wed, 22 Mar 2023 17:00:06 +0000",
+ "application/octet-stream",
+ "/",
+ S3_AUTHINFO_CREATION_ERROR.getErrorMessage()
+ )
+ );
+ }
+
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ @ParameterizedTest
+ @MethodSource("testAuthFilterFailuresInput")
+ public void testAuthFilterFailures(
+ String method, String authHeader, String contentMd5,
+ String host, String amzContentSha256, String date, String contentType,
+ String path, String expectedErrorMsg
+ ) {
+ try {
+ ContainerRequestContext context = setupContext(method, authHeader,
+ contentMd5, host, amzContentSha256, date, contentType, path);
+
+ AWSSignatureProcessor awsSignatureProcessor = new
AWSSignatureProcessor();
+ awsSignatureProcessor.setContext(context);
+
+ SignatureInfo signatureInfo = new SignatureInfo();
+
+ authorizationFilter.setSignatureParser(awsSignatureProcessor);
+ authorizationFilter.setSignatureInfo(signatureInfo);
+
+ authorizationFilter.filter(context);
+ if ("".equals(authHeader)) {
+ fail("Empty AuthHeader must fail");
+ }
+ } catch (WebApplicationException ex) {
+ if (authHeader == null || authHeader.isEmpty() ||
+ authHeader.startsWith("AWS ")) {
+ // Empty auth header and unsupported AWS signature
+ // should fail with Invalid Request.
+ Assert.assertEquals(HTTP_FORBIDDEN, ex.getResponse().getStatus());
+ Assert.assertEquals(expectedErrorMsg,
+ ex.getMessage());
+ } else {
+ // Other requests have stale timestamp and
+ // should fail with Malformed Authorization Header.
+ Assert.assertEquals(HTTP_BAD_REQUEST, ex.getResponse().getStatus());
+ Assert.assertEquals(expectedErrorMsg,
+ ex.getMessage());
+
+ }
+
+ } catch (Exception ex) {
+ fail("Unexpected exception: " + ex);
+ }
+ }
+
+ private static Stream<Arguments>testAuthFilterInput() {
+ return Stream.of(
+ // Path style URI
+ arguments(
+ "GET",
+ "AWS4-HMAC-SHA256 Credential=testuser1/" + CURDATE +
+ "/us-east-1/s3/aws4_request, " +
+ "SignedHeaders=host;x-amz-content-sha256;" +
+ "x-amz-date, " +
+ "Signature" +
+ "=56ec73ba1974f8feda8365c3caef89c5d4a688d5f9baccf47" +
+ "65f46a14cd745ad",
+ "Content-SHA",
+ "s3g:9878",
+ "Content-SHA",
+ DATETIME,
+ "",
+ "/bucket1/key1"
+ ),
+ // Virtual style URI
+ arguments(
+ "GET",
+ "AWS4-HMAC-SHA256 Credential=testuser1/" + CURDATE +
+ "/us-east-1/s3/aws4_request, " +
+ "SignedHeaders=host;x-amz-content-sha256;" +
+ "x-amz-date, " +
+ "Signature" +
+ "=56ec73ba1974f8feda8365c3caef89c5d4a688d5f9baccf47" +
+ "65f46a14cd745ad",
+ "Content-SHA",
+ "bucket1.s3g.internal:9878",
+ "Content-SHA",
+ DATETIME,
+ "",
+ "/key1"
+ ),
+ // S3 secret generation endpoint
+ arguments(
+ "POST",
+ null,
+ null,
+ "s3g:9878",
+ null,
+ null,
+ "",
+ "/secret/generate"
+ ),
+ // S3 secret generation endpoint
+ arguments(
+ "POST",
+ null,
+ null,
+ "s3g:9878",
+ null,
+ null,
+ "",
+ "/secret/revoke"
+ )
+ );
+ }
+
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ @ParameterizedTest
+ @MethodSource("testAuthFilterInput")
+ public void testAuthFilter(
+ String method, String authHeader, String contentMd5,
+ String host, String amzContentSha256, String date, String contentType,
+ String path
+ ) {
+ try {
+ ContainerRequestContext context = setupContext(method, authHeader,
+ contentMd5, host, amzContentSha256, date, contentType, path);
+
+ AWSSignatureProcessor awsSignatureProcessor = new
AWSSignatureProcessor();
+ awsSignatureProcessor.setContext(context);
+
+ SignatureInfo signatureInfo = new SignatureInfo();
+
+ authorizationFilter.setSignatureParser(awsSignatureProcessor);
+ authorizationFilter.setSignatureInfo(signatureInfo);
+
+ authorizationFilter.filter(context);
+
+ if (path.startsWith("/secret")) {
+ Assert.assertNull(
+ authorizationFilter.getSignatureInfo().getUnfilteredURI());
+
+ Assert.assertNull(
+ authorizationFilter.getSignatureInfo().getStringToSign());
+ } else {
+ String canonicalRequest = method + "\n"
+ + path + "\n"
+ + "\n"
+ + "host:" + host + "\nx-amz-content-sha256:" + amzContentSha256 +
+ "\n"
+ + "x-amz-date:" + DATETIME + "\n"
+ + "\n"
+ + "host;x-amz-content-sha256;x-amz-date\n"
+ + amzContentSha256;
+
+ MessageDigest md = MessageDigest.getInstance("SHA-256");
+ md.update(canonicalRequest.getBytes(StandardCharsets.UTF_8));
+
+ String expectedStrToSign = "AWS4-HMAC-SHA256\n"
+ + DATETIME + "\n"
+ + CURDATE + "/us-east-1/s3/aws4_request\n"
+ + Hex.encode(md.digest()).toLowerCase();
+
+ Assert.assertEquals("Unfiltered URI is not preserved",
+ path,
+ authorizationFilter.getSignatureInfo().getUnfilteredURI());
+
+ Assert.assertEquals("String to sign is invalid",
+ expectedStrToSign,
+ authorizationFilter.getSignatureInfo().getStringToSign());
+ }
+ } catch (Exception ex) {
+ fail("Unexpected exception: " + ex);
+ }
+ }
+
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ private ContainerRequestContext setupContext(
+ String method, String authHeader, String contentMd5,
+ String host, String amzContentSha256, String date, String contentType,
+ String path) throws URISyntaxException {
+ headerMap = new MultivaluedHashMap<>();
+ queryMap = new MultivaluedHashMap<>();
+ pathParamsMap = new MultivaluedHashMap<>();
+
+ System.err.println("Testing: " + authHeader);
+ headerMap.putSingle(AUTHORIZATION_HEADER, authHeader);
+ headerMap.putSingle(CONTENT_MD5, contentMd5);
+ headerMap.putSingle(HOST_HEADER, host);
+ headerMap.putSingle(X_AMZ_CONTENT_SHA256, amzContentSha256);
+ headerMap.putSingle(X_AMAZ_DATE, date);
+ headerMap.putSingle(CONTENT_TYPE, contentType);
+
+ UriInfo uriInfo = Mockito.mock(UriInfo.class);
+ ContainerRequestContext context = Mockito.mock(
+ ContainerRequestContext.class);
+ Mockito.when(uriInfo.getQueryParameters()).thenReturn(queryMap);
+ Mockito.when(uriInfo.getRequestUri()).thenReturn(
+ new URI("http://"; + host + path));
+
+ Mockito.when(context.getMethod()).thenReturn(method);
+ Mockito.when(context.getUriInfo()).thenReturn(uriInfo);
+ Mockito.when(context.getHeaders()).thenReturn(headerMap);
+ Mockito.when(context.getHeaderString(AUTHORIZATION_HEADER))
+ .thenReturn(authHeader);
+ Mockito.when(context.getUriInfo().getQueryParameters())
+ .thenReturn(queryMap);
+ Mockito.when(context.getUriInfo().getPathParameters())
+ .thenReturn(pathParamsMap);
+
+ return context;
+ }
+
+}
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestOzoneClientProducer.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestOzoneClientProducer.java
index 064d9836f6..ed7a8be944 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestOzoneClientProducer.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestOzoneClientProducer.java
@@ -17,131 +17,33 @@
*/
package org.apache.hadoop.ozone.s3;
-import javax.ws.rs.WebApplicationException;
-import javax.ws.rs.container.ContainerRequestContext;
-import javax.ws.rs.core.MultivaluedHashMap;
-import javax.ws.rs.core.MultivaluedMap;
-import javax.ws.rs.core.UriInfo;
import java.io.IOException;
-import java.net.URI;
-import java.util.Arrays;
-import java.util.Collection;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.om.OMConfigKeys;
-import org.apache.hadoop.ozone.s3.signature.AWSSignatureProcessor;
-import static java.net.HttpURLConnection.HTTP_BAD_REQUEST;
-import static java.net.HttpURLConnection.HTTP_FORBIDDEN;
-
-import static
org.apache.hadoop.ozone.s3.exception.S3ErrorTable.MALFORMED_HEADER;
-import static
org.apache.hadoop.ozone.s3.exception.S3ErrorTable.S3_AUTHINFO_CREATION_ERROR;
-import static
org.apache.hadoop.ozone.s3.signature.SignatureParser.AUTHORIZATION_HEADER;
-import static
org.apache.hadoop.ozone.s3.signature.SignatureProcessor.CONTENT_MD5;
-import static
org.apache.hadoop.ozone.s3.signature.SignatureProcessor.CONTENT_TYPE;
-import static
org.apache.hadoop.ozone.s3.signature.SignatureProcessor.HOST_HEADER;
-import static
org.apache.hadoop.ozone.s3.signature.StringToSignProducer.X_AMAZ_DATE;
-import static
org.apache.hadoop.ozone.s3.signature.StringToSignProducer.X_AMZ_CONTENT_SHA256;
import static org.junit.Assert.fail;
import org.junit.Assert;
import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.mockito.Mockito;
/**
* Test class for @{@link OzoneClientProducer}.
*/
-@RunWith(Parameterized.class)
public class TestOzoneClientProducer {
private OzoneClientProducer producer;
- private MultivaluedMap<String, String> headerMap;
- private MultivaluedMap<String, String> queryMap;
- private MultivaluedMap<String, String> pathParamsMap;
- private String authHeader;
- private String contentMd5;
- private String host;
- private String amzContentSha256;
- private String date;
- private String contentType;
- private ContainerRequestContext context;
- private UriInfo uriInfo;
- public TestOzoneClientProducer(
- String authHeader, String contentMd5,
- String host, String amzContentSha256, String date, String contentType
- )
+ public TestOzoneClientProducer()
throws Exception {
- this.authHeader = authHeader;
- this.contentMd5 = contentMd5;
- this.host = host;
- this.amzContentSha256 = amzContentSha256;
- this.date = date;
- this.contentType = contentType;
producer = new OzoneClientProducer();
- headerMap = new MultivaluedHashMap<>();
- queryMap = new MultivaluedHashMap<>();
- pathParamsMap = new MultivaluedHashMap<>();
- uriInfo = Mockito.mock(UriInfo.class);
- context = Mockito.mock(ContainerRequestContext.class);
OzoneConfiguration config = new OzoneConfiguration();
config.setBoolean(OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY, true);
config.set(OMConfigKeys.OZONE_OM_ADDRESS_KEY, "");
- setupContext();
producer.setOzoneConfiguration(config);
}
- @Parameterized.Parameters
- public static Collection<Object[]> data() {
- return Arrays.asList(new Object[][] {
- {
- "AWS4-HMAC-SHA256 Credential=testuser1/20190221/us-west-1/s3" +
- "/aws4_request, SignedHeaders=content-md5;host;" +
- "x-amz-content-sha256;x-amz-date, " +
- "Signature" +
- "=56ec73ba1974f8feda8365c3caef89c5d4a688d5f9baccf47" +
- "65f46a14cd745ad",
- "Zi68x2nPDDXv5qfDC+ZWTg==",
- "s3g:9878",
- "e2bd43f11c97cde3465e0e8d1aad77af7ec7aa2ed8e213cd0e24" +
- "1e28375860c6",
- "20190221T002037Z",
- ""
- },
- {
- "AWS4-HMAC-SHA256 " +
- "Credential=AKIDEXAMPLE/20150830/us-east-1/iam/aws4_request," +
- " SignedHeaders=content-type;host;x-amz-date, " +
- "Signature=" +
- "5d672d79c15b13162d9279b0855cfba6789a8edb4c82c400" +
- "e06b5924a6f2b5d7",
- "",
- "iam.amazonaws.com",
- "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
- "20150830T123600Z",
- "application/x-www-form-urlencoded; charset=utf-8"
- },
- {
- null, null, null, null, null, null
- },
- {
- "", null, null, null, null, null
- },
- // AWS V2 signature
- {
- "AWS AKIDEXAMPLE:St7bHPOdkmsX/GITGe98rOQiUCg=",
- "",
- "s3g:9878",
- "",
- "Wed, 22 Mar 2023 17:00:06 +0000",
- "application/octet-stream"
- }
- });
- }
-
@Test
public void testGetClientFailure() {
try {
@@ -152,40 +54,6 @@ public class TestOzoneClientProducer {
}
}
- @Test
- public void testGetSignature() {
- try {
- System.err.println("Testing: " + authHeader);
- OzoneConfiguration configuration = new OzoneConfiguration();
- configuration.set(OMConfigKeys.OZONE_OM_SERVICE_IDS_KEY, "ozone1");
- configuration.set(OMConfigKeys.OZONE_OM_ADDRESS_KEY, "ozone1addr:9399");
- producer.setOzoneConfiguration(configuration);
- producer.getSignature();
- if ("".equals(authHeader)) {
- fail("Empty AuthHeader must fail");
- }
- } catch (WebApplicationException ex) {
- if (authHeader == null || authHeader.isEmpty() ||
- authHeader.startsWith("AWS ")) {
- // Empty auth header and unsupported AWS signature
- // should fail with Invalid Request.
- Assert.assertEquals(HTTP_FORBIDDEN, ex.getResponse().getStatus());
- Assert.assertEquals(S3_AUTHINFO_CREATION_ERROR.getErrorMessage(),
- ex.getMessage());
- } else {
- // Other requests have stale timestamp and
- // should fail with Malformed Authorization Header.
- Assert.assertEquals(HTTP_BAD_REQUEST, ex.getResponse().getStatus());
- Assert.assertEquals(MALFORMED_HEADER.getErrorMessage(),
- ex.getMessage());
-
- }
-
- } catch (Exception ex) {
- fail("Unexpected exception: " + ex);
- }
- }
-
@Test
public void testGetClientFailureWithMultipleServiceIds() {
try {
@@ -219,30 +87,4 @@ public class TestOzoneClientProducer {
}
}
- private void setupContext() throws Exception {
- headerMap.putSingle(AUTHORIZATION_HEADER, authHeader);
- headerMap.putSingle(CONTENT_MD5, contentMd5);
- headerMap.putSingle(HOST_HEADER, host);
- headerMap.putSingle(X_AMZ_CONTENT_SHA256, amzContentSha256);
- headerMap.putSingle(X_AMAZ_DATE, date);
- headerMap.putSingle(CONTENT_TYPE, contentType);
-
- Mockito.when(uriInfo.getQueryParameters()).thenReturn(queryMap);
- Mockito.when(uriInfo.getRequestUri()).thenReturn(new URI(""));
-
- Mockito.when(context.getUriInfo()).thenReturn(uriInfo);
- Mockito.when(context.getHeaders()).thenReturn(headerMap);
- Mockito.when(context.getHeaderString(AUTHORIZATION_HEADER))
- .thenReturn(authHeader);
- Mockito.when(context.getUriInfo().getQueryParameters())
- .thenReturn(queryMap);
- Mockito.when(context.getUriInfo().getPathParameters())
- .thenReturn(pathParamsMap);
-
- AWSSignatureProcessor awsSignatureProcessor = new AWSSignatureProcessor();
- awsSignatureProcessor.setContext(context);
-
- producer.setSignatureParser(awsSignatureProcessor);
- }
-
}
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestVirtualHostStyleFilter.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestVirtualHostStyleFilter.java
index 19d9380de9..81b8e0c503 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestVirtualHostStyleFilter.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestVirtualHostStyleFilter.java
@@ -105,8 +105,7 @@ public class TestVirtualHostStyleFilter {
ContainerRequest containerRequest = createContainerRequest("mybucket" +
".localhost:9878", "/myfile", null, true);
virtualHostStyleFilter.filter(containerRequest);
- URI expected = new URI("http://"; + s3HttpAddr +
- "/mybucket/myfile");
+ URI expected = new URI("http://"; + s3HttpAddr + "/mybucket/myfile");
Assert.assertEquals(expected, containerRequest.getRequestUri());
}
@@ -126,6 +125,29 @@ public class TestVirtualHostStyleFilter {
}
+ @Test
+ public void testS3SecretEndpoint() throws Exception {
+
+ VirtualHostStyleFilter virtualHostStyleFilter =
+ new VirtualHostStyleFilter();
+ virtualHostStyleFilter.setConfiguration(conf);
+
+ ContainerRequest containerRequest = createContainerRequest("mybucket" +
+ ".localhost:9878", "/secret/generate",
+ null, true);
+ virtualHostStyleFilter.filter(containerRequest);
+ URI expected = new URI("http://"; + s3HttpAddr + "/secret/generate");
+ Assert.assertEquals(expected, containerRequest.getRequestUri());
+
+ containerRequest = createContainerRequest("mybucket" +
+ ".localhost:9878", "/secret/revoke",
+ null, true);
+ virtualHostStyleFilter.filter(containerRequest);
+ expected = new URI("http://"; + s3HttpAddr + "/secret/revoke");
+ Assert.assertEquals(expected, containerRequest.getRequestUri());
+
+ }
+
@Test
public void testVirtualHostStyleWithCreateBucketRequest() throws Exception {
@@ -141,17 +163,29 @@ public class TestVirtualHostStyleFilter {
}
+ @Test
+ public void testVirtualHostStyleWithCreateKeyRequest() throws Exception {
+ VirtualHostStyleFilter virtualHostStyleFilter =
+ new VirtualHostStyleFilter();
+ virtualHostStyleFilter.setConfiguration(conf);
+
+ ContainerRequest containerRequest = createContainerRequest("mybucket" +
+ ".localhost:9878", "/key1", null, true);
+ virtualHostStyleFilter.filter(containerRequest);
+ URI expected = new URI("http://"; + s3HttpAddr + "/mybucket/key1");
+ Assert.assertEquals(expected, containerRequest.getRequestUri());
+ }
+
@Test
public void testVirtualHostStyleWithQueryParams() throws Exception {
VirtualHostStyleFilter virtualHostStyleFilter =
new VirtualHostStyleFilter();
virtualHostStyleFilter.setConfiguration(conf);
-
+ URI expected = new URI("http://"; + s3HttpAddr + "/mybucket?prefix=bh");
ContainerRequest containerRequest = createContainerRequest("mybucket" +
".localhost:9878", null, "?prefix=bh", true);
virtualHostStyleFilter.filter(containerRequest);
- URI expected = new URI("http://"; + s3HttpAddr + "/mybucket?prefix=bh");
assertTrue(expected.toString().contains(containerRequest.getRequestUri()
.toString()));
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationV4QueryParser.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationV4QueryParser.java
index 7221dfbb02..181e962e5c 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationV4QueryParser.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationV4QueryParser.java
@@ -284,14 +284,14 @@ public class TestAuthorizationV4QueryParser {
};
final SignatureInfo signatureInfo = parser.parseSignature();
+ signatureInfo.setUnfilteredURI("/test.txt");
LowerCaseKeyStringMap headers = new LowerCaseKeyStringMap();
headers.put("host", "localhost");
final String stringToSign =
StringToSignProducer.createSignatureBase(signatureInfo, "https", "GET",
- "/test.txt", headers,
- queryParams);
+ headers, queryParams);
MessageDigest md = MessageDigest.getInstance("SHA-256");
md.update(canonicalRequest.getBytes(StandardCharsets.UTF_8));
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestStringToSignProducer.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestStringToSignProducer.java
index 9d52172179..14aa649476 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestStringToSignProducer.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestStringToSignProducer.java
@@ -95,6 +95,7 @@ public class TestStringToSignProducer {
//NOOP
}
}.parseSignature();
+ signatureInfo.setUnfilteredURI("/buckets");
headers.fixContentType();
@@ -103,7 +104,6 @@ public class TestStringToSignProducer {
signatureInfo,
"http",
"GET",
- "/buckets",
headers,
queryParameters);
@@ -203,6 +203,7 @@ public class TestStringToSignProducer {
SignatureInfo signatureInfo = new AuthorizationV4HeaderParser(
headerMap.getFirst("Authorization"),
headerMap.getFirst("X-Amz-Date")).parseSignature();
+ signatureInfo.setUnfilteredURI("/");
try {
StringToSignProducer.createSignatureBase(signatureInfo, context);
} catch (OS3Exception e) {
@@ -260,6 +261,7 @@ public class TestStringToSignProducer {
SignatureInfo signatureInfo = new AuthorizationV4HeaderParser(
headerMap.getFirst("Authorization"),
headerMap.getFirst("x-amz-date")).parseSignature();
+ signatureInfo.setUnfilteredURI("/");
try {
StringToSignProducer.createSignatureBase(signatureInfo, context);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
