This is an automated email from the ASF dual-hosted git repository. stevel pushed a commit to branch branch-3.3.4 in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit cf2675f7218ef292c0fb89238f31d41cf47041a0 Author: Steve Loughran <[email protected]> AuthorDate: Tue Jan 18 10:31:28 2022 +0000 HADOOP-18068. upgrade AWS SDK to 1.12.132 (#3864) With this update, the versions of key shaded dependencies are jackson 2.12.3 httpclient 4.5.13 Contributed by Steve Loughran Change-Id: Id9ed677352d54e8ea71b9729b6a4bfedc6142825 --- NOTICE-binary | 2 +- hadoop-project/pom.xml | 2 +- .../src/site/markdown/tools/hadoop-aws/testing.md | 1 + .../org/apache/hadoop/fs/s3a/TestArnResource.java | 19 +++++++++++++++---- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/NOTICE-binary b/NOTICE-binary index 60a71c6ce3d..2189de34e37 100644 --- a/NOTICE-binary +++ b/NOTICE-binary @@ -66,7 +66,7 @@ available from http://www.digip.org/jansson/. AWS SDK for Java -Copyright 2010-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. +Copyright 2010-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. This product includes software developed by Amazon Technologies, Inc (http://www.amazon.com/). diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 0f42e7e231c..bbf242fa54a 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -187,7 +187,7 @@ <exec-maven-plugin.version>1.3.1</exec-maven-plugin.version> <make-maven-plugin.version>1.0-beta-1</make-maven-plugin.version> <surefire.fork.timeout>900</surefire.fork.timeout> - <aws-java-sdk.version>1.11.1026</aws-java-sdk.version> + <aws-java-sdk.version>1.12.132</aws-java-sdk.version> <hsqldb.version>2.3.4</hsqldb.version> <frontend-maven-plugin.version>1.11.2</frontend-maven-plugin.version> <jasmine-maven-plugin.version>2.1</jasmine-maven-plugin.version> diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md index f5da283db36..8dec802dc27 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md @@ -1662,6 +1662,7 @@ to AWS services. * Try and get other people, especially anyone with their own endpoints, apps or different deployment environments, to run their own tests. * Run the load tests, especially `ILoadTestS3ABulkDeleteThrottling`. +* Checkout cloudstore, build it against your version of hadoop, then use its CLI to run some commands (`storediag` etc) ### Dealing with Deprecated APIs and New Features diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java index 97069c7c8a1..5ac47752ec8 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java @@ -19,6 +19,7 @@ package org.apache.hadoop.fs.s3a; import com.amazonaws.regions.Regions; +import org.assertj.core.api.Assertions; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,6 +28,14 @@ import org.apache.hadoop.test.HadoopTestBase; import static org.apache.hadoop.test.LambdaTestUtils.intercept; +/** + * Verifies the mapping of ARN declaration of resource to the associated + * access point. + * The region mapping assertions have been brittle to changes across AWS SDK + * versions to only verify partial matches, rather than the FQDN of the + * endpoints. + * + */ public class TestArnResource extends HadoopTestBase { private final static Logger LOG = LoggerFactory.getLogger(TestArnResource.class); @@ -37,10 +46,10 @@ public class TestArnResource extends HadoopTestBase { String accessPoint = "testAp"; String accountId = "123456789101"; String[][] regionPartitionEndpoints = new String[][] { - {Regions.EU_WEST_1.getName(), "aws", "s3-accesspoint.eu-west-1.amazonaws.com"}, + {Regions.EU_WEST_1.getName(), "aws", "eu-west-1.amazonaws.com"}, {Regions.US_GOV_EAST_1.getName(), "aws-us-gov", - "s3-accesspoint.us-gov-east-1.amazonaws.com"}, - {Regions.CN_NORTH_1.getName(), "aws-cn", "s3-accesspoint.cn-north-1.amazonaws.com.cn"}, + "us-gov-east-1.amazonaws.com"}, + {Regions.CN_NORTH_1.getName(), "aws-cn", "cn-north-1.amazonaws.com"}, }; for (String[] testPair : regionPartitionEndpoints) { @@ -57,7 +66,9 @@ public class TestArnResource extends HadoopTestBase { assertEquals("Access Point name does not match", accessPoint, resource.getName()); assertEquals("Account Id does not match", accountId, resource.getOwnerAccountId()); assertEquals("Region does not match", region, resource.getRegion()); - assertEquals("Endpoint does not match", endpoint, resource.getEndpoint()); + Assertions.assertThat(resource.getEndpoint()) + .describedAs("Endpoint does not match") + .contains(endpoint); } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
