HADOOP-13050. Upgrade to AWS SDK 10.11+. Contributed by Chris Nauroth and Steve Loughran
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/af791b77 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/af791b77 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/af791b77 Branch: refs/heads/YARN-2915 Commit: af791b774bdc3af26ab99487ffea7246c6788a9c Parents: 3a09e59 Author: Mingliang Liu <[email protected]> Authored: Mon Nov 21 17:29:18 2016 -0800 Committer: Mingliang Liu <[email protected]> Committed: Tue Nov 22 14:00:35 2016 -0800 ---------------------------------------------------------------------- hadoop-project/pom.xml | 7 ++++++- hadoop-tools/hadoop-aws/pom.xml | 4 ++++ .../main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java | 6 ++++-- .../apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java | 3 ++- .../java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java | 5 +++-- .../apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java | 3 +++ 6 files changed, 22 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/af791b77/hadoop-project/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 95fe839..112fa18 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -126,7 +126,7 @@ <make-maven-plugin.version>1.0-beta-1</make-maven-plugin.version> <native-maven-plugin.version>1.0-alpha-8</native-maven-plugin.version> <surefire.fork.timeout>900</surefire.fork.timeout> - <aws-java-sdk.version>1.10.6</aws-java-sdk.version> + <aws-java-sdk.version>1.11.45</aws-java-sdk.version> </properties> <dependencyManagement> @@ -876,6 +876,11 @@ <version>${jackson2.version}</version> </dependency> <dependency> + <groupId>com.fasterxml.jackson.dataformat</groupId> + <artifactId>jackson-dataformat-cbor</artifactId> + <version>${jackson2.version}</version> + </dependency> + <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-all</artifactId> <version>1.8.5</version> http://git-wip-us.apache.org/repos/asf/hadoop/blob/af791b77/hadoop-tools/hadoop-aws/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index e273428..1407661 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -345,6 +345,10 @@ <artifactId>jackson-annotations</artifactId> </dependency> <dependency> + <groupId>com.fasterxml.jackson.dataformat</groupId> + <artifactId>jackson-dataformat-cbor</artifactId> + </dependency> + <dependency> <groupId>joda-time</groupId> <artifactId>joda-time</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/hadoop/blob/af791b77/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java index 0a4dd02..871322d 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java @@ -180,7 +180,7 @@ interface S3ClientFactory { userAgent = userAgentPrefix + ", " + userAgent; } LOG.debug("Using User-Agent: {}", userAgent); - awsConf.setUserAgent(userAgent); + awsConf.setUserAgentPrefix(userAgent); } /** @@ -225,7 +225,9 @@ interface S3ClientFactory { final boolean pathStyleAccess = conf.getBoolean(PATH_STYLE_ACCESS, false); if (pathStyleAccess) { LOG.debug("Enabling path style access!"); - s3.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true)); + s3.setS3ClientOptions(S3ClientOptions.builder() + .setPathStyleAccess(true) + .build()); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/af791b77/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java index 819d9d8..d7f3411 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java @@ -110,7 +110,8 @@ public class ITestS3AAWSCredentialsProvider { @SuppressWarnings("unused") public GoodCredentialsProvider(URI name, Configuration conf) { super(new BasicAWSCredentialsProvider(conf.get(ACCESS_KEY), - conf.get(SECRET_KEY)), new InstanceProfileCredentialsProvider()); + conf.get(SECRET_KEY)), + InstanceProfileCredentialsProvider.getInstance()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/af791b77/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java index 9163b15..f7d77f4 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java @@ -399,7 +399,8 @@ public class ITestS3AConfiguration { assertNotNull(s3); ClientConfiguration awsConf = getField(s3, ClientConfiguration.class, "clientConfiguration"); - assertEquals("Hadoop " + VersionInfo.getVersion(), awsConf.getUserAgent()); + assertEquals("Hadoop " + VersionInfo.getVersion(), + awsConf.getUserAgentPrefix()); } @Test @@ -413,7 +414,7 @@ public class ITestS3AConfiguration { ClientConfiguration awsConf = getField(s3, ClientConfiguration.class, "clientConfiguration"); assertEquals("MyApp, Hadoop " + VersionInfo.getVersion(), - awsConf.getUserAgent()); + awsConf.getUserAgentPrefix()); } @Test http://git-wip-us.apache.org/repos/asf/hadoop/blob/af791b77/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java index 8421dad..d8ae313 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java @@ -117,6 +117,9 @@ public class ITestS3AMiniYarnCluster extends AbstractS3ATestBase { Map<String, Integer> result = new HashMap<>(); for (String line : outputAsStr.split("\n")) { String[] tokens = line.split("\t"); + assertTrue("Not enough tokens in in string \" "+ line + + "\" from output \"" + outputAsStr + "\"", + tokens.length > 1); result.put(tokens[0], Integer.parseInt(tokens[1])); } return result; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
