HADOOP-14114 S3A can no longer handle unencoded + in URIs. Contributed by Sean Mackrory.
(cherry picked from commit ff87ca84418a710c6dc884fe8c70947fcc6489d5) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9c22a916 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9c22a916 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9c22a916 Branch: refs/heads/YARN-5734 Commit: 9c22a91662af24569191ce45289ef8266e8755cc Parents: 132f758 Author: Steve Loughran <ste...@apache.org> Authored: Fri Feb 24 10:41:36 2017 +0000 Committer: Steve Loughran <ste...@apache.org> Committed: Fri Feb 24 10:41:36 2017 +0000 ---------------------------------------------------------------------- .../hadoop/fs/s3native/S3xLoginHelper.java | 15 ++++++++++- .../hadoop/fs/s3native/TestS3xLoginHelper.java | 28 ++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c22a916/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java index 97ece37..862ce6b 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java @@ -48,6 +48,13 @@ public final class S3xLoginHelper { "The Filesystem URI contains login details." +" This is insecure and may be unsupported in future."; + public static final String PLUS_WARNING = + "Secret key contains a special character that should be URL encoded! " + + "Attempting to resolve..."; + + public static final String PLUS_UNENCODED = "+"; + public static final String PLUS_ENCODED = "%2B"; + /** * Build the filesystem URI. This can include stripping down of part * of the URI. @@ -112,7 +119,13 @@ public final class S3xLoginHelper { int loginSplit = login.indexOf(':'); if (loginSplit > 0) { String user = login.substring(0, loginSplit); - String password = URLDecoder.decode(login.substring(loginSplit + 1), + String encodedPassword = login.substring(loginSplit + 1); + if (encodedPassword.contains(PLUS_UNENCODED)) { + LOG.warn(PLUS_WARNING); + encodedPassword = encodedPassword.replaceAll("\\" + PLUS_UNENCODED, + PLUS_ENCODED); + } + String password = URLDecoder.decode(encodedPassword, "UTF-8"); return new Login(user, password); } else if (loginSplit == 0) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c22a916/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestS3xLoginHelper.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestS3xLoginHelper.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestS3xLoginHelper.java index bd2ac1e..3761cb7 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestS3xLoginHelper.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestS3xLoginHelper.java @@ -32,9 +32,13 @@ public class TestS3xLoginHelper extends Assert { public static final String BUCKET = "s3a://bucket"; private static final URI ENDPOINT = uri(BUCKET); public static final String S = "%2f"; + public static final String P = "%2b"; + public static final String P_RAW = "+"; public static final String USER = "user"; public static final String PASS = "pass"; public static final String PASLASHSLASH = "pa" + S + S; + public static final String PAPLUS = "pa" + P; + public static final String PAPLUS_RAW = "pa" + P_RAW; public static final URI WITH_USER_AND_PASS = uri("s3a://user:pass@bucket"); public static final Path PATH_WITH_LOGIN = @@ -42,6 +46,10 @@ public class TestS3xLoginHelper extends Assert { public static final URI WITH_SLASH_IN_PASS = uri( "s3a://user:" + PASLASHSLASH + "@bucket"); + public static final URI WITH_PLUS_IN_PASS = uri( + "s3a://user:" + PAPLUS + "@bucket"); + public static final URI WITH_PLUS_RAW_IN_PASS = uri( + "s3a://user:" + PAPLUS_RAW + "@bucket"); public static final URI USER_NO_PASS = uri("s3a://user@bucket"); public static final URI WITH_USER_AND_COLON = uri("s3a://user:@bucket"); public static final URI NO_USER = uri("s3a://:pass@bucket"); @@ -117,6 +125,16 @@ public class TestS3xLoginHelper extends Assert { } @Test + public void testLoginWithPlusInPass() throws Throwable { + assertMatchesLogin(USER, "pa+", WITH_PLUS_IN_PASS); + } + + @Test + public void testLoginWithPlusRawInPass() throws Throwable { + assertMatchesLogin(USER, "pa+", WITH_PLUS_RAW_IN_PASS); + } + + @Test public void testLoginWithUser() throws Throwable { assertMatchesLogin(USER, "", USER_NO_PASS); } @@ -152,6 +170,16 @@ public class TestS3xLoginHelper extends Assert { } @Test + public void testFsUriWithPlusInPass() throws Throwable { + assertMatchesEndpoint(WITH_PLUS_IN_PASS); + } + + @Test + public void testFsUriWithPlusRawInPass() throws Throwable { + assertMatchesEndpoint(WITH_PLUS_RAW_IN_PASS); + } + + @Test public void testFsUriWithUser() throws Throwable { assertMatchesEndpoint(USER_NO_PASS); } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org