hadoop git commit: HDFS-13176. WebHdfs file path gets truncated when having semicolon inside. Contributed by Zsolt Venczel.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 689a8252a -> b412958f7
HDFS-13176. WebHdfs file path gets truncated when having semicolon inside.
Contributed by Zsolt Venczel.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b412958f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b412958f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b412958f
Branch: refs/heads/branch-2
Commit: b412958f706bf8cb7917a8ff98be8e7c7c172d09
Parents: 689a825
Author: Sean Mackrory
Authored: Thu Apr 5 08:50:55 2018 -0600
Committer: Sean Mackrory
Committed: Fri Apr 6 07:41:58 2018 -0600
--
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 27 +-
.../datanode/web/webhdfs/WebHdfsHandler.java| 3 +-
.../web/resources/NamenodeWebHdfsMethods.java | 7 ++-
.../apache/hadoop/hdfs/web/TestWebHdfsUrl.java | 57
4 files changed, 90 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b412958f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index 3ec680e..0eeaa61 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -37,6 +37,8 @@ import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
+import java.net.URLDecoder;
+import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
@@ -604,8 +606,31 @@ public class WebHdfsFileSystem extends FileSystem
URL toUrl(final HttpOpParam.Op op, final Path fspath,
final Param... parameters) throws IOException {
//initialize URI path and query
+Path encodedFSPath = fspath;
+if (fspath != null) {
+ URI fspathUri = fspath.toUri();
+ String fspathUriDecoded = fspathUri.getPath();
+ try {
+fspathUriDecoded = URLDecoder.decode(fspathUri.getPath(), "UTF-8");
+ } catch (IllegalArgumentException ex) {
+LOG.trace("Cannot decode URL encoded file", ex);
+ }
+ String[] fspathItems = fspathUriDecoded.split("/");
+
+ if (fspathItems.length > 0) {
+StringBuilder fsPathEncodedItems = new StringBuilder();
+for (String fsPathItem : fspathItems) {
+ fsPathEncodedItems.append("/");
+ fsPathEncodedItems.append(URLEncoder.encode(fsPathItem, "UTF-8"));
+}
+encodedFSPath = new Path(fspathUri.getScheme(),
+fspathUri.getAuthority(), fsPathEncodedItems.substring(1));
+ }
+}
+
final String path = PATH_PREFIX
-+ (fspath == null? "/": makeQualified(fspath).toUri().getRawPath());
++ (encodedFSPath == null ? "/" :
+makeQualified(encodedFSPath).toUri().getRawPath());
final String query = op.toQueryString()
+ Param.toSortedString("&", getAuthParameters(op))
+ Param.toSortedString("&", parameters);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b412958f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
index f20e565..2eb03a3 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
@@ -57,6 +57,7 @@ import java.io.OutputStream;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
+import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.security.PrivilegedExceptionAction;
import java.util.EnumSet;
@@ -126,7 +127,7 @@ public class WebHdfsHandler extends
SimpleChannelInboundHandler {
params = new ParameterParser(queryString, conf);
DataNodeUGIProvider ugiProvider = new DataNodeUGIProvider(params);
ugi = ugiProvider.ugi();
-path = params.path();
+path = URLDecoder.decode(params.path(), "UTF-8");
[01/50] hadoop git commit: HDFS-13176. WebHdfs file path gets truncated when having semicolon (; ) inside. Contributed by Zsolt Venczel.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-12996 85a0ed79b -> 8d4aa014d
HDFS-13176. WebHdfs file path gets truncated when having semicolon (;) inside.
Contributed by Zsolt Venczel.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/46d29e3d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/46d29e3d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/46d29e3d
Branch: refs/heads/HDFS-12996
Commit: 46d29e3d7ee8dc9bb1818b886d9cc5336b1d67a4
Parents: 037d783
Author: Sean Mackrory
Authored: Wed Mar 7 12:11:52 2018 -0700
Committer: Sean Mackrory
Committed: Wed Mar 7 13:33:41 2018 -0700
--
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 28 +-
.../datanode/web/webhdfs/WebHdfsHandler.java| 3 +-
.../web/resources/NamenodeWebHdfsMethods.java | 5 +-
.../apache/hadoop/hdfs/web/TestWebHdfsUrl.java | 57
4 files changed, 90 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d29e3d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index 45260f3..1c919c4 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -37,6 +37,8 @@ import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
+import java.net.URLDecoder;
+import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
@@ -598,8 +600,32 @@ public class WebHdfsFileSystem extends FileSystem
URL toUrl(final HttpOpParam.Op op, final Path fspath,
final Param... parameters) throws IOException {
//initialize URI path and query
+
+Path encodedFSPath = fspath;
+if (fspath != null) {
+ URI fspathUri = fspath.toUri();
+ String fspathUriDecoded = fspathUri.getPath();
+ try {
+fspathUriDecoded = URLDecoder.decode(fspathUri.getPath(), "UTF-8");
+ } catch (IllegalArgumentException ex) {
+LOG.trace("Cannot decode URL encoded file", ex);
+ }
+ String[] fspathItems = fspathUriDecoded.split("/");
+
+ if (fspathItems.length > 0) {
+StringBuilder fsPathEncodedItems = new StringBuilder();
+for (String fsPathItem : fspathItems) {
+ fsPathEncodedItems.append("/");
+ fsPathEncodedItems.append(URLEncoder.encode(fsPathItem, "UTF-8"));
+}
+encodedFSPath = new Path(fspathUri.getScheme(),
+fspathUri.getAuthority(), fsPathEncodedItems.substring(1));
+ }
+}
+
final String path = PATH_PREFIX
-+ (fspath == null? "/": makeQualified(fspath).toUri().getRawPath());
++ (encodedFSPath == null ? "/" :
+makeQualified(encodedFSPath).toUri().getRawPath());
final String query = op.toQueryString()
+ Param.toSortedString("&", getAuthParameters(op))
+ Param.toSortedString("&", parameters);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d29e3d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
index c5fc7ea..9a4b670 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
@@ -58,6 +58,7 @@ import java.io.OutputStream;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
+import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.security.PrivilegedExceptionAction;
import java.util.EnumSet;
@@ -127,7 +128,7 @@ public class WebHdfsHandler extends
SimpleChannelInboundHandler {
params = new ParameterParser(queryString, conf);
DataNodeUGIProvider ugiProvider = new DataNodeUGIProvider(params);
ugi = ugiProvider.ugi();
-path = params.path();
+path = URLDecoder.decode(params.path(), "
[34/35] hadoop git commit: HDFS-13176. WebHdfs file path gets truncated when having semicolon (; ) inside. Contributed by Zsolt Venczel.
HDFS-13176. WebHdfs file path gets truncated when having semicolon (;) inside.
Contributed by Zsolt Venczel.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/46d29e3d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/46d29e3d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/46d29e3d
Branch: refs/heads/HDFS-7240
Commit: 46d29e3d7ee8dc9bb1818b886d9cc5336b1d67a4
Parents: 037d783
Author: Sean Mackrory
Authored: Wed Mar 7 12:11:52 2018 -0700
Committer: Sean Mackrory
Committed: Wed Mar 7 13:33:41 2018 -0700
--
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 28 +-
.../datanode/web/webhdfs/WebHdfsHandler.java| 3 +-
.../web/resources/NamenodeWebHdfsMethods.java | 5 +-
.../apache/hadoop/hdfs/web/TestWebHdfsUrl.java | 57
4 files changed, 90 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d29e3d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index 45260f3..1c919c4 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -37,6 +37,8 @@ import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
+import java.net.URLDecoder;
+import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
@@ -598,8 +600,32 @@ public class WebHdfsFileSystem extends FileSystem
URL toUrl(final HttpOpParam.Op op, final Path fspath,
final Param... parameters) throws IOException {
//initialize URI path and query
+
+Path encodedFSPath = fspath;
+if (fspath != null) {
+ URI fspathUri = fspath.toUri();
+ String fspathUriDecoded = fspathUri.getPath();
+ try {
+fspathUriDecoded = URLDecoder.decode(fspathUri.getPath(), "UTF-8");
+ } catch (IllegalArgumentException ex) {
+LOG.trace("Cannot decode URL encoded file", ex);
+ }
+ String[] fspathItems = fspathUriDecoded.split("/");
+
+ if (fspathItems.length > 0) {
+StringBuilder fsPathEncodedItems = new StringBuilder();
+for (String fsPathItem : fspathItems) {
+ fsPathEncodedItems.append("/");
+ fsPathEncodedItems.append(URLEncoder.encode(fsPathItem, "UTF-8"));
+}
+encodedFSPath = new Path(fspathUri.getScheme(),
+fspathUri.getAuthority(), fsPathEncodedItems.substring(1));
+ }
+}
+
final String path = PATH_PREFIX
-+ (fspath == null? "/": makeQualified(fspath).toUri().getRawPath());
++ (encodedFSPath == null ? "/" :
+makeQualified(encodedFSPath).toUri().getRawPath());
final String query = op.toQueryString()
+ Param.toSortedString("&", getAuthParameters(op))
+ Param.toSortedString("&", parameters);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d29e3d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
index c5fc7ea..9a4b670 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
@@ -58,6 +58,7 @@ import java.io.OutputStream;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
+import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.security.PrivilegedExceptionAction;
import java.util.EnumSet;
@@ -127,7 +128,7 @@ public class WebHdfsHandler extends
SimpleChannelInboundHandler {
params = new ParameterParser(queryString, conf);
DataNodeUGIProvider ugiProvider = new DataNodeUGIProvider(params);
ugi = ugiProvider.ugi();
-path = params.path();
+path = URLDecoder.decode(params.path(), "UTF-8");
injectToken();
ugi.doAs(new PrivilegedExceptionAction() {
http://
hadoop git commit: HDFS-13176. WebHdfs file path gets truncated when having semicolon (; ) inside. Contributed by Zsolt Venczel.
Repository: hadoop
Updated Branches:
refs/heads/trunk 037d78348 -> 46d29e3d7
HDFS-13176. WebHdfs file path gets truncated when having semicolon (;) inside.
Contributed by Zsolt Venczel.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/46d29e3d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/46d29e3d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/46d29e3d
Branch: refs/heads/trunk
Commit: 46d29e3d7ee8dc9bb1818b886d9cc5336b1d67a4
Parents: 037d783
Author: Sean Mackrory
Authored: Wed Mar 7 12:11:52 2018 -0700
Committer: Sean Mackrory
Committed: Wed Mar 7 13:33:41 2018 -0700
--
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 28 +-
.../datanode/web/webhdfs/WebHdfsHandler.java| 3 +-
.../web/resources/NamenodeWebHdfsMethods.java | 5 +-
.../apache/hadoop/hdfs/web/TestWebHdfsUrl.java | 57
4 files changed, 90 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d29e3d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index 45260f3..1c919c4 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -37,6 +37,8 @@ import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
+import java.net.URLDecoder;
+import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
@@ -598,8 +600,32 @@ public class WebHdfsFileSystem extends FileSystem
URL toUrl(final HttpOpParam.Op op, final Path fspath,
final Param... parameters) throws IOException {
//initialize URI path and query
+
+Path encodedFSPath = fspath;
+if (fspath != null) {
+ URI fspathUri = fspath.toUri();
+ String fspathUriDecoded = fspathUri.getPath();
+ try {
+fspathUriDecoded = URLDecoder.decode(fspathUri.getPath(), "UTF-8");
+ } catch (IllegalArgumentException ex) {
+LOG.trace("Cannot decode URL encoded file", ex);
+ }
+ String[] fspathItems = fspathUriDecoded.split("/");
+
+ if (fspathItems.length > 0) {
+StringBuilder fsPathEncodedItems = new StringBuilder();
+for (String fsPathItem : fspathItems) {
+ fsPathEncodedItems.append("/");
+ fsPathEncodedItems.append(URLEncoder.encode(fsPathItem, "UTF-8"));
+}
+encodedFSPath = new Path(fspathUri.getScheme(),
+fspathUri.getAuthority(), fsPathEncodedItems.substring(1));
+ }
+}
+
final String path = PATH_PREFIX
-+ (fspath == null? "/": makeQualified(fspath).toUri().getRawPath());
++ (encodedFSPath == null ? "/" :
+makeQualified(encodedFSPath).toUri().getRawPath());
final String query = op.toQueryString()
+ Param.toSortedString("&", getAuthParameters(op))
+ Param.toSortedString("&", parameters);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d29e3d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
index c5fc7ea..9a4b670 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
@@ -58,6 +58,7 @@ import java.io.OutputStream;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
+import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.security.PrivilegedExceptionAction;
import java.util.EnumSet;
@@ -127,7 +128,7 @@ public class WebHdfsHandler extends
SimpleChannelInboundHandler {
params = new ParameterParser(queryString, conf);
DataNodeUGIProvider ugiProvider = new DataNodeUGIProvider(params);
ugi = ugiProvider.ugi();
-path = params.path();
+path = URLDecoder.decode(params.path(), "UTF-8");
