hadoop git commit: HDFS-6874. Add GETFILEBLOCKLOCATIONS operation to HttpFS. Contributed by Weiwei Yang

2017-07-12 Thread szetszwo 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 401b191a3 -> b4a108fa9


HDFS-6874. Add GETFILEBLOCKLOCATIONS operation to HttpFS.  Contributed by 
Weiwei Yang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b4a108fa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b4a108fa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b4a108fa

Branch: refs/heads/branch-2
Commit: b4a108fa9f38ee028978474fe6c298bbd88fda7a
Parents: 401b191
Author: Tsz-Wo Nicholas Sze 
Authored: Wed Jul 12 13:45:41 2017 -0700
Committer: Tsz-Wo Nicholas Sze 
Committed: Wed Jul 12 14:18:22 2017 -0700

--
 .../hadoop/fs/http/client/HttpFSFileSystem.java | 42 +
 .../hadoop/fs/http/server/FSOperations.java | 37 
 .../http/server/HttpFSParametersProvider.java   |  3 +-
 .../hadoop/fs/http/server/HttpFSServer.java | 21 -
 .../fs/http/client/BaseTestHttpFSWith.java  | 89 +++-
 5 files changed, 188 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4a108fa/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
index 2db92e0..94e8619 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
@@ -23,9 +23,12 @@ import java.util.Collection;
 import java.util.EnumSet;
 import java.util.List;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.type.MapType;
 import com.google.common.base.Charsets;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.DelegationTokenRenewer;
 import org.apache.hadoop.fs.FSDataInputStream;
@@ -120,6 +123,8 @@ public class HttpFSFileSystem extends FileSystem
   public static final String NEW_LENGTH_PARAM = "newlength";
   public static final String START_AFTER_PARAM = "startAfter";
   public static final String POLICY_NAME_PARAM = "storagepolicy";
+  public static final String OFFSET_PARAM = "offset";
+  public static final String LENGTH_PARAM = "length";
 
   public static final Short DEFAULT_PERMISSION = 0755;
   public static final String ACLSPEC_DEFAULT = "";
@@ -201,6 +206,7 @@ public class HttpFSFileSystem extends FileSystem
 
   public static final String STORAGE_POLICIES_JSON = "BlockStoragePolicies";
   public static final String STORAGE_POLICY_JSON = "BlockStoragePolicy";
+  public static final String BLOCK_LOCATIONS_JSON = "BlockLocations";
 
   public static final int HTTP_TEMPORARY_REDIRECT = 307;
 
@@ -1357,6 +1363,42 @@ public class HttpFSFileSystem extends FileSystem
 return createStoragePolicy((JSONObject) json.get(STORAGE_POLICY_JSON));
   }
 
+  @Override
+  public BlockLocation[] getFileBlockLocations(FileStatus file, long start,
+  long len) throws IOException {
+Map params = new HashMap();
+params.put(OP_PARAM, Operation.GETFILEBLOCKLOCATIONS.toString());
+params.put(OFFSET_PARAM, Long.toString(start));
+params.put(LENGTH_PARAM, Long.toString(len));
+HttpURLConnection conn =
+getConnection(Operation.GETFILEBLOCKLOCATIONS.getMethod(), params,
+file.getPath(), true);
+HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
+JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
+return toBlockLocations(json);
+  }
+
+  private BlockLocation[] toBlockLocations(JSONObject json)
+  throws IOException {
+ObjectMapper mapper = new ObjectMapper();
+MapType subType = mapper.getTypeFactory().constructMapType(
+Map.class,
+String.class,
+BlockLocation[].class);
+MapType rootType = mapper.getTypeFactory().constructMapType(
+Map.class,
+mapper.constructType(String.class),
+mapper.constructType(subType));
+
+Map> jsonMap = mapper
+.readValue(json.toJSONString(), rootType);
+Map locationMap = jsonMap
+.get(BLOCK_LOCATIONS_JSON);
+BlockLocation[] locationArray = locationMap.get(
+BlockLocation.class.getSimpleName());
+return locationArray;
+  }

hadoop git commit: HDFS-6874. Add GETFILEBLOCKLOCATIONS operation to HttpFS. Contributed by Weiwei Yang

2017-07-12 Thread szetszwo 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 655110393 -> 931a49800


HDFS-6874. Add GETFILEBLOCKLOCATIONS operation to HttpFS.  Contributed by 
Weiwei Yang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/931a4980
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/931a4980
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/931a4980

Branch: refs/heads/trunk
Commit: 931a49800ef05ee0a6fdc143be1799abb228735d
Parents: 6551103
Author: Tsz-Wo Nicholas Sze 
Authored: Wed Jul 12 13:45:41 2017 -0700
Committer: Tsz-Wo Nicholas Sze 
Committed: Wed Jul 12 14:15:04 2017 -0700

--
 .../hadoop/fs/http/client/HttpFSFileSystem.java | 42 +
 .../hadoop/fs/http/server/FSOperations.java | 37 
 .../http/server/HttpFSParametersProvider.java   |  3 +-
 .../hadoop/fs/http/server/HttpFSServer.java | 21 -
 .../fs/http/client/BaseTestHttpFSWith.java  | 89 +++-
 5 files changed, 188 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/931a4980/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
index 5922958..1ab890f 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
@@ -23,9 +23,12 @@ import java.util.Collection;
 import java.util.EnumSet;
 import java.util.List;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.type.MapType;
 import com.google.common.base.Charsets;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.DelegationTokenRenewer;
 import org.apache.hadoop.fs.FSDataInputStream;
@@ -119,6 +122,8 @@ public class HttpFSFileSystem extends FileSystem
   public static final String NEW_LENGTH_PARAM = "newlength";
   public static final String START_AFTER_PARAM = "startAfter";
   public static final String POLICY_NAME_PARAM = "storagepolicy";
+  public static final String OFFSET_PARAM = "offset";
+  public static final String LENGTH_PARAM = "length";
 
   public static final Short DEFAULT_PERMISSION = 0755;
   public static final String ACLSPEC_DEFAULT = "";
@@ -201,6 +206,7 @@ public class HttpFSFileSystem extends FileSystem
 
   public static final String STORAGE_POLICIES_JSON = "BlockStoragePolicies";
   public static final String STORAGE_POLICY_JSON = "BlockStoragePolicy";
+  public static final String BLOCK_LOCATIONS_JSON = "BlockLocations";
 
   public static final int HTTP_TEMPORARY_REDIRECT = 307;
 
@@ -1358,6 +1364,42 @@ public class HttpFSFileSystem extends FileSystem
 return createStoragePolicy((JSONObject) json.get(STORAGE_POLICY_JSON));
   }
 
+  @Override
+  public BlockLocation[] getFileBlockLocations(FileStatus file, long start,
+  long len) throws IOException {
+Map params = new HashMap();
+params.put(OP_PARAM, Operation.GETFILEBLOCKLOCATIONS.toString());
+params.put(OFFSET_PARAM, Long.toString(start));
+params.put(LENGTH_PARAM, Long.toString(len));
+HttpURLConnection conn =
+getConnection(Operation.GETFILEBLOCKLOCATIONS.getMethod(), params,
+file.getPath(), true);
+HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
+JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
+return toBlockLocations(json);
+  }
+
+  private BlockLocation[] toBlockLocations(JSONObject json)
+  throws IOException {
+ObjectMapper mapper = new ObjectMapper();
+MapType subType = mapper.getTypeFactory().constructMapType(
+Map.class,
+String.class,
+BlockLocation[].class);
+MapType rootType = mapper.getTypeFactory().constructMapType(
+Map.class,
+mapper.constructType(String.class),
+mapper.constructType(subType));
+
+Map> jsonMap = mapper
+.readValue(json.toJSONString(), rootType);
+Map locationMap = jsonMap
+.get(BLOCK_LOCATIONS_JSON);
+BlockLocation[] locationArray = locationMap.get(
+BlockLocation.class.getSimpleName());
+return locationArray;
+  }
+