[14/40] hadoop git commit: HDFS-9705. Refine the behaviour of getFileChecksum when length = 0. Contributed by Kai Zheng and SammiChen.

[liuml07]

HDFS-9705. Refine the behaviour of getFileChecksum when length = 0. Contributed 
by Kai Zheng and SammiChen.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cc1292e7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cc1292e7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cc1292e7

Branch: refs/heads/HADOOP-13345
Commit: cc1292e73acd39c1f1023ad4841ffe30176f7daf
Parents: 4c66a8d
Author: Andrew Wang 
Authored: Tue Mar 14 16:41:10 2017 -0700
Committer: Andrew Wang 
Committed: Tue Mar 14 16:41:10 2017 -0700

--
 .../java/org/apache/hadoop/hdfs/DFSClient.java  | 10 +++--
 .../apache/hadoop/hdfs/FileChecksumHelper.java  | 42 
 .../server/datanode/BlockChecksumHelper.java|  6 +--
 .../hadoop/hdfs/TestDistributedFileSystem.java  | 22 +-
 4 files changed, 48 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc1292e7/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index ae1d821..aaf8bdd 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -1731,10 +1731,14 @@ public class DFSClient implements java.io.Closeable, 
RemotePeerFactory,
 checkOpen();
 Preconditions.checkArgument(length >= 0);
 
-LocatedBlocks blockLocations = getBlockLocations(src, length);
+LocatedBlocks blockLocations = null;
+FileChecksumHelper.FileChecksumComputer maker = null;
+ErasureCodingPolicy ecPolicy = null;
+if (length > 0) {
+  blockLocations = getBlockLocations(src, length);
+  ecPolicy = blockLocations.getErasureCodingPolicy();
+}
 
-FileChecksumHelper.FileChecksumComputer maker;
-ErasureCodingPolicy ecPolicy = blockLocations.getErasureCodingPolicy();
 maker = ecPolicy != null ?
 new FileChecksumHelper.StripedFileNonStripedChecksumComputer(src,
 length, blockLocations, namenode, this, ecPolicy) :

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc1292e7/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java
index fe462f2..689d46d 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java
@@ -95,11 +95,13 @@ final class FileChecksumHelper {
   this.client = client;
 
   this.remaining = length;
-  if (src.contains(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR_SEPARATOR)) {
-this.remaining = Math.min(length, blockLocations.getFileLength());
-  }
 
-  this.locatedBlocks = blockLocations.getLocatedBlocks();
+  if (blockLocations != null) {
+if (src.contains(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR_SEPARATOR)) {
+  this.remaining = Math.min(length, blockLocations.getFileLength());
+}
+this.locatedBlocks = blockLocations.getLocatedBlocks();
+  }
 }
 
 String getSrc() {
@@ -203,9 +205,23 @@ final class FileChecksumHelper {
  * @throws IOException
  */
 void compute() throws IOException {
-  checksumBlocks();
-
-  fileChecksum = makeFinalResult();
+  /**
+   * request length is 0 or the file is empty, return one with the
+   * magic entry that matches what previous hdfs versions return.
+   */
+  if (locatedBlocks == null || locatedBlocks.isEmpty()) {
+// Explicitly specified here in case the default DataOutputBuffer
+// buffer length value is changed in future. This matters because the
+// fixed value 32 has to be used to repeat the magic value for previous
+// HDFS version.
+final int lenOfZeroBytes = 32;
+byte[] emptyBlockMd5 = new byte[lenOfZeroBytes];
+MD5Hash fileMD5 = MD5Hash.digest(emptyBlockMd5);
+fileChecksum =  new MD5MD5CRC32GzipFileChecksum(0, 0, fileMD5);
+  } else {
+checksumBlocks();
+fileChecksum = makeFinalResult();
+  }
 }
 
 /**
@@ -228,15 +244,7 @@ final class FileChecksumHelper {
  

[35/50] [abbrv] hadoop git commit: HDFS-9705. Refine the behaviour of getFileChecksum when length = 0. Contributed by Kai Zheng and SammiChen.

[inigoiri]

HDFS-9705. Refine the behaviour of getFileChecksum when length = 0. Contributed 
by Kai Zheng and SammiChen.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cc1292e7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cc1292e7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cc1292e7

Branch: refs/heads/HDFS-10467
Commit: cc1292e73acd39c1f1023ad4841ffe30176f7daf
Parents: 4c66a8d
Author: Andrew Wang 
Authored: Tue Mar 14 16:41:10 2017 -0700
Committer: Andrew Wang 
Committed: Tue Mar 14 16:41:10 2017 -0700

--
 .../java/org/apache/hadoop/hdfs/DFSClient.java  | 10 +++--
 .../apache/hadoop/hdfs/FileChecksumHelper.java  | 42 
 .../server/datanode/BlockChecksumHelper.java|  6 +--
 .../hadoop/hdfs/TestDistributedFileSystem.java  | 22 +-
 4 files changed, 48 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc1292e7/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index ae1d821..aaf8bdd 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -1731,10 +1731,14 @@ public class DFSClient implements java.io.Closeable, 
RemotePeerFactory,
 checkOpen();
 Preconditions.checkArgument(length >= 0);
 
-LocatedBlocks blockLocations = getBlockLocations(src, length);
+LocatedBlocks blockLocations = null;
+FileChecksumHelper.FileChecksumComputer maker = null;
+ErasureCodingPolicy ecPolicy = null;
+if (length > 0) {
+  blockLocations = getBlockLocations(src, length);
+  ecPolicy = blockLocations.getErasureCodingPolicy();
+}
 
-FileChecksumHelper.FileChecksumComputer maker;
-ErasureCodingPolicy ecPolicy = blockLocations.getErasureCodingPolicy();
 maker = ecPolicy != null ?
 new FileChecksumHelper.StripedFileNonStripedChecksumComputer(src,
 length, blockLocations, namenode, this, ecPolicy) :

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc1292e7/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java
index fe462f2..689d46d 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java
@@ -95,11 +95,13 @@ final class FileChecksumHelper {
   this.client = client;
 
   this.remaining = length;
-  if (src.contains(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR_SEPARATOR)) {
-this.remaining = Math.min(length, blockLocations.getFileLength());
-  }
 
-  this.locatedBlocks = blockLocations.getLocatedBlocks();
+  if (blockLocations != null) {
+if (src.contains(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR_SEPARATOR)) {
+  this.remaining = Math.min(length, blockLocations.getFileLength());
+}
+this.locatedBlocks = blockLocations.getLocatedBlocks();
+  }
 }
 
 String getSrc() {
@@ -203,9 +205,23 @@ final class FileChecksumHelper {
  * @throws IOException
  */
 void compute() throws IOException {
-  checksumBlocks();
-
-  fileChecksum = makeFinalResult();
+  /**
+   * request length is 0 or the file is empty, return one with the
+   * magic entry that matches what previous hdfs versions return.
+   */
+  if (locatedBlocks == null || locatedBlocks.isEmpty()) {
+// Explicitly specified here in case the default DataOutputBuffer
+// buffer length value is changed in future. This matters because the
+// fixed value 32 has to be used to repeat the magic value for previous
+// HDFS version.
+final int lenOfZeroBytes = 32;
+byte[] emptyBlockMd5 = new byte[lenOfZeroBytes];
+MD5Hash fileMD5 = MD5Hash.digest(emptyBlockMd5);
+fileChecksum =  new MD5MD5CRC32GzipFileChecksum(0, 0, fileMD5);
+  } else {
+checksumBlocks();
+fileChecksum = makeFinalResult();
+  }
 }
 
 /**
@@ -228,15 +244,7 @@ final class FileChecksumHelper {

hadoop git commit: HDFS-9705. Refine the behaviour of getFileChecksum when length = 0. Contributed by Kai Zheng and SammiChen.

[wang]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 4c66a8d19 -> cc1292e73


HDFS-9705. Refine the behaviour of getFileChecksum when length = 0. Contributed 
by Kai Zheng and SammiChen.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cc1292e7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cc1292e7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cc1292e7

Branch: refs/heads/trunk
Commit: cc1292e73acd39c1f1023ad4841ffe30176f7daf
Parents: 4c66a8d
Author: Andrew Wang 
Authored: Tue Mar 14 16:41:10 2017 -0700
Committer: Andrew Wang 
Committed: Tue Mar 14 16:41:10 2017 -0700

--
 .../java/org/apache/hadoop/hdfs/DFSClient.java  | 10 +++--
 .../apache/hadoop/hdfs/FileChecksumHelper.java  | 42 
 .../server/datanode/BlockChecksumHelper.java|  6 +--
 .../hadoop/hdfs/TestDistributedFileSystem.java  | 22 +-
 4 files changed, 48 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc1292e7/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index ae1d821..aaf8bdd 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -1731,10 +1731,14 @@ public class DFSClient implements java.io.Closeable, 
RemotePeerFactory,
 checkOpen();
 Preconditions.checkArgument(length >= 0);
 
-LocatedBlocks blockLocations = getBlockLocations(src, length);
+LocatedBlocks blockLocations = null;
+FileChecksumHelper.FileChecksumComputer maker = null;
+ErasureCodingPolicy ecPolicy = null;
+if (length > 0) {
+  blockLocations = getBlockLocations(src, length);
+  ecPolicy = blockLocations.getErasureCodingPolicy();
+}
 
-FileChecksumHelper.FileChecksumComputer maker;
-ErasureCodingPolicy ecPolicy = blockLocations.getErasureCodingPolicy();
 maker = ecPolicy != null ?
 new FileChecksumHelper.StripedFileNonStripedChecksumComputer(src,
 length, blockLocations, namenode, this, ecPolicy) :

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc1292e7/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java
index fe462f2..689d46d 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java
@@ -95,11 +95,13 @@ final class FileChecksumHelper {
   this.client = client;
 
   this.remaining = length;
-  if (src.contains(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR_SEPARATOR)) {
-this.remaining = Math.min(length, blockLocations.getFileLength());
-  }
 
-  this.locatedBlocks = blockLocations.getLocatedBlocks();
+  if (blockLocations != null) {
+if (src.contains(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR_SEPARATOR)) {
+  this.remaining = Math.min(length, blockLocations.getFileLength());
+}
+this.locatedBlocks = blockLocations.getLocatedBlocks();
+  }
 }
 
 String getSrc() {
@@ -203,9 +205,23 @@ final class FileChecksumHelper {
  * @throws IOException
  */
 void compute() throws IOException {
-  checksumBlocks();
-
-  fileChecksum = makeFinalResult();
+  /**
+   * request length is 0 or the file is empty, return one with the
+   * magic entry that matches what previous hdfs versions return.
+   */
+  if (locatedBlocks == null || locatedBlocks.isEmpty()) {
+// Explicitly specified here in case the default DataOutputBuffer
+// buffer length value is changed in future. This matters because the
+// fixed value 32 has to be used to repeat the magic value for previous
+// HDFS version.
+final int lenOfZeroBytes = 32;
+byte[] emptyBlockMd5 = new byte[lenOfZeroBytes];
+MD5Hash fileMD5 = MD5Hash.digest(emptyBlockMd5);
+fileChecksum =  new MD5MD5CRC32GzipFileChecksum(0, 0, fileMD5);
+  } else {
+checksumBlocks();
+fileChecksum = makeFinalResult();
+  }