hadoop git commit: HDFS-10512. VolumeScanner may terminate due to NPE in DataNode.reportBadBlocks. Contributed by Wei-Chiu Chuang and Yiqun Lin.

2016-10-18 Thread weichiu 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.7 86f291f05 -> b2dfab432


HDFS-10512. VolumeScanner may terminate due to NPE in DataNode.reportBadBlocks. 
Contributed by Wei-Chiu Chuang and Yiqun Lin.

(cherry picked from commit da6f1b88dd47e22b24d44f6fc8bbee73e85746f7)
(cherry picked from commit 96e68e722f0fe99d1609b38377014c04ef5b6640)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b2dfab43
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b2dfab43
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b2dfab43

Branch: refs/heads/branch-2.7
Commit: b2dfab4326605b4b282a3ae75d00223b35979662
Parents: 86f291f
Author: Yongjun Zhang 
Authored: Fri Jul 8 19:40:44 2016 -0700
Committer: Wei-Chiu Chuang 
Committed: Tue Oct 18 09:09:06 2016 -0700

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 ++
 .../hadoop/hdfs/server/datanode/DataNode.java   | 23 ++-
 .../hdfs/server/datanode/VolumeScanner.java |  2 +-
 .../datanode/fsdataset/impl/FsDatasetImpl.java  |  3 +-
 .../fsdataset/impl/TestFsDatasetImpl.java   | 42 
 5 files changed, 70 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2dfab43/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 277efe1..a13a566 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -154,6 +154,9 @@ Release 2.7.4 - UNRELEASED
 HDFS-11002. Fix broken attr/getfattr/setfattr links in
 ExtendedAttributes.md. (Mingliang Liu via aajisaka)
 
+HDFS-10512. VolumeScanner may terminate due to NPE in 
+DataNode.reportBadBlocks. Contributed by Wei-Chiu Chuang and Yiqun Lin.
+
 Release 2.7.3 - 2016-08-25
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2dfab43/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 9ef23d4..eb159eb 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -990,8 +990,25 @@ public class DataNode extends ReconfigurableBase
* Report a bad block which is hosted on the local DN.
*/
   public void reportBadBlocks(ExtendedBlock block) throws IOException{
-BPOfferService bpos = getBPOSForBlock(block);
 FsVolumeSpi volume = getFSDataset().getVolume(block);
+if (volume == null) {
+  LOG.warn("Cannot find FsVolumeSpi to report bad block: " + block);
+  return;
+}
+reportBadBlocks(block, volume);
+  }
+
+  /**
+   * Report a bad block which is hosted on the local DN.
+   *
+   * @param block the bad block which is hosted on the local DN
+   * @param volume the volume that block is stored in and the volume
+   *must not be null
+   * @throws IOException
+   */
+  public void reportBadBlocks(ExtendedBlock block, FsVolumeSpi volume)
+  throws IOException {
+BPOfferService bpos = getBPOSForBlock(block);
 bpos.reportBadBlocks(
 block, volume.getStorageID(), volume.getStorageType());
   }
@@ -1860,6 +1877,10 @@ public class DataNode extends ReconfigurableBase
   private void reportBadBlock(final BPOfferService bpos,
   final ExtendedBlock block, final String msg) {
 FsVolumeSpi volume = getFSDataset().getVolume(block);
+if (volume == null) {
+  LOG.warn("Cannot find FsVolumeSpi to report bad block: " + block);
+  return;
+}
 bpos.reportBadBlocks(
 block, volume.getStorageID(), volume.getStorageType());
 LOG.warn(msg);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2dfab43/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
index ad546d2..d0300f6 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.

hadoop git commit: HDFS-10512. VolumeScanner may terminate due to NPE in DataNode.reportBadBlocks. Contributed by Wei-Chiu Chuang and Yiqun Lin.

2016-07-08 Thread yjzhangal 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 845daee46 -> 96e68e722


HDFS-10512. VolumeScanner may terminate due to NPE in DataNode.reportBadBlocks. 
Contributed by Wei-Chiu Chuang and Yiqun Lin.

(cherry picked from commit da6f1b88dd47e22b24d44f6fc8bbee73e85746f7)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/96e68e72
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/96e68e72
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/96e68e72

Branch: refs/heads/branch-2.8
Commit: 96e68e722f0fe99d1609b38377014c04ef5b6640
Parents: 845daee
Author: Yongjun Zhang 
Authored: Fri Jul 8 19:40:44 2016 -0700
Committer: Yongjun Zhang 
Committed: Fri Jul 8 22:20:27 2016 -0700

--
 .../hadoop/hdfs/server/datanode/DataNode.java   | 23 ++-
 .../hdfs/server/datanode/VolumeScanner.java |  2 +-
 .../datanode/fsdataset/impl/FsDatasetImpl.java  |  3 +-
 .../fsdataset/impl/TestFsDatasetImpl.java   | 43 
 4 files changed, 68 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/96e68e72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 9a5735d..5d6514e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -1127,8 +1127,25 @@ public class DataNode extends ReconfigurableBase
* Report a bad block which is hosted on the local DN.
*/
   public void reportBadBlocks(ExtendedBlock block) throws IOException{
-BPOfferService bpos = getBPOSForBlock(block);
 FsVolumeSpi volume = getFSDataset().getVolume(block);
+if (volume == null) {
+  LOG.warn("Cannot find FsVolumeSpi to report bad block: " + block);
+  return;
+}
+reportBadBlocks(block, volume);
+  }
+
+  /**
+   * Report a bad block which is hosted on the local DN.
+   *
+   * @param block the bad block which is hosted on the local DN
+   * @param volume the volume that block is stored in and the volume
+   *must not be null
+   * @throws IOException
+   */
+  public void reportBadBlocks(ExtendedBlock block, FsVolumeSpi volume)
+  throws IOException {
+BPOfferService bpos = getBPOSForBlock(block);
 bpos.reportBadBlocks(
 block, volume.getStorageID(), volume.getStorageType());
   }
@@ -2043,6 +2060,10 @@ public class DataNode extends ReconfigurableBase
   private void reportBadBlock(final BPOfferService bpos,
   final ExtendedBlock block, final String msg) {
 FsVolumeSpi volume = getFSDataset().getVolume(block);
+if (volume == null) {
+  LOG.warn("Cannot find FsVolumeSpi to report bad block: " + block);
+  return;
+}
 bpos.reportBadBlocks(
 block, volume.getStorageID(), volume.getStorageType());
 LOG.warn(msg);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96e68e72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
index d1f2d05..6da6eb0 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
@@ -283,7 +283,7 @@ public class VolumeScanner extends Thread {
   }
   LOG.warn("Reporting bad {} on {}", block, volume.getBasePath());
   try {
-scanner.datanode.reportBadBlocks(block);
+scanner.datanode.reportBadBlocks(block, volume);
   } catch (IOException ie) {
 // This is bad, but not bad enough to shut down the scanner.
 LOG.warn("Cannot report bad " + block.getBlockId(), e);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96e68e72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs

hadoop git commit: HDFS-10512. VolumeScanner may terminate due to NPE in DataNode.reportBadBlocks. Contributed by Wei-Chiu Chuang and Yiqun Lin.

2016-07-08 Thread yjzhangal 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 e4023f8b1 -> ee23094b1


HDFS-10512. VolumeScanner may terminate due to NPE in DataNode.reportBadBlocks. 
Contributed by Wei-Chiu Chuang and Yiqun Lin.

(cherry picked from commit da6f1b88dd47e22b24d44f6fc8bbee73e85746f7)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee23094b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee23094b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee23094b

Branch: refs/heads/branch-2
Commit: ee23094b14bf70481ea2f453863f88c4804e5a35
Parents: e4023f8
Author: Yongjun Zhang 
Authored: Fri Jul 8 19:40:44 2016 -0700
Committer: Yongjun Zhang 
Committed: Fri Jul 8 22:01:31 2016 -0700

--
 .../hadoop/hdfs/server/datanode/DataNode.java   | 23 ++-
 .../hdfs/server/datanode/VolumeScanner.java |  2 +-
 .../datanode/fsdataset/impl/FsDatasetImpl.java  |  3 +-
 .../fsdataset/impl/TestFsDatasetImpl.java   | 43 
 4 files changed, 68 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee23094b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index e7b72f7..f9298f1 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -1139,8 +1139,25 @@ public class DataNode extends ReconfigurableBase
* Report a bad block which is hosted on the local DN.
*/
   public void reportBadBlocks(ExtendedBlock block) throws IOException{
-BPOfferService bpos = getBPOSForBlock(block);
 FsVolumeSpi volume = getFSDataset().getVolume(block);
+if (volume == null) {
+  LOG.warn("Cannot find FsVolumeSpi to report bad block: " + block);
+  return;
+}
+reportBadBlocks(block, volume);
+  }
+
+  /**
+   * Report a bad block which is hosted on the local DN.
+   *
+   * @param block the bad block which is hosted on the local DN
+   * @param volume the volume that block is stored in and the volume
+   *must not be null
+   * @throws IOException
+   */
+  public void reportBadBlocks(ExtendedBlock block, FsVolumeSpi volume)
+  throws IOException {
+BPOfferService bpos = getBPOSForBlock(block);
 bpos.reportBadBlocks(
 block, volume.getStorageID(), volume.getStorageType());
   }
@@ -2056,6 +2073,10 @@ public class DataNode extends ReconfigurableBase
   private void reportBadBlock(final BPOfferService bpos,
   final ExtendedBlock block, final String msg) {
 FsVolumeSpi volume = getFSDataset().getVolume(block);
+if (volume == null) {
+  LOG.warn("Cannot find FsVolumeSpi to report bad block: " + block);
+  return;
+}
 bpos.reportBadBlocks(
 block, volume.getStorageID(), volume.getStorageType());
 LOG.warn(msg);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee23094b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
index d0dc9ed..7a9ecf2 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
@@ -283,7 +283,7 @@ public class VolumeScanner extends Thread {
   }
   LOG.warn("Reporting bad {} on {}", block, volume.getBasePath());
   try {
-scanner.datanode.reportBadBlocks(block);
+scanner.datanode.reportBadBlocks(block, volume);
   } catch (IOException ie) {
 // This is bad, but not bad enough to shut down the scanner.
 LOG.warn("Cannot report bad " + block.getBlockId(), e);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee23094b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ser

hadoop git commit: HDFS-10512. VolumeScanner may terminate due to NPE in DataNode.reportBadBlocks. Contributed by Wei-Chiu Chuang and Yiqun Lin.

2016-07-08 Thread yjzhangal 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 932aed64d -> da6f1b88d


HDFS-10512. VolumeScanner may terminate due to NPE in DataNode.reportBadBlocks. 
Contributed by Wei-Chiu Chuang and Yiqun Lin.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/da6f1b88
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/da6f1b88
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/da6f1b88

Branch: refs/heads/trunk
Commit: da6f1b88dd47e22b24d44f6fc8bbee73e85746f7
Parents: 932aed6
Author: Yongjun Zhang 
Authored: Fri Jul 8 19:40:44 2016 -0700
Committer: Yongjun Zhang 
Committed: Fri Jul 8 19:40:44 2016 -0700

--
 .../hadoop/hdfs/server/datanode/DataNode.java   | 23 ++-
 .../hdfs/server/datanode/VolumeScanner.java |  2 +-
 .../datanode/fsdataset/impl/FsDatasetImpl.java  |  3 +-
 .../fsdataset/impl/TestFsDatasetImpl.java   | 43 
 4 files changed, 68 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/da6f1b88/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 32de34c..1cd2dee 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -1169,8 +1169,25 @@ public class DataNode extends ReconfigurableBase
* Report a bad block which is hosted on the local DN.
*/
   public void reportBadBlocks(ExtendedBlock block) throws IOException{
-BPOfferService bpos = getBPOSForBlock(block);
 FsVolumeSpi volume = getFSDataset().getVolume(block);
+if (volume == null) {
+  LOG.warn("Cannot find FsVolumeSpi to report bad block: " + block);
+  return;
+}
+reportBadBlocks(block, volume);
+  }
+
+  /**
+   * Report a bad block which is hosted on the local DN.
+   *
+   * @param block the bad block which is hosted on the local DN
+   * @param volume the volume that block is stored in and the volume
+   *must not be null
+   * @throws IOException
+   */
+  public void reportBadBlocks(ExtendedBlock block, FsVolumeSpi volume)
+  throws IOException {
+BPOfferService bpos = getBPOSForBlock(block);
 bpos.reportBadBlocks(
 block, volume.getStorageID(), volume.getStorageType());
   }
@@ -2101,6 +2118,10 @@ public class DataNode extends ReconfigurableBase
   private void reportBadBlock(final BPOfferService bpos,
   final ExtendedBlock block, final String msg) {
 FsVolumeSpi volume = getFSDataset().getVolume(block);
+if (volume == null) {
+  LOG.warn("Cannot find FsVolumeSpi to report bad block: " + block);
+  return;
+}
 bpos.reportBadBlocks(
 block, volume.getStorageID(), volume.getStorageType());
 LOG.warn(msg);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/da6f1b88/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
index d0dc9ed..7a9ecf2 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
@@ -283,7 +283,7 @@ public class VolumeScanner extends Thread {
   }
   LOG.warn("Reporting bad {} on {}", block, volume.getBasePath());
   try {
-scanner.datanode.reportBadBlocks(block);
+scanner.datanode.reportBadBlocks(block, volume);
   } catch (IOException ie) {
 // This is bad, but not bad enough to shut down the scanner.
 LOG.warn("Cannot report bad " + block.getBlockId(), e);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/da6f1b88/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 2b40538..c0f2fbd 100644