HDFS-14042. Fix NPE when PROVIDED storage is missing. Contributed by Virajith Jalaparti.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f3f5e7ad Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f3f5e7ad Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f3f5e7ad Branch: refs/heads/HDFS-13891 Commit: f3f5e7ad005a88afad6fa09602073eaa450e21ed Parents: 50f40e0 Author: Giovanni Matteo Fumarola <gif...@apache.org> Authored: Mon Nov 5 11:02:31 2018 -0800 Committer: Giovanni Matteo Fumarola <gif...@apache.org> Committed: Mon Nov 5 11:02:31 2018 -0800 ---------------------------------------------------------------------- .../hdfs/server/blockmanagement/BlockManager.java | 13 ++++++++++++- .../server/blockmanagement/DatanodeDescriptor.java | 4 ++-- .../hdfs/server/blockmanagement/HeartbeatManager.java | 2 +- .../hdfs/server/datanode/TestDataNodeLifeline.java | 5 +++++ 4 files changed, 20 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3f5e7ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index d74b523..a5fb0b1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -2447,7 +2447,7 @@ public class BlockManager implements BlockStatsMXBean { return providedStorageMap.getCapacity(); } - public void updateHeartbeat(DatanodeDescriptor node, StorageReport[] reports, + void updateHeartbeat(DatanodeDescriptor node, StorageReport[] reports, long cacheCapacity, long cacheUsed, int xceiverCount, int failedVolumes, VolumeFailureSummary volumeFailureSummary) { @@ -2458,6 +2458,17 @@ public class BlockManager implements BlockStatsMXBean { failedVolumes, volumeFailureSummary); } + void updateHeartbeatState(DatanodeDescriptor node, + StorageReport[] reports, long cacheCapacity, long cacheUsed, + int xceiverCount, int failedVolumes, + VolumeFailureSummary volumeFailureSummary) { + for (StorageReport report: reports) { + providedStorageMap.updateStorage(node, report.getStorage()); + } + node.updateHeartbeatState(reports, cacheCapacity, cacheUsed, xceiverCount, + failedVolumes, volumeFailureSummary); + } + /** * StatefulBlockInfo is used to build the "toUC" list, which is a list of * updates to the information about under-construction blocks. http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3f5e7ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java index 12b5c33..6aa2376 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java @@ -373,7 +373,7 @@ public class DatanodeDescriptor extends DatanodeInfo { /** * Updates stats from datanode heartbeat. */ - public void updateHeartbeat(StorageReport[] reports, long cacheCapacity, + void updateHeartbeat(StorageReport[] reports, long cacheCapacity, long cacheUsed, int xceiverCount, int volFailures, VolumeFailureSummary volumeFailureSummary) { updateHeartbeatState(reports, cacheCapacity, cacheUsed, xceiverCount, @@ -384,7 +384,7 @@ public class DatanodeDescriptor extends DatanodeInfo { /** * process datanode heartbeat or stats initialization. */ - public void updateHeartbeatState(StorageReport[] reports, long cacheCapacity, + void updateHeartbeatState(StorageReport[] reports, long cacheCapacity, long cacheUsed, int xceiverCount, int volFailures, VolumeFailureSummary volumeFailureSummary) { updateStorageStats(reports, cacheCapacity, cacheUsed, xceiverCount, http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3f5e7ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java index 1972a6d..d2c279f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java @@ -251,7 +251,7 @@ class HeartbeatManager implements DatanodeStatistics { // updateHeartbeat, because we don't want to modify the // heartbeatedSinceRegistration flag. Arrival of a lifeline message does // not count as arrival of the first heartbeat. - node.updateHeartbeatState(reports, cacheCapacity, cacheUsed, + blockManager.updateHeartbeatState(node, reports, cacheCapacity, cacheUsed, xceiverCount, failedVolumes, volumeFailureSummary); stats.add(node); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3f5e7ad/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeLifeline.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeLifeline.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeLifeline.java index 28427bc..14134e6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeLifeline.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeLifeline.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.server.datanode; import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_LIFELINE_INTERVAL_SECONDS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY; @@ -196,6 +197,10 @@ public class TestDataNodeLifeline { namesystem.getNumDeadDataNodes()); assertEquals("Expect DataNode not marked stale due to lifeline.", 0, namesystem.getNumStaleDataNodes()); + // add a new volume on the next heartbeat + cluster.getDataNodes().get(0).reconfigurePropertyImpl( + DFS_DATANODE_DATA_DIR_KEY, + cluster.getDataDirectory().concat("/data-new")); } // Verify that we did in fact call the lifeline RPC. --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
