HDDS-401. Update storage statistics on dead node. Contributed by LiXin Ge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/184544ef Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/184544ef Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/184544ef Branch: refs/heads/HDDS-4 Commit: 184544eff8b3d5951e4f04d080fa00de8e1eec95 Parents: f9c0221 Author: Ajay Kumar <a...@apache.com> Authored: Thu Sep 27 09:56:30 2018 -0700 Committer: Ajay Kumar <a...@apache.com> Committed: Thu Sep 27 09:56:30 2018 -0700 ---------------------------------------------------------------------- .../hadoop/hdds/scm/node/DeadNodeHandler.java | 7 +- .../hadoop/hdds/scm/node/NodeManager.java | 7 ++ .../hadoop/hdds/scm/node/SCMNodeManager.java | 16 +++ .../scm/server/StorageContainerManager.java | 2 +- .../hdds/scm/container/MockNodeManager.java | 15 +++ .../hdds/scm/node/TestDeadNodeHandler.java | 109 ++++++++++++++++--- .../testutils/ReplicationNodeManagerMock.java | 9 ++ 7 files changed, 145 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/184544ef/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java index 7fda67d..88f5df7 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java @@ -42,19 +42,24 @@ public class DeadNodeHandler implements EventHandler<DatanodeDetails> { private final ContainerStateManager containerStateManager; + private final NodeManager nodeManager; + private static final Logger LOG = LoggerFactory.getLogger(DeadNodeHandler.class); public DeadNodeHandler( Node2ContainerMap node2ContainerMap, - ContainerStateManager containerStateManager) { + ContainerStateManager containerStateManager, NodeManager nodeManager) { this.node2ContainerMap = node2ContainerMap; this.containerStateManager = containerStateManager; + this.nodeManager = nodeManager; } @Override public void onMessage(DatanodeDetails datanodeDetails, EventPublisher publisher) { + nodeManager.processDeadNode(datanodeDetails.getUuid()); + Set<ContainerID> containers = node2ContainerMap.getContainers(datanodeDetails.getUuid()); if (containers == null) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/184544ef/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java index deb1628..fc73253 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java @@ -147,4 +147,11 @@ public interface NodeManager extends StorageContainerNodeProtocol, * @param nodeReport */ void processNodeReport(UUID dnUuid, NodeReportProto nodeReport); + + /** + * Process a dead node event in this Node Manager. + * + * @param dnUuid datanode uuid. + */ + void processDeadNode(UUID dnUuid); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/184544ef/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java index 58da1cc..fd5f5de 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java @@ -490,4 +490,20 @@ public class SCMNodeManager addDatanodeCommand(commandForDatanode.getDatanodeId(), commandForDatanode.getCommand()); } + + /** + * Remove the node stats and update the storage stats + * in this SCM Node Manager. + * + * @param dnUuid datanode uuid. + */ + @Override + public void processDeadNode(UUID dnUuid) { + SCMNodeStat stat = nodeStats.get(dnUuid); + LOG.trace("Update stat values as Datanode {} is dead.", dnUuid); + if (stat != null) { + scmStat.subtract(stat); + stat.set(0, 0, 0); + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/184544ef/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index f491d30..52af62d 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -229,7 +229,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl new StaleNodeHandler(node2ContainerMap, scmContainerManager.getPipelineSelector()); DeadNodeHandler deadNodeHandler = new DeadNodeHandler(node2ContainerMap, - getScmContainerManager().getStateManager()); + getScmContainerManager().getStateManager(), scmNodeManager); ContainerActionsHandler actionsHandler = new ContainerActionsHandler(); PendingDeleteHandler pendingDeleteHandler = new PendingDeleteHandler(scmBlockManager.getSCMBlockDeletingService()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/184544ef/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java index 21e44a3..5173b23 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java @@ -422,6 +422,21 @@ public class MockNodeManager implements NodeManager { } /** + * Remove the node stats and update the storage stats + * in this Node Manager. + * + * @param dnUuid UUID of the datanode. + */ + @Override + public void processDeadNode(UUID dnUuid) { + SCMNodeStat stat = this.nodeMetricMap.get(dnUuid); + if (stat != null) { + aggregateStat.subtract(stat); + stat.set(0, 0, 0); + } + } + + /** * A class to declare some values for the nodes so that our tests * won't fail. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/184544ef/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java index 5ca9cb7..698332e 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java @@ -27,19 +27,27 @@ import java.util.stream.Collectors; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto; import org.apache.hadoop.hdds.scm.TestUtils; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerStateManager; import org.apache.hadoop.hdds.scm.container.Mapping; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo; +import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric; import org.apache.hadoop.hdds.scm.container.replication.ReplicationRequest; import org.apache.hadoop.hdds.scm.events.SCMEvents; +import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat; import org.apache.hadoop.hdds.scm.exceptions.SCMException; import org.apache.hadoop.hdds.scm.node.states.Node2ContainerMap; import org.apache.hadoop.hdds.scm.pipelines.PipelineSelector; +import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.NodeReportFromDatanode; import org.apache.hadoop.hdds.server.events.EventPublisher; +import org.apache.hadoop.hdds.server.events.EventQueue; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; import static org.mockito.Matchers.eq; @@ -51,6 +59,29 @@ import org.mockito.Mockito; public class TestDeadNodeHandler { private List<ReplicationRequest> sentEvents = new ArrayList<>(); + private SCMNodeManager nodeManager; + private Node2ContainerMap node2ContainerMap; + private ContainerStateManager containerStateManager; + private NodeReportHandler nodeReportHandler; + private DeadNodeHandler deadNodeHandler; + private EventPublisher publisher; + private EventQueue eventQueue; + + @Before + public void setup() throws IOException { + OzoneConfiguration conf = new OzoneConfiguration(); + node2ContainerMap = new Node2ContainerMap(); + containerStateManager = new ContainerStateManager(conf, + Mockito.mock(Mapping.class), + Mockito.mock(PipelineSelector.class)); + eventQueue = new EventQueue(); + nodeManager = new SCMNodeManager(conf, "cluster1", null, eventQueue); + deadNodeHandler = new DeadNodeHandler(node2ContainerMap, + containerStateManager, nodeManager); + eventQueue.addHandler(SCMEvents.DEAD_NODE, deadNodeHandler); + publisher = Mockito.mock(EventPublisher.class); + nodeReportHandler = new NodeReportHandler(nodeManager); + } @Test public void testOnMessage() throws IOException { @@ -58,13 +89,6 @@ public class TestDeadNodeHandler { DatanodeDetails datanode1 = TestUtils.randomDatanodeDetails(); DatanodeDetails datanode2 = TestUtils.randomDatanodeDetails(); - Node2ContainerMap node2ContainerMap = new Node2ContainerMap(); - ContainerStateManager containerStateManager = new ContainerStateManager( - new OzoneConfiguration(), - Mockito.mock(Mapping.class), - Mockito.mock(PipelineSelector.class) - ); - ContainerInfo container1 = TestUtils.allocateContainer(containerStateManager); ContainerInfo container2 = @@ -72,9 +96,6 @@ public class TestDeadNodeHandler { ContainerInfo container3 = TestUtils.allocateContainer(containerStateManager); - DeadNodeHandler handler = - new DeadNodeHandler(node2ContainerMap, containerStateManager); - registerReplicas(node2ContainerMap, datanode1, container1, container2); registerReplicas(node2ContainerMap, datanode2, container1, container3); @@ -84,10 +105,8 @@ public class TestDeadNodeHandler { TestUtils.closeContainer(containerStateManager, container1); - EventPublisher publisher = Mockito.mock(EventPublisher.class); - //WHEN datanode1 is dead - handler.onMessage(datanode1, publisher); + deadNodeHandler.onMessage(datanode1, publisher); //THEN //node2ContainerMap has not been changed @@ -128,22 +147,76 @@ public class TestDeadNodeHandler { replicationRequestParameter.getValue().getExpecReplicationCount()); } - private void registerReplicas(ContainerStateManager containerStateManager, + @Test + public void testStatisticsUpdate() throws Exception { + //GIVEN + DatanodeDetails datanode1 = TestUtils.randomDatanodeDetails(); + DatanodeDetails datanode2 = TestUtils.randomDatanodeDetails(); + String storagePath1 = GenericTestUtils.getRandomizedTempPath() + .concat("/" + datanode1.getUuidString()); + String storagePath2 = GenericTestUtils.getRandomizedTempPath() + .concat("/" + datanode2.getUuidString()); + + StorageReportProto storageOne = TestUtils.createStorageReport( + datanode1.getUuid(), storagePath1, 100, 10, 90, null); + StorageReportProto storageTwo = TestUtils.createStorageReport( + datanode2.getUuid(), storagePath2, 200, 20, 180, null); + nodeReportHandler.onMessage(getNodeReport(datanode1, storageOne), + Mockito.mock(EventPublisher.class)); + nodeReportHandler.onMessage(getNodeReport(datanode2, storageTwo), + Mockito.mock(EventPublisher.class)); + + ContainerInfo container1 = + TestUtils.allocateContainer(containerStateManager); + registerReplicas(node2ContainerMap, datanode1, container1); + + SCMNodeStat stat = nodeManager.getStats(); + Assert.assertTrue(stat.getCapacity().get() == 300); + Assert.assertTrue(stat.getRemaining().get() == 270); + Assert.assertTrue(stat.getScmUsed().get() == 30); + + SCMNodeMetric nodeStat = nodeManager.getNodeStat(datanode1); + Assert.assertTrue(nodeStat.get().getCapacity().get() == 100); + Assert.assertTrue(nodeStat.get().getRemaining().get() == 90); + Assert.assertTrue(nodeStat.get().getScmUsed().get() == 10); + + //WHEN datanode1 is dead. + eventQueue.fireEvent(SCMEvents.DEAD_NODE, datanode1); + Thread.sleep(100); + + //THEN statistics in SCM should changed. + stat = nodeManager.getStats(); + Assert.assertTrue(stat.getCapacity().get() == 200); + Assert.assertTrue(stat.getRemaining().get() == 180); + Assert.assertTrue(stat.getScmUsed().get() == 20); + + nodeStat = nodeManager.getNodeStat(datanode1); + Assert.assertTrue(nodeStat.get().getCapacity().get() == 0); + Assert.assertTrue(nodeStat.get().getRemaining().get() == 0); + Assert.assertTrue(nodeStat.get().getScmUsed().get() == 0); + } + + private void registerReplicas(ContainerStateManager csm, ContainerInfo container, DatanodeDetails... datanodes) { - containerStateManager.getContainerStateMap() + csm.getContainerStateMap() .addContainerReplica(new ContainerID(container.getContainerID()), datanodes); } - private void registerReplicas(Node2ContainerMap node2ContainerMap, + private void registerReplicas(Node2ContainerMap node2ConMap, DatanodeDetails datanode, ContainerInfo... containers) throws SCMException { - node2ContainerMap + node2ConMap .insertNewDatanode(datanode.getUuid(), Arrays.stream(containers) .map(container -> new ContainerID(container.getContainerID())) .collect(Collectors.toSet())); } -} \ No newline at end of file + private NodeReportFromDatanode getNodeReport(DatanodeDetails dn, + StorageReportProto... reports) { + NodeReportProto nodeReportProto = TestUtils.createNodeReport(reports); + return new NodeReportFromDatanode(dn, nodeReportProto); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/184544ef/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java index e8a6892..4238703 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java @@ -307,4 +307,13 @@ public class ReplicationNodeManagerMock implements NodeManager { EventPublisher publisher) { // do nothing. } + + /** + * Empty implementation for processDeadNode. + * @param dnUuid + */ + @Override + public void processDeadNode(UUID dnUuid) { + // do nothing. + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
