Repository: hadoop Updated Branches: refs/heads/HDFS-7240 558b478ff -> 68eab679e
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68eab679/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/placement/TestContainerPlacement.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/placement/TestContainerPlacement.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/placement/TestContainerPlacement.java new file mode 100644 index 0000000..d798c61 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/placement/TestContainerPlacement.java @@ -0,0 +1,130 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.container.placement; + +import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.protocol.DatanodeID; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.scm.container.MockNodeManager; +import org.apache.hadoop.ozone.scm.container.placement.algorithms.SCMContainerPlacementCapacity; +import org.apache.hadoop.ozone.scm.container.placement.algorithms.SCMContainerPlacementRandom; +import org.apache.hadoop.ozone.scm.exceptions.SCMException; +import org.apache.hadoop.ozone.scm.node.NodeManager; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; +import java.util.Random; + +import static org.junit.Assert.assertEquals; + +/** + * Asserts that allocation strategy works as expected. + */ +public class TestContainerPlacement { + + private DescriptiveStatistics computeStatistics(NodeManager nodeManager) { + DescriptiveStatistics descriptiveStatistics = new DescriptiveStatistics(); + for (DatanodeID id : nodeManager.getNodes(NodeManager.NODESTATE.HEALTHY)) { + float weightedValue = + nodeManager.getNodeStat(id).get().getScmUsed().get() / (float) + nodeManager.getNodeStat(id).get().getCapacity().get(); + descriptiveStatistics.addValue(weightedValue); + } + return descriptiveStatistics; + } + + /** + * This test simulates lots of Cluster I/O and updates the metadata in SCM. + * We simulate adding and removing containers from the cluster. It asserts + * that our placement algorithm has taken the capacity of nodes into + * consideration by asserting that standard deviation of used space on these + * has improved. + */ + @Test + public void testCapacityPlacementYieldsBetterDataDistribution() throws + SCMException { + final int opsCount = 200 * 1000; + final int nodesRequired = 3; + Random random = new Random(); + + // The nature of init code in MockNodeManager yields similar clusters. + MockNodeManager nodeManagerCapacity = new MockNodeManager(true, 100); + MockNodeManager nodeManagerRandom = new MockNodeManager(true, 100); + DescriptiveStatistics beforeCapacity = + computeStatistics(nodeManagerCapacity); + DescriptiveStatistics beforeRandom = computeStatistics(nodeManagerRandom); + + //Assert that our initial layout of clusters are similar. + assertEquals(beforeCapacity.getStandardDeviation(), beforeRandom + .getStandardDeviation(), 0.001); + + SCMContainerPlacementCapacity capacityPlacer = new + SCMContainerPlacementCapacity(nodeManagerCapacity, new Configuration()); + SCMContainerPlacementRandom randomPlacer = new + SCMContainerPlacementRandom(nodeManagerRandom, new Configuration()); + + for (int x = 0; x < opsCount; x++) { + long containerSize = random.nextInt(100) * OzoneConsts.GB; + List<DatanodeID> nodesCapacity = + capacityPlacer.chooseDatanodes(nodesRequired, containerSize); + assertEquals(nodesRequired, nodesCapacity.size()); + + List<DatanodeID> nodesRandom = randomPlacer.chooseDatanodes(nodesRequired, + containerSize); + + // One fifth of all calls are delete + if (x % 5 == 0) { + deleteContainer(nodeManagerCapacity, nodesCapacity, containerSize); + deleteContainer(nodeManagerRandom, nodesRandom, containerSize); + } else { + createContainer(nodeManagerCapacity, nodesCapacity, containerSize); + createContainer(nodeManagerRandom, nodesRandom, containerSize); + } + } + DescriptiveStatistics postCapacity = computeStatistics(nodeManagerCapacity); + DescriptiveStatistics postRandom = computeStatistics(nodeManagerRandom); + + // This is a very bold claim, and needs large number of I/O operations. + // The claim in this assertion is that we improved the data distribution + // of this cluster in relation to the start state of the cluster. + Assert.assertTrue(beforeCapacity.getStandardDeviation() > + postCapacity.getStandardDeviation()); + + // This asserts that Capacity placement yields a better placement + // algorithm than random placement, since both cluster started at an + // identical state. + + Assert.assertTrue(postRandom.getStandardDeviation() > + postCapacity.getStandardDeviation()); + } + + private void deleteContainer(MockNodeManager nodeManager, + List<DatanodeID> nodes, long containerSize) { + for (DatanodeID id : nodes) { + nodeManager.delContainer(id, containerSize); + } + } + + private void createContainer(MockNodeManager nodeManager, + List<DatanodeID> nodes, long containerSize) { + for (DatanodeID id : nodes) { + nodeManager.addContainer(id, containerSize); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/68eab679/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/placement/TestDatanodeMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/placement/TestDatanodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/placement/TestDatanodeMetrics.java new file mode 100644 index 0000000..ccc23c3 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/placement/TestDatanodeMetrics.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.container.placement; + +import org.apache.hadoop.ozone.scm.container.placement.metrics.SCMNodeMetric; +import org.apache.hadoop.ozone.scm.container.placement.metrics.SCMNodeStat; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Tests that test Metrics that support placement. + */ +public class TestDatanodeMetrics { + @Rule + public ExpectedException exception = ExpectedException.none(); + @Test + public void testSCMNodeMetric() { + SCMNodeStat stat = new SCMNodeStat(100L, 10L, 90L); + assertEquals((long) stat.getCapacity().get(), 100L); + assertEquals((long) stat.getScmUsed().get(), 10L); + assertEquals((long) stat.getRemaining().get(), 90L); + SCMNodeMetric metric = new SCMNodeMetric(stat); + + SCMNodeStat newStat = new SCMNodeStat(100L, 10L, 90L); + assertEquals((long) stat.getCapacity().get(), 100L); + assertEquals((long) stat.getScmUsed().get(), 10L); + assertEquals((long) stat.getRemaining().get(), 90L); + + SCMNodeMetric newMetric = new SCMNodeMetric(newStat); + assertTrue(metric.isEqual(newMetric.get())); + + newMetric.add(stat); + assertTrue(newMetric.isGreater(metric.get())); + + SCMNodeMetric zeroMetric = new SCMNodeMetric(new SCMNodeStat()); + // Assert we can handle zero capacity. + assertTrue(metric.isGreater(zeroMetric.get())); + + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/68eab679/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java index 8a8ea68..a30707c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java @@ -22,8 +22,8 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfiguration; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.scm.container.ContainerPlacementPolicy; -import org.apache.hadoop.ozone.scm.container.SCMContainerPlacementCapacity; +import org.apache.hadoop.ozone.scm.container.placement.algorithms.ContainerPlacementPolicy; +import org.apache.hadoop.ozone.scm.container.placement.algorithms.SCMContainerPlacementCapacity; import org.apache.hadoop.scm.ScmConfigKeys; import org.apache.hadoop.scm.protocolPB .StorageContainerLocationProtocolClientSideTranslatorPB; http://git-wip-us.apache.org/repos/asf/hadoop/blob/68eab679/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/container/MockNodeManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/container/MockNodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/container/MockNodeManager.java index 14c36fc..638de9e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/container/MockNodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/container/MockNodeManager.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.scm.container; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.common.SCMTestUtils; import org.apache.hadoop.ozone.protocol.VersionResponse; import org.apache.hadoop.ozone.protocol.commands.SCMCommand; @@ -25,9 +26,9 @@ import org.apache.hadoop.ozone.protocol.proto .StorageContainerDatanodeProtocolProtos; import org.apache.hadoop.ozone.protocol.proto .StorageContainerDatanodeProtocolProtos.SCMNodeReport; - +import org.apache.hadoop.ozone.scm.container.placement.metrics.SCMNodeMetric; +import org.apache.hadoop.ozone.scm.container.placement.metrics.SCMNodeStat; import org.apache.hadoop.ozone.scm.node.NodeManager; -import org.apache.hadoop.ozone.scm.node.SCMNodeStat; import java.io.IOException; import java.util.HashMap; @@ -39,21 +40,55 @@ import java.util.Map; * Test Helper for testing container Mapping. */ public class MockNodeManager implements NodeManager { - private final List<DatanodeID> healthyNodes; private static final int HEALTHY_NODE_COUNT = 10; + private final static NodeData[] NODES = { + new NodeData(10L * OzoneConsts.TB, OzoneConsts.GB), + new NodeData(64L * OzoneConsts.TB, 100 * OzoneConsts.GB), + new NodeData(128L * OzoneConsts.TB, 256 * OzoneConsts.GB), + new NodeData(40L * OzoneConsts.TB, OzoneConsts.TB), + new NodeData(256L * OzoneConsts.TB, 200 * OzoneConsts.TB), + new NodeData(20L * OzoneConsts.TB, 10 * OzoneConsts.GB), + new NodeData(32L * OzoneConsts.TB, 16 * OzoneConsts.TB), + new NodeData(OzoneConsts.TB, 900 * OzoneConsts.GB), + }; + private final List<DatanodeID> healthyNodes; + private final Map<String, SCMNodeStat> nodeMetricMap; + private final SCMNodeStat aggregateStat; private boolean chillmode; - public MockNodeManager() { + public MockNodeManager(boolean initializeFakeNodes, int nodeCount) { this.healthyNodes = new LinkedList<>(); - for (int x = 0; x < 10; x++) { - healthyNodes.add(SCMTestUtils.getDatanodeID()); + this.nodeMetricMap = new HashMap<>(); + aggregateStat = new SCMNodeStat(); + if (initializeFakeNodes) { + for (int x = 0; x < nodeCount; x++) { + DatanodeID id = SCMTestUtils.getDatanodeID(); + healthyNodes.add(id); + populateNodeMetric(id, x); + } } chillmode = false; } /** + * Invoked from ctor to create some node Metrics. + * + * @param datanodeID - Datanode ID + */ + private void populateNodeMetric(DatanodeID datanodeID, int x) { + SCMNodeStat newStat = new SCMNodeStat(); + long remaining = + NODES[x % NODES.length].capacity - NODES[x % NODES.length].used; + newStat.set( + (NODES[x % NODES.length].capacity), + (NODES[x % NODES.length].used), remaining); + this.nodeMetricMap.put(datanodeID.toString(), newStat); + aggregateStat.add(newStat); + } + + /** * Sets the chill mode value. - * @param chillmode boolean + * @param chillmode boolean */ public void setChillmode(boolean chillmode) { this.chillmode = chillmode; @@ -184,7 +219,7 @@ public class MockNodeManager implements NodeManager { */ @Override public SCMNodeStat getStats() { - return null; + return aggregateStat; } /** @@ -193,7 +228,7 @@ public class MockNodeManager implements NodeManager { */ @Override public Map<String, SCMNodeStat> getNodeStats() { - return null; + return nodeMetricMap; } /** @@ -202,8 +237,8 @@ public class MockNodeManager implements NodeManager { * @return node stat if it is live/stale, null if it is dead or does't exist. */ @Override - public SCMNodeStat getNodeStat(DatanodeID datanodeID) { - return null; + public SCMNodeMetric getNodeStat(DatanodeID datanodeID) { + return new SCMNodeMetric(nodeMetricMap.get(datanodeID.toString())); } /** @@ -283,15 +318,103 @@ public class MockNodeManager implements NodeManager { @Override public List<SCMCommand> sendHeartbeat(DatanodeID datanodeID, SCMNodeReport nodeReport) { + if ((datanodeID != null) && (nodeReport != null) && (nodeReport + .getStorageReportCount() > 0)) { + SCMNodeStat stat = this.nodeMetricMap.get(datanodeID.toString()); + + long totalCapacity = 0L; + long totalRemaining = 0L; + long totalScmUsed = 0L; + List<StorageContainerDatanodeProtocolProtos.SCMStorageReport> + storageReports = nodeReport.getStorageReportList(); + for (StorageContainerDatanodeProtocolProtos.SCMStorageReport report : + storageReports) { + totalCapacity += report.getCapacity(); + totalRemaining +=report.getRemaining(); + totalScmUsed += report.getScmUsed(); + } + aggregateStat.subtract(stat); + stat.set(totalCapacity, totalScmUsed, totalRemaining); + aggregateStat.add(stat); + nodeMetricMap.put(datanodeID.toString(), stat); + + } return null; } @Override public Map<String, Integer> getNodeCount() { Map<String, Integer> nodeCountMap = new HashMap<String, Integer>(); - for(NodeManager.NODESTATE state : NodeManager.NODESTATE.values()) { + for (NodeManager.NODESTATE state : NodeManager.NODESTATE.values()) { nodeCountMap.put(state.toString(), getNodeCount(state)); } return nodeCountMap; } + + /** + * Makes it easy to add a container. + * + * @param datanodeID datanode ID + * @param size number of bytes. + */ + public void addContainer(DatanodeID datanodeID, long size) { + SCMNodeStat stat = this.nodeMetricMap.get(datanodeID.toString()); + if (stat != null) { + aggregateStat.subtract(stat); + stat.getCapacity().add(size); + aggregateStat.add(stat); + nodeMetricMap.put(datanodeID.toString(), stat); + } + } + + /** + * Makes it easy to simulate a delete of a container. + * + * @param datanodeID datanode ID + * @param size number of bytes. + */ + public void delContainer(DatanodeID datanodeID, long size) { + SCMNodeStat stat = this.nodeMetricMap.get(datanodeID.toString()); + if (stat != null) { + aggregateStat.subtract(stat); + stat.getCapacity().subtract(size); + aggregateStat.add(stat); + nodeMetricMap.put(datanodeID.toString(), stat); + } + } + + /** + * A class to declare some values for the nodes so that our tests + * won't fail. + */ + private static class NodeData { + private long capacity, used; + + /** + * Constructs a nodeDefinition. + * + * @param capacity capacity. + * @param used used. + */ + NodeData(long capacity, long used) { + this.capacity = capacity; + this.used = used; + } + + public long getCapacity() { + return capacity; + } + + public void setCapacity(long capacity) { + this.capacity = capacity; + } + + public long getUsed() { + return used; + } + + public void setUsed(long used) { + this.used = used; + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/68eab679/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/container/TestContainerMapping.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/container/TestContainerMapping.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/container/TestContainerMapping.java index d83053a..0b9b76f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/container/TestContainerMapping.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/container/TestContainerMapping.java @@ -59,15 +59,17 @@ public class TestContainerMapping { testDir = Paths.get(path).toFile(); boolean folderExisted = testDir.exists() || testDir.mkdirs(); if (!folderExisted) { - throw new IOException("Unable to create test diectory path"); + throw new IOException("Unable to create test directory path"); } - nodeManager = new MockNodeManager(); + nodeManager = new MockNodeManager(true, 10); mapping = new ContainerMapping(conf, nodeManager, 128); } @AfterClass public static void cleanup() throws IOException { - mapping.close(); + if(mapping != null) { + mapping.close(); + } FileUtil.fullyDelete(testDir); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/68eab679/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestContainerPlacement.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestContainerPlacement.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestContainerPlacement.java index 02f1486..d28866e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestContainerPlacement.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestContainerPlacement.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -26,10 +26,11 @@ import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConfiguration; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.common.SCMTestUtils; -import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto + .StorageContainerDatanodeProtocolProtos; import org.apache.hadoop.ozone.scm.container.ContainerMapping; -import org.apache.hadoop.ozone.scm.container.ContainerPlacementPolicy; -import org.apache.hadoop.ozone.scm.container.SCMContainerPlacementCapacity; +import org.apache.hadoop.ozone.scm.container.placement.algorithms.ContainerPlacementPolicy; +import org.apache.hadoop.ozone.scm.container.placement.algorithms.SCMContainerPlacementCapacity; import org.apache.hadoop.scm.ScmConfigKeys; import org.apache.hadoop.scm.client.ScmClient; import org.apache.hadoop.scm.container.common.helpers.Pipeline; @@ -93,6 +94,7 @@ public class TestContainerPlacement { return new ContainerMapping(config, scmNodeManager, cacheSize); } + /** * Test capacity based container placement policy with node reports. * @@ -122,7 +124,7 @@ public class TestContainerPlacement { List<DatanodeID> datanodes = SCMTestUtils.getRegisteredDatanodeIDs(nodeManager, nodeCount); try { - for (DatanodeID datanodeID: datanodes) { + for (DatanodeID datanodeID : datanodes) { StorageContainerDatanodeProtocolProtos.SCMNodeReport.Builder nrb = StorageContainerDatanodeProtocolProtos.SCMNodeReport.newBuilder(); StorageContainerDatanodeProtocolProtos.SCMStorageReport.Builder srb = @@ -139,11 +141,11 @@ public class TestContainerPlacement { 100, 4 * 1000); assertEquals(nodeCount, nodeManager.getNodeCount(HEALTHY)); assertEquals(capacity * nodeCount, - nodeManager.getStats().getCapacity()); + (long) nodeManager.getStats().getCapacity().get()); assertEquals(used * nodeCount, - nodeManager.getStats().getScmUsed()); + (long) nodeManager.getStats().getScmUsed().get()); assertEquals(remaining * nodeCount, - nodeManager.getStats().getRemaining()); + (long) nodeManager.getStats().getRemaining().get()); assertTrue(nodeManager.isOutOfNodeChillMode()); @@ -155,7 +157,7 @@ public class TestContainerPlacement { final long newUsed = 7L * OzoneConsts.GB; final long newRemaining = capacity - newUsed; - for (DatanodeID datanodeID: datanodes) { + for (DatanodeID datanodeID : datanodes) { StorageContainerDatanodeProtocolProtos.SCMNodeReport.Builder nrb = StorageContainerDatanodeProtocolProtos.SCMNodeReport.newBuilder(); StorageContainerDatanodeProtocolProtos.SCMStorageReport.Builder srb = @@ -168,14 +170,14 @@ public class TestContainerPlacement { nrb.addStorageReport(srb).build()); } - GenericTestUtils.waitFor(() -> nodeManager.getStats().getRemaining() == - nodeCount * newRemaining, + GenericTestUtils.waitFor(() -> nodeManager.getStats().getRemaining() + .get() == nodeCount * newRemaining, 100, 4 * 1000); thrown.expect(IOException.class); thrown.expectMessage( - startsWith("No healthy node found with enough remaining capacity to" + - " allocate container.")); + startsWith("Unable to find enough nodes that meet the space " + + "requirement in healthy node set.")); String container2 = UUID.randomUUID().toString(); containerManager.allocateContainer(container2, ScmClient.ReplicationFactor.THREE); http://git-wip-us.apache.org/repos/asf/hadoop/blob/68eab679/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestNodeManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestNodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestNodeManager.java index da567cb..25ef7cc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestNodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestNodeManager.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,6 +26,7 @@ import org.apache.hadoop.ozone.protocol.proto .StorageContainerDatanodeProtocolProtos.SCMNodeReport; import org.apache.hadoop.ozone.protocol.proto .StorageContainerDatanodeProtocolProtos.SCMStorageReport; +import org.apache.hadoop.ozone.scm.container.placement.metrics.SCMNodeStat; import org.apache.hadoop.scm.ScmConfigKeys; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.PathUtils; @@ -45,14 +46,14 @@ import java.util.List; import java.util.UUID; import java.util.concurrent.TimeoutException; +import static org.apache.hadoop.ozone.scm.node.NodeManager.NODESTATE.DEAD; +import static org.apache.hadoop.ozone.scm.node.NodeManager.NODESTATE.HEALTHY; +import static org.apache.hadoop.ozone.scm.node.NodeManager.NODESTATE.STALE; import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL_MS; import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_INTERVAL_SECONDS; import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS; import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_MAX_HB_COUNT_TO_PROCESS; import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL_MS; -import static org.apache.hadoop.ozone.scm.node.NodeManager.NODESTATE.HEALTHY; -import static org.apache.hadoop.ozone.scm.node.NodeManager.NODESTATE.STALE; -import static org.apache.hadoop.ozone.scm.node.NodeManager.NODESTATE.DEAD; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.StringStartsWith.startsWith; @@ -158,7 +159,7 @@ public class TestNodeManager { GenericTestUtils.waitFor(() -> nodeManager.waitForHeartbeatProcessed(), 100, 4 * 1000); assertFalse("No heartbeats, Node manager should have been in" + - " chill mode.", nodeManager.isOutOfNodeChillMode()); + " chill mode.", nodeManager.isOutOfNodeChillMode()); } } @@ -208,7 +209,7 @@ public class TestNodeManager { GenericTestUtils.waitFor(() -> nodeManager.waitForHeartbeatProcessed(), 100, 4 * 1000); assertFalse("Not enough nodes have send heartbeat to node" + - "manager.", nodeManager.isOutOfNodeChillMode()); + "manager.", nodeManager.isOutOfNodeChillMode()); } } @@ -631,11 +632,12 @@ public class TestNodeManager { * @throws InterruptedException */ private void heartbeatNodeSet(SCMNodeManager manager, List<DatanodeID> list, - int sleepDuration) throws InterruptedException { + int sleepDuration) throws InterruptedException { while (!Thread.currentThread().isInterrupted()) { for (DatanodeID dn : list) { manager.sendHeartbeat(dn, null); - } Thread.sleep(sleepDuration); + } + Thread.sleep(sleepDuration); } } @@ -664,7 +666,7 @@ public class TestNodeManager { * @return true if we found the expected number. */ private boolean findNodes(NodeManager nodeManager, int count, - NodeManager.NODESTATE state) { + NodeManager.NODESTATE state) { return count == nodeManager.getNodeCount(state); } @@ -690,7 +692,6 @@ public class TestNodeManager { conf.setInt(OZONE_SCM_MAX_HB_COUNT_TO_PROCESS, 7000); - try (SCMNodeManager nodeManager = createNodeManager(conf)) { List<DatanodeID> healthyNodeList = createNodeSet(nodeManager, healthyCount, "Healthy"); @@ -902,7 +903,7 @@ public class TestNodeManager { // Assert that node manager force enter cannot be overridden by nodes HBs. - for(int x= 0; x < 20; x++) { + for (int x = 0; x < 20; x++) { DatanodeID datanode = SCMTestUtils.getDatanodeID(nodeManager); nodeManager.sendHeartbeat(datanode, null); } @@ -952,12 +953,12 @@ public class TestNodeManager { GenericTestUtils.waitFor(() -> nodeManager.waitForHeartbeatProcessed(), 100, 4 * 1000); assertEquals(nodeCount, nodeManager.getNodeCount(HEALTHY)); - assertEquals(capacity * nodeCount, - nodeManager.getStats().getCapacity()); - assertEquals(used * nodeCount, - nodeManager.getStats().getScmUsed()); - assertEquals(remaining * nodeCount, - nodeManager.getStats().getRemaining()); + assertEquals(capacity * nodeCount, (long) nodeManager.getStats() + .getCapacity().get()); + assertEquals(used * nodeCount, (long) nodeManager.getStats() + .getScmUsed().get()); + assertEquals(remaining * nodeCount, (long) nodeManager.getStats() + .getRemaining().get()); } } @@ -998,31 +999,41 @@ public class TestNodeManager { Thread.sleep(100); } - final long expectedScmUsed = usedPerHeartbeat * (heartbeatCount -1); - final long expectedRemaining = capacity - - usedPerHeartbeat * (heartbeatCount - 1); + final long expectedScmUsed = usedPerHeartbeat * (heartbeatCount - 1); + final long expectedRemaining = capacity - expectedScmUsed; GenericTestUtils.waitFor( - () -> nodeManager.getStats().getScmUsed() == expectedScmUsed, 100, - 4 * 1000); + () -> nodeManager.getStats().getScmUsed().get() == expectedScmUsed, + 100, 4 * 1000); - assertEquals(capacity, nodeManager.getStats().getCapacity()); - assertEquals(expectedScmUsed, nodeManager.getStats().getScmUsed()); - assertEquals(expectedRemaining, nodeManager.getStats().getRemaining()); + long foundCapacity = nodeManager.getStats().getCapacity().get(); + assertEquals(capacity, foundCapacity); + + long foundScmUsed = nodeManager.getStats().getScmUsed().get(); + assertEquals(expectedScmUsed, foundScmUsed); + + long foundRemaining = nodeManager.getStats().getRemaining().get(); + assertEquals(expectedRemaining, foundRemaining); // Test NodeManager#getNodeStats assertEquals(nodeCount, nodeManager.getNodeStats().size()); - assertEquals(capacity, nodeManager.getNodeStat(datanodeID).getCapacity()); - assertEquals(expectedScmUsed, - nodeManager.getNodeStat(datanodeID).getScmUsed()); - assertEquals(expectedRemaining, - nodeManager.getNodeStat(datanodeID).getRemaining()); + long nodeCapacity = nodeManager.getNodeStat(datanodeID).get() + .getCapacity().get(); + assertEquals(capacity, nodeCapacity); + + foundScmUsed = nodeManager.getNodeStat(datanodeID).get().getScmUsed() + .get(); + assertEquals(expectedScmUsed, foundScmUsed); + + foundRemaining = nodeManager.getNodeStat(datanodeID).get() + .getRemaining().get(); + assertEquals(expectedRemaining, foundRemaining); // Compare the result from // NodeManager#getNodeStats and NodeManager#getNodeStat SCMNodeStat stat1 = nodeManager.getNodeStats(). get(datanodeID.getDatanodeUuid()); - SCMNodeStat stat2 = nodeManager.getNodeStat(datanodeID); + SCMNodeStat stat2 = nodeManager.getNodeStat(datanodeID).get(); assertEquals(stat1, stat2); // Wait up to 4s so that the node becomes stale @@ -1031,11 +1042,17 @@ public class TestNodeManager { () -> nodeManager.getNodeCount(NodeManager.NODESTATE.STALE) == 1, 100, 4 * 1000); assertEquals(nodeCount, nodeManager.getNodeStats().size()); - assertEquals(capacity, nodeManager.getNodeStat(datanodeID).getCapacity()); - assertEquals(expectedScmUsed, - nodeManager.getNodeStat(datanodeID).getScmUsed()); - assertEquals(expectedRemaining, - nodeManager.getNodeStat(datanodeID).getRemaining()); + + foundCapacity = nodeManager.getNodeStat(datanodeID).get() + .getCapacity().get(); + assertEquals(capacity, foundCapacity); + foundScmUsed = nodeManager.getNodeStat(datanodeID).get() + .getScmUsed().get(); + assertEquals(expectedScmUsed, foundScmUsed); + + foundRemaining = nodeManager.getNodeStat(datanodeID).get(). + getRemaining().get(); + assertEquals(expectedRemaining, foundRemaining); // Wait up to 4 more seconds so the node becomes dead // Verify usage info should be updated. @@ -1044,11 +1061,16 @@ public class TestNodeManager { 4 * 1000); assertEquals(0, nodeManager.getNodeStats().size()); - assertEquals(0, nodeManager.getStats().getCapacity()); - assertEquals(0, nodeManager.getStats().getScmUsed()); - assertEquals(0, nodeManager.getStats().getRemaining()); + foundCapacity = nodeManager.getStats().getCapacity().get(); + assertEquals(0, foundCapacity); + + foundScmUsed = nodeManager.getStats().getScmUsed().get(); + assertEquals(0, foundScmUsed); + + foundRemaining = nodeManager.getStats().getRemaining().get(); + assertEquals(0, foundRemaining); - // Send a new report to bring the dead node back to healty + // Send a new report to bring the dead node back to healthy SCMNodeReport.Builder nrb = SCMNodeReport.newBuilder(); SCMStorageReport.Builder srb = SCMStorageReport.newBuilder(); srb.setStorageUuid(UUID.randomUUID().toString()); @@ -1063,14 +1085,18 @@ public class TestNodeManager { () -> nodeManager.getNodeCount(NodeManager.NODESTATE.HEALTHY) == 1, 100, 5 * 1000); GenericTestUtils.waitFor( - () -> nodeManager.getStats().getScmUsed() == expectedScmUsed, 100, - 4 * 1000); + () -> nodeManager.getStats().getScmUsed().get() == expectedScmUsed, + 100, 4 * 1000); assertEquals(nodeCount, nodeManager.getNodeStats().size()); - assertEquals(capacity, nodeManager.getNodeStat(datanodeID).getCapacity()); - assertEquals(expectedScmUsed, - nodeManager.getNodeStat(datanodeID).getScmUsed()); - assertEquals(expectedRemaining, - nodeManager.getNodeStat(datanodeID).getRemaining()); + foundCapacity = nodeManager.getNodeStat(datanodeID).get() + .getCapacity().get(); + assertEquals(capacity, foundCapacity); + foundScmUsed = nodeManager.getNodeStat(datanodeID).get().getScmUsed() + .get(); + assertEquals(expectedScmUsed, foundScmUsed); + foundRemaining = nodeManager.getNodeStat(datanodeID).get() + .getRemaining().get(); + assertEquals(expectedRemaining, foundRemaining); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/68eab679/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodePoolManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodePoolManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodePoolManager.java index 3655544..2fd2c83 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodePoolManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodePoolManager.java @@ -24,8 +24,8 @@ import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConfiguration; import org.apache.hadoop.ozone.container.common.SCMTestUtils; -import org.apache.hadoop.ozone.scm.container.ContainerPlacementPolicy; -import org.apache.hadoop.ozone.scm.container.SCMContainerPlacementCapacity; +import org.apache.hadoop.ozone.scm.container.placement.algorithms.ContainerPlacementPolicy; +import org.apache.hadoop.ozone.scm.container.placement.algorithms.SCMContainerPlacementCapacity; import org.apache.hadoop.scm.ScmConfigKeys; import org.apache.hadoop.test.PathUtils; import org.junit.Rule; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
