Repository: hadoop Updated Branches: refs/heads/trunk fa2945e7a -> b3161c4dd
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3161c4d/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java new file mode 100644 index 0000000..7e3969c --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ + +package org.apache.hadoop.hdds.scm.pipeline; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; +import org.apache.hadoop.hdds.scm.container.ContainerMapping; +import org.apache.hadoop.hdds.scm.container.common.helpers + .ContainerWithPipeline; +import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; +import org.apache.hadoop.hdds.scm.server.StorageContainerManager; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos + .ReplicationFactor.THREE; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos + .ReplicationType.RATIS; + +/** + * Test Node failure detection and handling in Ratis. + */ +public class TestNodeFailure { + + private static MiniOzoneCluster cluster; + private static OzoneConfiguration conf; + private static ContainerWithPipeline ratisContainer1; + private static ContainerWithPipeline ratisContainer2; + private static ContainerMapping mapping; + private static long timeForFailure; + + /** + * Create a MiniDFSCluster for testing. + * + * @throws IOException + */ + @BeforeClass + public static void init() throws Exception { + conf = new OzoneConfiguration(); + conf.setTimeDuration(OzoneConfigKeys.DFS_RATIS_SERVER_FAILURE_DURATION_KEY, + 10, TimeUnit.SECONDS); + conf.setTimeDuration( + ScmConfigKeys.OZONE_SCM_CONTAINER_CREATION_LEASE_TIMEOUT, + 10, TimeUnit.SECONDS); + cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(6) + .setHbInterval(1000) + .setHbProcessorInterval(1000) + .build(); + cluster.waitForClusterToBeReady(); + StorageContainerManager scm = cluster.getStorageContainerManager(); + mapping = (ContainerMapping)scm.getScmContainerManager(); + ratisContainer1 = mapping.allocateContainer(RATIS, THREE, "testOwner"); + ratisContainer2 = mapping.allocateContainer(RATIS, THREE, "testOwner"); + // At this stage, there should be 2 pipeline one with 1 open container each. + // Try closing the both the pipelines, one with a closed container and + // the other with an open container. + timeForFailure = conf.getTimeDuration( + OzoneConfigKeys.DFS_RATIS_SERVER_FAILURE_DURATION_KEY, + OzoneConfigKeys.DFS_RATIS_SERVER_FAILURE_DURATION_DEFAULT + .getDuration(), TimeUnit.MILLISECONDS); + } + + /** + * Shutdown MiniDFSCluster. + */ + @AfterClass + public static void shutdown() { + if (cluster != null) { + cluster.shutdown(); + } + } + + @Test + public void testPipelineFail() throws InterruptedException, IOException, + TimeoutException { + Assert.assertEquals(ratisContainer1.getPipeline().getLifeCycleState(), + HddsProtos.LifeCycleState.OPEN); + Pipeline pipelineToFail = ratisContainer1.getPipeline(); + DatanodeDetails dnToFail = pipelineToFail.getMachines().get(0); + cluster.shutdownHddsDatanode(dnToFail); + + // wait for sufficient time for the callback to be triggered + Thread.sleep(3 * timeForFailure); + + Assert.assertEquals(HddsProtos.LifeCycleState.CLOSED, + ratisContainer1.getPipeline().getLifeCycleState()); + Assert.assertEquals(HddsProtos.LifeCycleState.OPEN, + ratisContainer2.getPipeline().getLifeCycleState()); + Assert.assertNull( + mapping.getPipelineSelector().getPipeline(pipelineToFail.getId())); + // Now restart the datanode and make sure that a new pipeline is created. + cluster.restartHddsDatanode(dnToFail); + ContainerWithPipeline ratisContainer3 = + mapping.allocateContainer(RATIS, THREE, "testOwner"); + //Assert that new container is not created from the ratis 2 pipeline + Assert.assertNotEquals(ratisContainer3.getPipeline().getId(), + ratisContainer2.getPipeline().getId()); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3161c4d/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java index eb15396..0f8f925 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java @@ -112,8 +112,7 @@ public class TestPipelineClose { pipelineSelector.finalizePipeline(ratisContainer1.getPipeline()); Pipeline pipeline1 = pipelineSelector - .getPipeline(ratisContainer1.getPipeline().getId(), - ratisContainer1.getContainerInfo().getReplicationType()); + .getPipeline(ratisContainer1.getPipeline().getId()); Assert.assertNull(pipeline1); Assert.assertEquals(ratisContainer1.getPipeline().getLifeCycleState(), HddsProtos.LifeCycleState.CLOSED); @@ -140,8 +139,7 @@ public class TestPipelineClose { Assert.assertEquals(ratisContainer2.getPipeline().getLifeCycleState(), HddsProtos.LifeCycleState.CLOSING); Pipeline pipeline2 = pipelineSelector - .getPipeline(ratisContainer2.getPipeline().getId(), - ratisContainer2.getContainerInfo().getReplicationType()); + .getPipeline(ratisContainer2.getPipeline().getId()); Assert.assertEquals(pipeline2.getLifeCycleState(), HddsProtos.LifeCycleState.CLOSING); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3161c4d/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java index ae6a91e..e11cf9b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java @@ -19,6 +19,7 @@ package org.apache.hadoop.ozone; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.om.OzoneManager; @@ -156,6 +157,13 @@ public interface MiniOzoneCluster { TimeoutException; /** + * Restart a particular HddsDatanode. + * + * @param dn HddsDatanode in the MiniOzoneCluster + */ + void restartHddsDatanode(DatanodeDetails dn) throws InterruptedException, + TimeoutException, IOException; + /** * Shutdown a particular HddsDatanode. * * @param i index of HddsDatanode in the MiniOzoneCluster @@ -163,6 +171,13 @@ public interface MiniOzoneCluster { void shutdownHddsDatanode(int i); /** + * Shutdown a particular HddsDatanode. + * + * @param dn HddsDatanode in the MiniOzoneCluster + */ + void shutdownHddsDatanode(DatanodeDetails dn) throws IOException; + + /** * Shutdown the MiniOzoneCluster. */ void shutdown(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3161c4d/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java index e06e2f6..7b9bb0e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java @@ -157,6 +157,16 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster { return hddsDatanodes; } + private int getHddsDatanodeIndex(DatanodeDetails dn) throws IOException { + for (HddsDatanodeService service : hddsDatanodes) { + if (service.getDatanodeDetails().equals(dn)) { + return hddsDatanodes.indexOf(service); + } + } + throw new IOException( + "Not able to find datanode with datanode Id " + dn.getUuid()); + } + @Override public OzoneClient getClient() throws IOException { return OzoneClientFactory.getClient(conf); @@ -243,11 +253,22 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster { } @Override + public void restartHddsDatanode(DatanodeDetails dn) + throws InterruptedException, TimeoutException, IOException { + restartHddsDatanode(getHddsDatanodeIndex(dn)); + } + + @Override public void shutdownHddsDatanode(int i) { hddsDatanodes.get(i).stop(); } @Override + public void shutdownHddsDatanode(DatanodeDetails dn) throws IOException { + shutdownHddsDatanode(getHddsDatanodeIndex(dn)); + } + + @Override public void shutdown() { try { LOG.info("Shutting down the Mini Ozone Cluster"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3161c4d/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java index 8b324b5..b53e683 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java @@ -156,7 +156,8 @@ public class TestCSMMetrics { conf.set(OzoneConfigKeys.DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR, dir); final ContainerDispatcher dispatcher = new TestContainerDispatcher(); - return XceiverServerRatis.newXceiverServerRatis(dn, conf, dispatcher); + return XceiverServerRatis.newXceiverServerRatis(dn, conf, dispatcher, + null); } static void initXceiverServerRatis( http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3161c4d/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java index ebcc930..3abc8f8 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java @@ -138,7 +138,8 @@ public class TestContainerServer { conf.set(OzoneConfigKeys.DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR, dir); final ContainerDispatcher dispatcher = new TestContainerDispatcher(); - return XceiverServerRatis.newXceiverServerRatis(dn, conf, dispatcher); + return XceiverServerRatis + .newXceiverServerRatis(dn, conf, dispatcher, null); } static void initXceiverServerRatis( http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3161c4d/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerStateMachine.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerStateMachine.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerStateMachine.java index 448742e..8c83fd3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerStateMachine.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerStateMachine.java @@ -69,7 +69,7 @@ public class TestContainerStateMachine { new ArrayBlockingQueue<>(1024), new ThreadPoolExecutor.CallerRunsPolicy()); private ContainerStateMachine stateMachine = - new ContainerStateMachine(new TestContainerDispatcher(), executor); + new ContainerStateMachine(new TestContainerDispatcher(), executor, null); @Test --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
