This is an automated email from the ASF dual-hosted git repository. avijayan pushed a commit to branch HDDS-3698-nonrolling-upgrade in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 1686e565bd06fc2c8ce96a800fe1f2f5f8aa51af Author: Ethan Rose <[email protected]> AuthorDate: Mon May 3 20:34:05 2021 -0400 Fix compilation errors afte merge Update javassist in recon pom Fix changes introduced in merge that failed TestSCMNodeManager upgrade tests Fix checkstyle Fix intermittent test failure TestSCMNodeManager#testSetNodeOpStateAndCommandFired after merge Skip scm init default layout version in TestOzoneConfigurationFields --- .../org/apache/hadoop/hdds/scm/ha/SCMContext.java | 5 +--- .../scm/server/upgrade/SCMUpgradeFinalizer.java | 8 ++++--- .../hadoop/hdds/scm/node/TestSCMNodeManager.java | 28 ++++++++++++++++++---- .../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 23 ++++++++++-------- .../hadoop/ozone/TestOzoneConfigurationFields.java | 2 ++ hadoop-ozone/recon/pom.xml | 5 ++++ 6 files changed, 49 insertions(+), 22 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMContext.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMContext.java index 2d4941f..0f32417 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMContext.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMContext.java @@ -46,14 +46,11 @@ public final class SCMContext { */ public static final long INVALID_TERM = -1; - private static final SCMContext EMPTY_CONTEXT - = new SCMContext.Builder().build(); - /** * Used by non-HA mode SCM, Recon and Unit Tests. */ public static SCMContext emptyContext() { - return EMPTY_CONTEXT; + return new SCMContext.Builder().build(); } /** diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizer.java index 39ad1a4..d674ed5 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizer.java @@ -22,6 +22,7 @@ import static org.apache.hadoop.hdds.scm.pipeline.Pipeline.PipelineState.CLOSED; import java.io.IOException; +import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; @@ -101,9 +102,10 @@ public class SCMUpgradeFinalizer extends // finalization, so clients can write. boolean hasPipeline = false; while (!hasPipeline) { - int pipelineCount = pipelineManager.getPipelines( - HddsProtos.ReplicationType.RATIS, HddsProtos.ReplicationFactor.THREE, - Pipeline.PipelineState.OPEN).size(); + ReplicationConfig ratisThree = + ReplicationConfig.fromTypeAndFactor(HddsProtos.ReplicationType.RATIS, + HddsProtos.ReplicationFactor.THREE); + int pipelineCount = pipelineManager.getPipelines(ratisThree).size(); hasPipeline = (pipelineCount >= 1); if (!hasPipeline) { diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java index 12f453a..ba815cf 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java @@ -34,6 +34,7 @@ import java.util.concurrent.TimeoutException; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.DFSConfigKeysLegacy; import org.apache.hadoop.hdds.HddsConfigKeys; +import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.protocol.MockDatanodeDetails; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -93,6 +94,7 @@ import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.STALE; import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type.finalizeNewLayoutVersionCommand; import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMRegisteredResponseProto.ErrorCode.errorNodeNotPermitted; import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMRegisteredResponseProto.ErrorCode.success; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_AUTO_CREATE_FACTOR_ONE; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_RATIS_PIPELINE_LIMIT; import static org.apache.hadoop.hdds.scm.TestUtils.getRandomPipelineReports; import static org.apache.hadoop.hdds.scm.events.SCMEvents.*; @@ -384,9 +386,10 @@ public class TestSCMNodeManager { private void assertPipelineCreationFailsWithNotEnoughNodes( int actualNodeCount) throws Exception { try { - scm.getPipelineManager() - .createPipeline(HddsProtos.ReplicationType.RATIS, + ReplicationConfig ratisThree = + ReplicationConfig.fromTypeAndFactor(HddsProtos.ReplicationType.RATIS, HddsProtos.ReplicationFactor.THREE); + scm.getPipelineManager().createPipeline(ratisThree); Assert.fail("3 nodes should not have been found for a pipeline."); } catch (SCMException ex) { Assert.assertTrue(ex.getMessage().contains("Required 3. Found " + @@ -404,9 +407,13 @@ public class TestSCMNodeManager { LambdaTestUtils.await(10000, 1000, () -> { + ReplicationConfig replConfig = + ReplicationConfig.fromTypeAndFactor(HddsProtos.ReplicationType.RATIS, + factor); + // Make sure that none of these pipelines use nodes outside of allowedDNs. List<Pipeline> pipelines = scm.getPipelineManager() - .getPipelines(HddsProtos.ReplicationType.RATIS, factor); + .getPipelines(replConfig); for (Pipeline pipeline: pipelines) { for(DatanodeDetails pipelineDN: pipeline.getNodes()) { @@ -556,10 +563,21 @@ public class TestSCMNodeManager { OzoneConfiguration conf = getConf(); conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, interval, MILLISECONDS); + // If factor 1 pipelines are auto created, registering the new node will + // trigger a pipeline creation command which may interfere with command + // checking in this test. + conf.setBoolean(OZONE_SCM_PIPELINE_AUTO_CREATE_FACTOR_ONE, false); try (SCMNodeManager nodeManager = createNodeManager(conf)) { DatanodeDetails dn = TestUtils.createRandomDatanodeAndRegister( nodeManager); + + LayoutVersionManager versionManager = + nodeManager.getLayoutVersionManager(); + final LayoutVersionProto layoutInfo = toLayoutVersionProto( + versionManager.getMetadataLayoutVersion(), + versionManager.getSoftwareLayoutVersion()); + long expiry = System.currentTimeMillis() / 1000 + 1000; nodeManager.setNodeOperationalState(dn, HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, expiry); @@ -567,7 +585,7 @@ public class TestSCMNodeManager { // If found mismatch, leader SCM fires a SetNodeOperationalStateCommand // to update the opState persisted in Datanode. scm.getScmContext().updateLeaderAndTerm(true, 1); - List<SCMCommand> commands = nodeManager.processHeartbeat(dn); + List<SCMCommand> commands = nodeManager.processHeartbeat(dn, layoutInfo); Assert.assertTrue(commands.get(0).getClass().equals( SetNodeOperationalStateCommand.class)); @@ -576,7 +594,7 @@ public class TestSCMNodeManager { // If found mismatch, follower SCM update its own opState according // to the heartbeat, and no SCMCommand will be fired. scm.getScmContext().updateLeaderAndTerm(false, 2); - commands = nodeManager.processHeartbeat(dn); + commands = nodeManager.processHeartbeat(dn, layoutInfo); assertEquals(0, commands.size()); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSUpgrade.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSUpgrade.java index 1d5a511..26993cc 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSUpgrade.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSUpgrade.java @@ -25,8 +25,6 @@ import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Con import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.QUASI_CLOSED; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY_READONLY; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.RATIS; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT; import static org.apache.hadoop.hdds.scm.pipeline.Pipeline.PipelineState.OPEN; import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.INITIAL_VERSION; @@ -50,6 +48,7 @@ import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -116,6 +115,10 @@ public class TestHDDSUpgrade { private HDDSLayoutVersionManager scmVersionManager; private AtomicBoolean testPassed = new AtomicBoolean(true); + private static final ReplicationConfig RATIS_THREE = + ReplicationConfig.fromTypeAndFactor(HddsProtos.ReplicationType.RATIS, + HddsProtos.ReplicationFactor.THREE); + /** * Create a MiniDFSCluster for testing. * @@ -212,7 +215,7 @@ public class TestHDDSUpgrade { // pipeline to use. try { GenericTestUtils.waitFor(() -> { - int pipelineCount = scmPipelineManager.getPipelines(RATIS, THREE, OPEN) + int pipelineCount = scmPipelineManager.getPipelines(RATIS_THREE, OPEN) .size(); if (pipelineCount >= 1) { return true; @@ -222,7 +225,7 @@ public class TestHDDSUpgrade { } catch (TimeoutException | InterruptedException e) { Assert.fail("Timeout waiting for Upgrade to complete on SCM."); } - int pipelineCount = scmPipelineManager.getPipelines(RATIS, THREE, OPEN) + int pipelineCount = scmPipelineManager.getPipelines(RATIS_THREE, OPEN) .size(); Assert.assertTrue(pipelineCount >= 1); @@ -321,11 +324,11 @@ public class TestHDDSUpgrade { * Helper function to test that we can create new pipelines Post-Upgrade. */ private void testPostUpgradePipelineCreation() throws IOException { - Pipeline ratisPipeline1 = scmPipelineManager.createPipeline(RATIS, THREE); + Pipeline ratisPipeline1 = scmPipelineManager.createPipeline(RATIS_THREE); scmPipelineManager.openPipeline(ratisPipeline1.getId()); Assert.assertEquals(0, scmPipelineManager.getNumberOfContainers(ratisPipeline1.getId())); - PipelineID pid = scmContainerManager.allocateContainer(RATIS, THREE, + PipelineID pid = scmContainerManager.allocateContainer(RATIS_THREE, "Owner1").getPipelineID(); Assert.assertEquals(1, scmPipelineManager.getNumberOfContainers(pid)); Assert.assertEquals(pid, ratisPipeline1.getId()); @@ -362,7 +365,7 @@ public class TestHDDSUpgrade { private void waitForPipelineCreated() throws Exception { LambdaTestUtils.await(10000, 500, () -> { List<Pipeline> pipelines = - scmPipelineManager.getPipelines(RATIS, THREE, OPEN); + scmPipelineManager.getPipelines(RATIS_THREE, OPEN); return pipelines.size() == 1; }); } @@ -373,7 +376,7 @@ public class TestHDDSUpgrade { private void createTestContainers() throws IOException { XceiverClientManager xceiverClientManager = new XceiverClientManager(conf); ContainerInfo ci1 = scmContainerManager.allocateContainer( - RATIS, THREE, "Owner1"); + RATIS_THREE, "Owner1"); Pipeline ratisPipeline1 = scmPipelineManager.getPipeline(ci1.getPipelineID()); scmPipelineManager.openPipeline(ratisPipeline1.getId()); @@ -399,7 +402,7 @@ public class TestHDDSUpgrade { testPreUpgradeConditionsDataNodes(); Set<PipelineID> preUpgradeOpenPipelines = - scmPipelineManager.getPipelines(RATIS, THREE, OPEN) + scmPipelineManager.getPipelines(RATIS_THREE, OPEN) .stream() .map(Pipeline::getId) .collect(Collectors.toSet()); @@ -414,7 +417,7 @@ public class TestHDDSUpgrade { } Set<PipelineID> postUpgradeOpenPipelines = - scmPipelineManager.getPipelines(RATIS, THREE, OPEN) + scmPipelineManager.getPipelines(RATIS_THREE, OPEN) .stream() .map(Pipeline::getId) .collect(Collectors.toSet()); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java index 508725d..c78b746 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java @@ -61,6 +61,8 @@ public class TestOzoneConfigurationFields extends TestConfigurationFieldsBase { xmlPrefixToSkipCompare.add("ipc.client.rpc-timeout.ms"); xmlPropsToSkipCompare.add("ozone.om.leader.election.minimum.timeout" + ".duration"); // Deprecated config + configurationPropsToSkipCompare + .add(ScmConfig.ConfigStrings.HDDS_SCM_INIT_DEFAULT_LAYOUT_VERSION); addPropertiesNotInXml(); } diff --git a/hadoop-ozone/recon/pom.xml b/hadoop-ozone/recon/pom.xml index 3aca1fc..11a4ecd 100644 --- a/hadoop-ozone/recon/pom.xml +++ b/hadoop-ozone/recon/pom.xml @@ -350,5 +350,10 @@ <artifactId>spotbugs</artifactId> <scope>provided</scope> </dependency> + <dependency> + <groupId>org.javassist</groupId> + <artifactId>javassist</artifactId> + <version>3.26.0-GA</version> + </dependency> </dependencies> </project> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
