Repository: zookeeper Updated Branches: refs/heads/master c5df1c9ac -> 201cac20e
ZOOKEEPER-2683: RaceConditionTest is flaky RaceConditionTest assertion is wrong, It fails when old leader becomes leader again. Author: Mohammad Arshad <[email protected]> Reviewers: Michael Han <[email protected]> Closes #175 from arshadmohammad/ZOOKEEPER-2683-03 Project: http://git-wip-us.apache.org/repos/asf/zookeeper/repo Commit: http://git-wip-us.apache.org/repos/asf/zookeeper/commit/201cac20 Tree: http://git-wip-us.apache.org/repos/asf/zookeeper/tree/201cac20 Diff: http://git-wip-us.apache.org/repos/asf/zookeeper/diff/201cac20 Branch: refs/heads/master Commit: 201cac20e1a66e7d4180a6c7e18834c0ca6094dc Parents: c5df1c9 Author: Mohammad Arshad <[email protected]> Authored: Mon Feb 13 13:09:59 2017 -0800 Committer: Michael Han <[email protected]> Committed: Mon Feb 13 13:09:59 2017 -0800 ---------------------------------------------------------------------- .../zookeeper/server/quorum/QuorumPeerMain.java | 7 ++- .../server/quorum/RaceConditionTest.java | 49 +++++++++----------- 2 files changed, 28 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zookeeper/blob/201cac20/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeerMain.java ---------------------------------------------------------------------- diff --git a/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeerMain.java b/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeerMain.java index bd49dbf..cde193e 100644 --- a/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeerMain.java +++ b/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeerMain.java @@ -154,7 +154,7 @@ public class QuorumPeerMain { true); } - quorumPeer = new QuorumPeer(); + quorumPeer = getQuorumPeer(); quorumPeer.setTxnFactory(new FileTxnSnapLog( config.getDataLogDir(), config.getDataDir())); @@ -189,4 +189,9 @@ public class QuorumPeerMain { LOG.warn("Quorum Peer interrupted", e); } } + + // @VisibleForTesting + protected QuorumPeer getQuorumPeer() { + return new QuorumPeer(); + } } http://git-wip-us.apache.org/repos/asf/zookeeper/blob/201cac20/src/java/test/org/apache/zookeeper/server/quorum/RaceConditionTest.java ---------------------------------------------------------------------- diff --git a/src/java/test/org/apache/zookeeper/server/quorum/RaceConditionTest.java b/src/java/test/org/apache/zookeeper/server/quorum/RaceConditionTest.java index 46ebcf2..c65a794 100644 --- a/src/java/test/org/apache/zookeeper/server/quorum/RaceConditionTest.java +++ b/src/java/test/org/apache/zookeeper/server/quorum/RaceConditionTest.java @@ -21,12 +21,9 @@ import static org.apache.zookeeper.test.ClientBase.CONNECTION_TIMEOUT; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; -import java.io.File; import java.io.IOException; -import java.net.InetSocketAddress; import java.net.SocketException; import java.nio.ByteBuffer; -import java.util.Map; import org.apache.zookeeper.PortAssignment; import org.apache.zookeeper.ZooDefs; @@ -34,13 +31,10 @@ import org.apache.zookeeper.server.FinalRequestProcessor; import org.apache.zookeeper.server.PrepRequestProcessor; import org.apache.zookeeper.server.Request; import org.apache.zookeeper.server.RequestProcessor; -import org.apache.zookeeper.server.ServerCnxnFactory; import org.apache.zookeeper.server.SyncRequestProcessor; import org.apache.zookeeper.server.ZooKeeperServer; -import org.apache.zookeeper.server.admin.AdminServer.AdminServerException; import org.apache.zookeeper.server.persistence.FileTxnSnapLog; import org.apache.zookeeper.server.quorum.QuorumPeer.ServerState; -import org.apache.zookeeper.server.quorum.flexible.QuorumMaj; import org.apache.zookeeper.test.ClientBase; import org.junit.After; import org.junit.Assert; @@ -67,12 +61,28 @@ public class RaceConditionTest extends QuorumPeerTestBase { mt = startQuorum(); // get leader QuorumPeer leader = getLeader(mt); + long oldLeaderCurrentEpoch = leader.getCurrentEpoch(); assertNotNull("Leader should not be null", leader); // shutdown 2 followers so that leader does not have majority and goes - // into looking state or following state. + // into looking state or following/leading state. shutdownFollowers(mt); - assertTrue("Leader failed to transition to LOOKING or FOLLOWING state", ClientBase.waitForServerState(leader, - 15000, QuorumStats.Provider.LOOKING_STATE, QuorumStats.Provider.FOLLOWING_STATE)); + /** + * <pre> + * Verify that there is no deadlock in following ways: + * 1) If leader is in LOOKING or FOLLOWING, we are sure there is no deadlock. + * 2) If leader in in LEADING state then we have to check that this LEADING state is + * after the leader election, not the old LEADING state. + * </pre> + */ + boolean leaderStateChanged = ClientBase.waitForServerState(leader, 15000, + QuorumStats.Provider.LOOKING_STATE, QuorumStats.Provider.FOLLOWING_STATE); + // Wait for the old leader to start completely + Assert.assertTrue("Failed to bring up the old leader server", ClientBase + .waitForServerUp("127.0.0.1:" + leader.getClientPort(), CONNECTION_TIMEOUT)); + assertTrue( + "Leader failed to transition to new state. Current state is " + + leader.getServerState(), + leaderStateChanged || (leader.getCurrentEpoch() > oldLeaderCurrentEpoch)); } @After @@ -150,13 +160,6 @@ public class RaceConditionTest extends QuorumPeerTestBase { this.stopPing = stopPing; } - public CustomQuorumPeer(Map<Long, QuorumServer> quorumPeers, File snapDir, File logDir, int clientPort, - int electionAlg, long myid, int tickTime, int initLimit, int syncLimit) - throws IOException { - super(quorumPeers, snapDir, logDir, electionAlg, myid, tickTime, initLimit, syncLimit, false, - ServerCnxnFactory.createFactory(new InetSocketAddress(clientPort), -1), new QuorumMaj(quorumPeers)); - } - @Override protected Follower makeFollower(FileTxnSnapLog logFactory) throws IOException { @@ -234,18 +237,10 @@ public class RaceConditionTest extends QuorumPeerTestBase { } private static class MockTestQPMain extends TestQPMain { + @Override - public void runFromConfig(QuorumPeerConfig config) - throws IOException, AdminServerException { - quorumPeer = new CustomQuorumPeer(config.getQuorumVerifier().getAllMembers(), config.getDataDir(), - config.getDataLogDir(), config.getClientPortAddress().getPort(), config.getElectionAlg(), - config.getServerId(), config.getTickTime(), config.getInitLimit(), config.getSyncLimit()); - quorumPeer.start(); - try { - quorumPeer.join(); - } catch (InterruptedException e) { - LOG.warn("Quorum Peer interrupted", e); - } + protected QuorumPeer getQuorumPeer() { + return new CustomQuorumPeer(); } } }
