Repository: zookeeper Updated Branches: refs/heads/branch-3.4 e5b4b94de -> fbcc83c15
ZOOKEEPER-3042: testFailedTxnAsPartOfQuorumLoss is flaky - relaxed check of outstanding proposals queue - close clients after restart - restart client after old leader restart Author: Bogdan Kanivets <[email protected]> Reviewers: [email protected] Closes #570 from lavacat/testFailedTxnAsPartOfQuorumLoss-fix-3.4 Project: http://git-wip-us.apache.org/repos/asf/zookeeper/repo Commit: http://git-wip-us.apache.org/repos/asf/zookeeper/commit/fbcc83c1 Tree: http://git-wip-us.apache.org/repos/asf/zookeeper/tree/fbcc83c1 Diff: http://git-wip-us.apache.org/repos/asf/zookeeper/diff/fbcc83c1 Branch: refs/heads/branch-3.4 Commit: fbcc83c15e05ea744fb48bde63a15793cb212575 Parents: e5b4b94 Author: Bogdan Kanivets <[email protected]> Authored: Tue Jul 17 10:40:01 2018 +0200 Committer: Andor Molnar <[email protected]> Committed: Tue Jul 17 10:40:01 2018 +0200 ---------------------------------------------------------------------- .../server/quorum/QuorumPeerMainTest.java | 35 +++++++++++++++++--- 1 file changed, 30 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zookeeper/blob/fbcc83c1/src/java/test/org/apache/zookeeper/server/quorum/QuorumPeerMainTest.java ---------------------------------------------------------------------- diff --git a/src/java/test/org/apache/zookeeper/server/quorum/QuorumPeerMainTest.java b/src/java/test/org/apache/zookeeper/server/quorum/QuorumPeerMainTest.java index 0099c93..714525c 100644 --- a/src/java/test/org/apache/zookeeper/server/quorum/QuorumPeerMainTest.java +++ b/src/java/test/org/apache/zookeeper/server/quorum/QuorumPeerMainTest.java @@ -444,6 +444,7 @@ public class QuorumPeerMainTest extends QuorumPeerTestBase { boolean someoneNotConnected = true; while (someoneNotConnected) { if (iterations-- == 0) { + logStates(zks); ClientBase.logAllStackTraces(); throw new RuntimeException("Waiting too long"); } @@ -459,6 +460,14 @@ public class QuorumPeerMainTest extends QuorumPeerTestBase { } } + private void logStates(ZooKeeper[] zks) { + StringBuilder sbBuilder = new StringBuilder("Connection States: {"); + for (int i = 0; i < zks.length; i++) { + sbBuilder.append(i + " : " + zks[i].getState() + ", "); + } + sbBuilder.append('}'); + LOG.error(sbBuilder.toString()); + } // This class holds the servers and clients for those servers private class Servers { MainThread mt[]; @@ -471,7 +480,7 @@ public class QuorumPeerMainTest extends QuorumPeerTestBase { } } - public void restartAllServersAndClients(Watcher watcher) throws IOException { + public void restartAllServersAndClients(Watcher watcher) throws IOException, InterruptedException { for (MainThread t : mt) { if (!t.isAlive()) { t.start(); @@ -482,7 +491,10 @@ public class QuorumPeerMainTest extends QuorumPeerTestBase { } } - public void restartClient(int clientIndex, Watcher watcher) throws IOException { + public void restartClient(int clientIndex, Watcher watcher) throws IOException, InterruptedException { + if (zk[clientIndex] != null) { + zk[clientIndex].close(); + } zk[clientIndex] = new ZooKeeper("127.0.0.1:" + clientPorts[clientIndex], ClientBase.CONNECTION_TIMEOUT, watcher); } @@ -1199,9 +1211,11 @@ public class QuorumPeerMainTest extends QuorumPeerTestBase { // just make sure that we actually did get it in process at the // leader - Assert.assertEquals(1, outstanding.size()); - Proposal p = outstanding.values().iterator().next(); - Assert.assertEquals(OpCode.create, p.request.hdr.getType()); + // there can be extra sessionClose proposals + Assert.assertTrue(outstanding.size() > 0); + Proposal p = findProposalOfType(outstanding, OpCode.create); + LOG.info(String.format("Old leader id: %d. All proposals: %s", leader, outstanding)); + Assert.assertNotNull("Old leader doesn't have 'create' proposal", p); // make sure it has a chance to write it to disk int sleepTime = 0; @@ -1235,6 +1249,8 @@ public class QuorumPeerMainTest extends QuorumPeerTestBase { // 7. restart the previous leader to force it to replay the edits and possibly come up in a bad state servers.mt[leader].shutdown(); servers.mt[leader].start(); + // old client session can expire, restart it + servers.restartClient(leader, this); waitForAll(servers, States.CONNECTED); // 8. check the node exist in previous leader but not others @@ -1358,4 +1374,13 @@ public class QuorumPeerMainTest extends QuorumPeerTestBase { } } } + + private Proposal findProposalOfType(Map<Long, Proposal> proposals, int type) { + for (Proposal proposal : proposals.values()) { + if (proposal.request.hdr.getType() == type) { + return proposal; + } + } + return null; + } }
