This is an automated email from the ASF dual-hosted git repository. jxue pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/helix.git
commit 7a96d9ba8c7dd671f6847bb8f01f50dbf41d7ca9 Author: xyuanlu <[email protected]> AuthorDate: Mon Aug 7 09:56:18 2023 -0700 Add stress test for Metaclient leader election (#2574) --- .../zk/TestConnectStateChangeListenerAndRetry.java | 11 ++- .../metaclient/impl/zk/ZkMetaClientTestBase.java | 2 + .../helix/metaclient/puppy/AbstractPuppy.java | 2 +- .../leaderelection/LeaderElectionPuppy.java | 87 ++++++++++++++++++++++ .../recipes/leaderelection/TestLeaderElection.java | 55 ++++++++------ .../TestMultiClientLeaderElection.java | 82 ++++++++++++++++++++ 6 files changed, 214 insertions(+), 25 deletions(-) diff --git a/meta-client/src/test/java/org/apache/helix/metaclient/impl/zk/TestConnectStateChangeListenerAndRetry.java b/meta-client/src/test/java/org/apache/helix/metaclient/impl/zk/TestConnectStateChangeListenerAndRetry.java index c74b7d7ef..086db51c7 100644 --- a/meta-client/src/test/java/org/apache/helix/metaclient/impl/zk/TestConnectStateChangeListenerAndRetry.java +++ b/meta-client/src/test/java/org/apache/helix/metaclient/impl/zk/TestConnectStateChangeListenerAndRetry.java @@ -19,6 +19,8 @@ package org.apache.helix.metaclient.impl.zk; * under the License. */ +import java.io.File; +import java.io.IOException; import java.util.Date; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; @@ -27,17 +29,21 @@ import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import org.apache.commons.io.FileUtils; import org.apache.helix.metaclient.api.ConnectStateChangeListener; import org.apache.helix.metaclient.api.MetaClientInterface; import org.apache.helix.metaclient.impl.zk.factory.ZkMetaClientConfig; import org.apache.helix.metaclient.policy.ExponentialBackoffReconnectPolicy; +import org.apache.helix.zookeeper.zkclient.IDefaultNameSpace; import org.apache.helix.zookeeper.zkclient.ZkClient; import org.apache.helix.zookeeper.zkclient.ZkServer; import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.Watcher; import org.testng.Assert; import org.testng.annotations.AfterSuite; +import org.testng.annotations.AfterTest; import org.testng.annotations.BeforeSuite; +import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; import static org.apache.helix.metaclient.constants.MetaClientConstants.DEFAULT_INIT_EXP_BACKOFF_RETRY_INTERVAL_MS; @@ -69,14 +75,14 @@ public class TestConnectStateChangeListenerAndRetry { zkClient.process(event); } - @BeforeSuite + @BeforeTest public void prepare() { System.out.println("START TestConnectStateChangeListenerAndRetry at " + new Date(System.currentTimeMillis())); // start local zookeeper server _zkServer = ZkMetaClientTestBase.startZkServer(ZK_ADDR); } - @AfterSuite + @AfterTest public void cleanUp() { System.out.println("END TestConnectStateChangeListenerAndRetry at " + new Date(System.currentTimeMillis())); } @@ -162,7 +168,6 @@ public class TestConnectStateChangeListenerAndRetry { zkMetaClient.create("/key", "value"); Assert.fail("Create call after close should throw IllegalStateException"); } catch (Exception ex) { - System.out.println("ex " + ex); Assert.assertTrue(ex instanceof IllegalStateException); } } diff --git a/meta-client/src/test/java/org/apache/helix/metaclient/impl/zk/ZkMetaClientTestBase.java b/meta-client/src/test/java/org/apache/helix/metaclient/impl/zk/ZkMetaClientTestBase.java index 2a5f4b97b..eade017c1 100644 --- a/meta-client/src/test/java/org/apache/helix/metaclient/impl/zk/ZkMetaClientTestBase.java +++ b/meta-client/src/test/java/org/apache/helix/metaclient/impl/zk/ZkMetaClientTestBase.java @@ -49,6 +49,7 @@ public abstract class ZkMetaClientTestBase { */ @BeforeSuite public void prepare() { + System.out.println("ZkMetaClientTestBase start "); // Enable extended types and create a ZkClient System.setProperty("zookeeper.extendedTypesEnabled", "true"); // start local zookeeper server @@ -57,6 +58,7 @@ public abstract class ZkMetaClientTestBase { @AfterSuite public void cleanUp() { + System.out.println("ZkMetaClientTestBase shut down"); _zkServer.shutdown(); } diff --git a/meta-client/src/test/java/org/apache/helix/metaclient/puppy/AbstractPuppy.java b/meta-client/src/test/java/org/apache/helix/metaclient/puppy/AbstractPuppy.java index 9ce21fc15..b16f78615 100644 --- a/meta-client/src/test/java/org/apache/helix/metaclient/puppy/AbstractPuppy.java +++ b/meta-client/src/test/java/org/apache/helix/metaclient/puppy/AbstractPuppy.java @@ -22,6 +22,7 @@ package org.apache.helix.metaclient.puppy; import org.apache.helix.metaclient.api.MetaClientInterface; import java.util.HashMap; + /** * AbstractPuppy object contains interfaces to implement puppy and main logics to manage puppy life cycle */ @@ -32,7 +33,6 @@ public abstract class AbstractPuppy implements Runnable { public final HashMap<String, Integer> _eventChangeCounterMap; public int _unhandledErrorCounter; - public AbstractPuppy(MetaClientInterface<String> metaclient, PuppySpec puppySpec) { _metaclient = metaclient; _puppySpec = puppySpec; diff --git a/meta-client/src/test/java/org/apache/helix/metaclient/recipes/leaderelection/LeaderElectionPuppy.java b/meta-client/src/test/java/org/apache/helix/metaclient/recipes/leaderelection/LeaderElectionPuppy.java new file mode 100644 index 000000000..5f1fdf631 --- /dev/null +++ b/meta-client/src/test/java/org/apache/helix/metaclient/recipes/leaderelection/LeaderElectionPuppy.java @@ -0,0 +1,87 @@ +package org.apache.helix.metaclient.recipes.leaderelection; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.util.Random; +import java.util.concurrent.TimeUnit; +import org.apache.helix.metaclient.MetaClientTestUtil; +import org.apache.helix.metaclient.api.MetaClientInterface; +import org.apache.helix.metaclient.exception.MetaClientException; +import org.apache.helix.metaclient.puppy.AbstractPuppy; +import org.apache.helix.metaclient.puppy.PuppySpec; +import org.testng.Assert; + + +public class LeaderElectionPuppy extends AbstractPuppy { + String _leaderGroup; + String _participant; + private final Random _random; + LeaderElectionClient _leaderElectionClient; + + public LeaderElectionPuppy(MetaClientInterface<String> metaclient, PuppySpec puppySpec) { + super(metaclient, puppySpec); + _random = new Random(); + } + + public LeaderElectionPuppy(LeaderElectionClient leaderElectionClient, PuppySpec puppySpec, String leaderGroup, + String participant) { + super(leaderElectionClient.getMetaClient(), puppySpec); + _leaderElectionClient = leaderElectionClient; + _leaderGroup = leaderGroup; + _random = new Random(); + _participant = participant; + } + + @Override + protected void bark() throws Exception { + int randomNumber = _random.nextInt((int) TimeUnit.SECONDS.toMillis(5)); + System.out.println("LeaderElectionPuppy " + _participant + " Joining"); + _leaderElectionClient.joinLeaderElectionParticipantPool(_leaderGroup); + + Assert.assertTrue(MetaClientTestUtil.verify(() -> { + return (_leaderElectionClient.getLeader(_leaderGroup) != null); + }, MetaClientTestUtil.WAIT_DURATION)); + if (_random.nextBoolean()) { + _leaderElectionClient.relinquishLeader(_leaderGroup); + } + Assert.assertTrue(MetaClientTestUtil.verify(() -> { + return (_leaderElectionClient.getParticipantInfo(_leaderGroup, _participant) != null); + }, MetaClientTestUtil.WAIT_DURATION)); + + Thread.sleep(randomNumber); + System.out.println("LeaderElectionPuppy " + _participant + " Leaving"); + _leaderElectionClient.exitLeaderElectionParticipantPool(_leaderGroup); + Assert.assertTrue(MetaClientTestUtil.verify(() -> { + return (_leaderElectionClient.getParticipantInfo(_leaderGroup, _participant) == null); + }, MetaClientTestUtil.WAIT_DURATION)); + + Thread.sleep(randomNumber); + } + + @Override + protected void cleanup() { + try { + System.out.println("Cleaning - LeaderElectionPuppy " + _participant + " Leaving"); + _leaderElectionClient.exitLeaderElectionParticipantPool(_leaderGroup); + } catch (MetaClientException ignore) { + // already leave the pool. OK to throw exception. + } + } +} diff --git a/meta-client/src/test/java/org/apache/helix/metaclient/recipes/leaderelection/TestLeaderElection.java b/meta-client/src/test/java/org/apache/helix/metaclient/recipes/leaderelection/TestLeaderElection.java index b0b396c1a..71d85fdb9 100644 --- a/meta-client/src/test/java/org/apache/helix/metaclient/recipes/leaderelection/TestLeaderElection.java +++ b/meta-client/src/test/java/org/apache/helix/metaclient/recipes/leaderelection/TestLeaderElection.java @@ -19,7 +19,7 @@ public class TestLeaderElection extends ZkMetaClientTestBase { private static final String PARTICIPANT_NAME2 = "participant_2"; private static final String LEADER_PATH = "/LEADER_ELECTION_GROUP_1"; - public LeaderElectionClient createLeaderElectionClient(String participantName) { + public static LeaderElectionClient createLeaderElectionClient(String participantName) { MetaClientConfig.StoreType storeType = MetaClientConfig.StoreType.ZOOKEEPER; MetaClientConfig config = new MetaClientConfig.MetaClientConfigBuilder<>().setConnectionAddress(ZK_ADDR).setStoreType(storeType).build(); @@ -28,50 +28,57 @@ public class TestLeaderElection extends ZkMetaClientTestBase { @Test public void testAcquireLeadership() throws Exception { - String leaderPath = LEADER_PATH + "testAcquireLeadership"; + System.out.println("START TestLeaderElection.testAcquireLeadership"); + String leaderPath = LEADER_PATH + "/testAcquireLeadership"; // create 2 clients representing 2 participants LeaderElectionClient clt1 = createLeaderElectionClient(PARTICIPANT_NAME1); LeaderElectionClient clt2 = createLeaderElectionClient(PARTICIPANT_NAME2); + clt1.getMetaClient().create(LEADER_PATH, new LeaderInfo(LEADER_PATH)); + clt1.joinLeaderElectionParticipantPool(leaderPath); clt2.joinLeaderElectionParticipantPool(leaderPath); // First client joining the leader election group should be current leader Assert.assertTrue(MetaClientTestUtil.verify(() -> { - return (clt1.getLeader(LEADER_PATH) != null); + return (clt1.getLeader(leaderPath) != null); }, MetaClientTestUtil.WAIT_DURATION)); - Assert.assertNotNull(clt1.getLeader(LEADER_PATH)); - Assert.assertEquals(clt1.getLeader(LEADER_PATH), clt2.getLeader(LEADER_PATH)); - Assert.assertEquals(clt1.getLeader(LEADER_PATH), PARTICIPANT_NAME1); + Assert.assertNotNull(clt1.getLeader(leaderPath)); + Assert.assertEquals(clt1.getLeader(leaderPath), clt2.getLeader(leaderPath)); + Assert.assertEquals(clt1.getLeader(leaderPath), PARTICIPANT_NAME1); + // client 1 exit leader election group, and client 2 should be current leader. - clt1.exitLeaderElectionParticipantPool(LEADER_PATH); + clt1.exitLeaderElectionParticipantPool(leaderPath); + Assert.assertTrue(MetaClientTestUtil.verify(() -> { - return (clt1.getLeader(LEADER_PATH) != null); + return (clt1.getLeader(leaderPath) != null); }, MetaClientTestUtil.WAIT_DURATION)); Assert.assertTrue(MetaClientTestUtil.verify(() -> { - return (clt1.getLeader(LEADER_PATH).equals(PARTICIPANT_NAME2)); + return (clt1.getLeader(leaderPath).equals(PARTICIPANT_NAME2)); }, MetaClientTestUtil.WAIT_DURATION)); // client1 join and client2 leave. client 1 should be leader. - clt1.joinLeaderElectionParticipantPool(LEADER_PATH); - clt2.exitLeaderElectionParticipantPool(LEADER_PATH); + clt1.joinLeaderElectionParticipantPool(leaderPath); + clt2.exitLeaderElectionParticipantPool(leaderPath); Assert.assertTrue(MetaClientTestUtil.verify(() -> { - return (clt1.getLeader(LEADER_PATH) != null); + return (clt1.getLeader(leaderPath) != null); }, MetaClientTestUtil.WAIT_DURATION)); Assert.assertTrue(MetaClientTestUtil.verify(() -> { - return (clt1.getLeader(LEADER_PATH).equals(PARTICIPANT_NAME1)); + return (clt1.getLeader(leaderPath).equals(PARTICIPANT_NAME1)); }, MetaClientTestUtil.WAIT_DURATION)); - Assert.assertTrue(clt1.isLeader(LEADER_PATH)); - Assert.assertFalse(clt2.isLeader(LEADER_PATH)); + Assert.assertTrue(clt1.isLeader(leaderPath)); + Assert.assertFalse(clt2.isLeader(leaderPath)); clt1.close(); clt2.close(); + System.out.println("END TestLeaderElection.testAcquireLeadership"); } @Test public void testElectionPoolMembership() throws Exception { - String leaderPath = LEADER_PATH + "/testElectionPoolMembership"; + System.out.println("START TestLeaderElection.testElectionPoolMembership"); + String leaderPath = LEADER_PATH + "/_testElectionPoolMembership"; LeaderInfo participantInfo = new LeaderInfo(PARTICIPANT_NAME1); participantInfo.setSimpleField("Key1", "value1"); LeaderInfo participantInfo2 = new LeaderInfo(PARTICIPANT_NAME2); @@ -103,12 +110,14 @@ public class TestLeaderElection extends ZkMetaClientTestBase { clt1.exitLeaderElectionParticipantPool(leaderPath); clt2.exitLeaderElectionParticipantPool(leaderPath); - Assert.assertNull(clt2.getParticipantInfo(LEADER_PATH, PARTICIPANT_NAME2)); + Assert.assertNull(clt2.getParticipantInfo(leaderPath, PARTICIPANT_NAME2)); + System.out.println("END TestLeaderElection.testElectionPoolMembership"); } @Test public void testSessionExpire() throws Exception { - String leaderPath = LEADER_PATH + "/testSessionExpire"; + System.out.println("START TestLeaderElection.testSessionExpire"); + String leaderPath = LEADER_PATH + "/_testSessionExpire"; LeaderInfo participantInfo = new LeaderInfo(PARTICIPANT_NAME1); participantInfo.setSimpleField("Key1", "value1"); LeaderInfo participantInfo2 = new LeaderInfo(PARTICIPANT_NAME2); @@ -142,10 +151,12 @@ public class TestLeaderElection extends ZkMetaClientTestBase { Assert.assertEquals(clt2.getParticipantInfo(leaderPath, PARTICIPANT_NAME1).getSimpleField("Key1"), "value1"); Assert.assertEquals(clt1.getParticipantInfo(leaderPath, PARTICIPANT_NAME2).getSimpleField("Key2"), "value2"); Assert.assertEquals(clt2.getParticipantInfo(leaderPath, PARTICIPANT_NAME2).getSimpleField("Key2"), "value2"); + System.out.println("END TestLeaderElection.testSessionExpire"); } @Test (dependsOnMethods = "testAcquireLeadership") public void testLeadershipListener() throws Exception { - String leaderPath = LEADER_PATH + "testLeadershipListener"; + System.out.println("START TestLeaderElection.testLeadershipListener"); + String leaderPath = LEADER_PATH + "/testLeadershipListener"; // create 2 clients representing 2 participants LeaderElectionClient clt1 = createLeaderElectionClient(PARTICIPANT_NAME1); LeaderElectionClient clt2 = createLeaderElectionClient(PARTICIPANT_NAME2); @@ -189,7 +200,6 @@ public class TestLeaderElection extends ZkMetaClientTestBase { Assert.assertEquals(numNewLeaderEvent[0], count*2); clt3.unsubscribeLeadershipChanges(leaderPath, listener); - // listener shouldn't receive any event after unsubscribe joinPoolTestHelper(leaderPath, clt1, clt2); Assert.assertEquals(numLeaderGoneEvent[0], count*2); @@ -198,6 +208,7 @@ public class TestLeaderElection extends ZkMetaClientTestBase { clt1.close(); clt2.close(); clt3.close(); + System.out.println("END TestLeaderElection.testLeadershipListener"); } private void joinPoolTestHelper(String leaderPath, LeaderElectionClient clt1, LeaderElectionClient clt2) throws Exception { @@ -221,7 +232,8 @@ public class TestLeaderElection extends ZkMetaClientTestBase { @Test (dependsOnMethods = "testLeadershipListener") public void testRelinquishLeadership() throws Exception { - String leaderPath = LEADER_PATH + "testRelinquishLeadership"; + System.out.println("START TestLeaderElection.testRelinquishLeadership"); + String leaderPath = LEADER_PATH + "/testRelinquishLeadership"; LeaderElectionClient clt1 = createLeaderElectionClient(PARTICIPANT_NAME1); LeaderElectionClient clt2 = createLeaderElectionClient(PARTICIPANT_NAME2); LeaderElectionClient clt3 = createLeaderElectionClient(PARTICIPANT_NAME2); @@ -268,6 +280,7 @@ public class TestLeaderElection extends ZkMetaClientTestBase { }, MetaClientTestUtil.WAIT_DURATION)); clt2.exitLeaderElectionParticipantPool(leaderPath); + System.out.println("END TestLeaderElection.testRelinquishLeadership"); } } diff --git a/meta-client/src/test/java/org/apache/helix/metaclient/recipes/leaderelection/TestMultiClientLeaderElection.java b/meta-client/src/test/java/org/apache/helix/metaclient/recipes/leaderelection/TestMultiClientLeaderElection.java new file mode 100644 index 000000000..0fe424501 --- /dev/null +++ b/meta-client/src/test/java/org/apache/helix/metaclient/recipes/leaderelection/TestMultiClientLeaderElection.java @@ -0,0 +1,82 @@ +package org.apache.helix.metaclient.recipes.leaderelection; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.helix.metaclient.MetaClientTestUtil; +import org.apache.helix.metaclient.exception.MetaClientException; +import org.apache.helix.metaclient.impl.zk.TestMultiThreadStressTest.CreatePuppy; +import org.apache.helix.metaclient.impl.zk.ZkMetaClient; +import org.apache.helix.metaclient.impl.zk.ZkMetaClientTestBase; +import org.apache.helix.metaclient.puppy.ExecDelay; +import org.apache.helix.metaclient.puppy.PuppyManager; +import org.apache.helix.metaclient.puppy.PuppyMode; +import org.apache.helix.metaclient.puppy.PuppySpec; +import org.apache.helix.zookeeper.exception.ZkClientException; +import org.testng.annotations.AfterTest; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + +import static org.apache.helix.metaclient.impl.zk.ZkMetaClientTestBase.*; + + +public class TestMultiClientLeaderElection extends ZkMetaClientTestBase { + private final String _leaderElectionGroup = "/Parent/a/LEADER_ELECTION_GROUP"; + private ZkMetaClient<String> _zkMetaClient; + private final String _participant1 = "participant_1"; + private final String _participant2 = "participant_2"; + + @BeforeTest + private void setUp() { + System.out.println("STARTING TestMultiClientLeaderElection"); + this._zkMetaClient = createZkMetaClient(); + this._zkMetaClient.connect(); + _zkMetaClient.create("/Parent", ""); + _zkMetaClient.create("/Parent/a", ""); + } + @AfterTest + public void cleanUp() { + try { + _zkMetaClient.recursiveDelete(_leaderElectionGroup); + } catch (MetaClientException ex) { + _zkMetaClient.recursiveDelete(_leaderElectionGroup); + } + } + + @Test + public void testLeaderElectionPuppy() { + System.out.println("Starting TestMultiClientLeaderElection.testLeaderElectionPuppy"); + PuppySpec puppySpec = + new org.apache.helix.metaclient.puppy.PuppySpec(PuppyMode.REPEAT, 0.2f, new ExecDelay(5000, 0.1f), 5); + LeaderElectionPuppy leaderElectionPuppy1 = + new LeaderElectionPuppy(TestLeaderElection.createLeaderElectionClient(_participant1), + puppySpec, _leaderElectionGroup, _participant1); + LeaderElectionPuppy leaderElectionPuppy2 = + new LeaderElectionPuppy(TestLeaderElection.createLeaderElectionClient(_participant2), + puppySpec, _leaderElectionGroup, _participant2); + + PuppyManager puppyManager = new PuppyManager(); + puppyManager.addPuppy(leaderElectionPuppy1); + puppyManager.addPuppy(leaderElectionPuppy2); + + puppyManager.start(60); + System.out.println("Ending TestMultiClientLeaderElection.testLeaderElectionPuppy"); + + } +}
