hanm commented on a change in pull request #1235: ZOOKEEPER-3706: ZooKeeper.close() would leak SendThread when the netw… URL: https://github.com/apache/zookeeper/pull/1235#discussion_r381084467
########## File path: zookeeper-server/src/test/java/org/apache/zookeeper/ClientCnxnSocketFragilityTest.java ########## @@ -0,0 +1,321 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.Queue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import org.apache.zookeeper.ClientCnxn.Packet; +import org.apache.zookeeper.Watcher.Event.KeeperState; +import org.apache.zookeeper.ZooDefs.Ids; +import org.apache.zookeeper.client.HostProvider; +import org.apache.zookeeper.client.ZKClientConfig; +import org.apache.zookeeper.data.Stat; +import org.apache.zookeeper.server.quorum.QuorumPeerTestBase; +import org.apache.zookeeper.test.ClientBase; +import org.junit.Assert; +import org.junit.Test; + +public class ClientCnxnSocketFragilityTest extends QuorumPeerTestBase { + + private static final int SERVER_COUNT = 3; + + private static final int SESSION_TIMEOUT = 40000; + + public static final int CONNECTION_TIMEOUT = 30000; + + private final UnsafeCoordinator unsafeCoordinator = new UnsafeCoordinator(); + + private volatile CustomZooKeeper zk = null; + + private volatile FragileClientCnxnSocketNIO socket = null; + + private volatile CustomClientCnxn cnxn = null; + + private String getCxnString(int[] clientPorts) { + StringBuffer hostPortBuffer = new StringBuffer(); + for (int i = 0; i < clientPorts.length; i++) { + hostPortBuffer.append("127.0.0.1:"); + hostPortBuffer.append(clientPorts[i]); + if (i != (clientPorts.length - 1)) { + hostPortBuffer.append(','); + } + } + return hostPortBuffer.toString(); + } + + private void closeZookeeper(ZooKeeper zk) { + Executors.newSingleThreadExecutor().submit(() -> { + try { + LOG.info("closeZookeeper is fired"); + zk.close(); + } catch (InterruptedException e) { + } + }); + } + + private void getDataBackgroundRetryForever(CustomZooKeeper zk, String path) { + new Thread(() -> { + for (;;) { + try { + zk.getData(path, false, new Stat()); + } catch (Exception e) { + } + } + }).start(); + } + + @Test + public void testClientCnxnSocketFragility() throws Exception { + System.setProperty(ZKClientConfig.ZOOKEEPER_CLIENT_CNXN_SOCKET, FragileClientCnxnSocketNIO.class.getName()); + System.setProperty(ZKClientConfig.ZOOKEEPER_REQUEST_TIMEOUT, "1000"); + final int[] clientPorts = new int[SERVER_COUNT]; + StringBuilder sb = new StringBuilder(); + String server; + + for (int i = 0; i < SERVER_COUNT; i++) { + clientPorts[i] = PortAssignment.unique(); + server = "server." + i + "=127.0.0.1:" + PortAssignment.unique() + ":" + PortAssignment.unique() + + ":participant;127.0.0.1:" + clientPorts[i]; + sb.append(server + "\n"); + } + String currentQuorumCfgSection = sb.toString(); + MainThread[] mt = new MainThread[SERVER_COUNT]; + + for (int i = 0; i < SERVER_COUNT; i++) { + mt[i] = new MainThread(i, clientPorts[i], currentQuorumCfgSection, false); + mt[i].start(); + } + + // Ensure server started + for (int i = 0; i < SERVER_COUNT; i++) { + Assert.assertTrue( + "waiting for server " + i + " being up", + ClientBase.waitForServerUp("127.0.0.1:" + clientPorts[i], CONNECTION_TIMEOUT)); + } + String path = "/testClientCnxnSocketFragility"; + String data = "balabala"; + ClientWatcher watcher = new ClientWatcher(); + zk = new CustomZooKeeper(getCxnString(clientPorts), SESSION_TIMEOUT, watcher); + watcher.watchFor(zk); + + // Let's see some successful operations + zk.create(path, data.getBytes(), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + Assert.assertEquals(new String(zk.getData(path, false, new Stat())), data); + Assert.assertTrue(!watcher.isSessionExpired()); + + // Let's make a broken operation + socket.mute(); + boolean catchKeeperException = false; + try { + zk.getData(path, false, new Stat()); + } catch (KeeperException e) { + catchKeeperException = true; + Assert.assertFalse(e instanceof KeeperException.SessionExpiredException); + } + socket.unmute(); + Assert.assertTrue(catchKeeperException); + Assert.assertTrue(!watcher.isSessionExpired()); + + getDataBackgroundRetryForever(zk, path); + // Let's make a broken network + socket.mute(); + + // Let's attempt to close ZooKeeper + cnxn.attemptClose(); + + // Wait some time to expect continuous reconnecting. + // We try to make reconnecting hit the unsafe point. + TimeUnit.MILLISECONDS.sleep(3000); Review comment: Is it possible to instead of using sleep, we do explicit synchronization and only execute when certain criteria is satisfied? We generally discourage the usage of sleep in zookeeper unit test as the use of sleep proved to be a source of flaky-ness in the past when running on different environment. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
